buckyos 0.7.165 → 0.7.168

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -481,7 +481,7 @@ function publishCommand(): CommandDefinition {
481
481
  property: 'app_type',
482
482
  description: 'App type; defaults to the template app_type',
483
483
  type: 'string',
484
- enum: ['dapp', 'web', 'agent'],
484
+ enum: ['dapp', 'web', 'agent', 'ext_tools'],
485
485
  },
486
486
  ],
487
487
  inputSchema: {
@@ -489,7 +489,7 @@ function publishCommand(): CommandDefinition {
489
489
  properties: {
490
490
  local_dir: { type: 'string', minLength: 1 },
491
491
  app_doc: { type: 'string', minLength: 1 },
492
- app_type: { type: 'string', enum: ['dapp', 'web', 'agent'] },
492
+ app_type: { type: 'string', enum: ['dapp', 'web', 'agent', 'ext_tools'] },
493
493
  },
494
494
  required: ['local_dir', 'app_doc'],
495
495
  additionalProperties: false,
@@ -502,6 +502,7 @@ function publishCommand(): CommandDefinition {
502
502
  examples: [
503
503
  'buckyos app publish /srv/build/demo-web --app-doc ./demo-web.appdoc.json',
504
504
  'buckyos app publish /srv/build/demo-agent --app-doc ./demo-agent.appdoc.json --app-type agent',
505
+ 'buckyos app publish /srv/build/ext-media --app-doc ./dapp_dist/APPDOC.json',
505
506
  ],
506
507
  handler: async (ctx, input) => {
507
508
  const localDir = expectString(input, 'local_dir')
@@ -591,6 +592,12 @@ async function installApp(
591
592
  ? await recomputePlan(ctx, source, input, submittedPlan)
592
593
  : await inspectSource(ctx, source, input)
593
594
  verifyPikgBinding(source, submittedPlan ?? inspection.plan)
595
+ if ((inspection.plan.app_doc as Record<string, unknown> | undefined)?.app_type === 'ext_tools') {
596
+ throw new ToolError(
597
+ 'EXT_TOOLS_NOT_AN_APP',
598
+ 'an extension tool package is not installed as an App: publish it to the Repo with `buckyos app publish`, then enable it for the Zone on the management page',
599
+ )
600
+ }
594
601
  const app = expectObject(inspection.plan.app, 'inspection.plan.app')
595
602
  const appDid = expectString(app, 'did')
596
603
  const plannedOwner = submittedPlan ? expectString(submittedPlan, 'owner_user_id') : undefined
@@ -14,6 +14,14 @@ import {
14
14
  DIST_MANIFEST_NAME,
15
15
  expectNonEmptyString,
16
16
  expectObject,
17
+ EXT_TOOLS_MANIFEST_FILE,
18
+ EXT_TOOLS_MANIFEST_SCHEMA,
19
+ EXT_TOOLS_PLATFORMS,
20
+ extToolsManifestErrors,
21
+ extToolsPayloadErrors,
22
+ extToolsPlatformOfKey,
23
+ type ExtToolsManifest,
24
+ type ExtToolsPayloadTree,
17
25
  inspectPikg,
18
26
  normalizeArch,
19
27
  PACKAGE_META_ENTRY,
@@ -56,14 +64,17 @@ const SHA256_ID = /^sha256:[0-9a-f]{64}$/
56
64
  const OBJECT_ID = /^(?:appdoc|pkg):[0-9a-f]{64}$/
57
65
  const SAFE_GENERATED_FILE = /^[A-Za-z0-9._-]+$/
58
66
  const OBJECT_OUTPUT = { type: 'object' as const, additionalProperties: true }
59
- const APP_KINDS = ['static-web', 'script', 'docker', 'agent-skills', 'aiws-controls']
67
+ const APP_KINDS = ['static-web', 'script', 'docker', 'agent-skills', 'aiws-controls', 'ext-tools']
60
68
  const PATH_KINDS: Record<string, { key: string; category: string; selectorType: string }> = {
61
69
  'static-web': { key: 'web', category: 'web', selectorType: 'static' },
62
70
  script: { key: 'script', category: 'dapp', selectorType: 'single' },
63
71
  'agent-skills': { key: 'agent_skills', category: 'agent_skills', selectorType: 'single' },
64
72
  // AI Workspace controls: one bundle (aiws-manifest.json + controls/<dir>/), installed into the AI Workspace library
65
73
  'aiws-controls': { key: 'aiws', category: 'aiws', selectorType: 'single' },
74
+ // Extension tools: ext-manifest.json + payload/<platform>/, one sub package per platform, enabled for the Zone
75
+ 'ext-tools': { key: 'ext_tools', category: 'ext_tools', selectorType: 'single' },
66
76
  }
77
+ const EXT_TOOLS_LAUNCHER_LIMIT = 1024 * 1024
67
78
 
68
79
  export interface DockerImageInfo {
69
80
  id: string
@@ -183,9 +194,19 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
183
194
  positionals: [
184
195
  { name: 'meta_dir', description: 'dapp_meta directory', required: false },
185
196
  ],
197
+ options: [
198
+ {
199
+ name: 'catalog',
200
+ description: 'ext-tools: a directory of other packages (<id>/ext-manifest.json) to check requires and same-named commands against',
201
+ type: 'string',
202
+ },
203
+ ],
186
204
  inputSchema: {
187
205
  type: 'object',
188
- properties: { meta_dir: { type: 'string', minLength: 1 } },
206
+ properties: {
207
+ meta_dir: { type: 'string', minLength: 1 },
208
+ catalog: { type: 'string', minLength: 1 },
209
+ },
189
210
  additionalProperties: false,
190
211
  },
191
212
  outputSchema: OBJECT_OUTPUT,
@@ -195,7 +216,11 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
195
216
  requiresSession: false,
196
217
  execution: 'local',
197
218
  networkAccess: false,
198
- examples: ['buckyos pikg doctor', 'buckyos pikg doctor ./dapp_meta'],
219
+ examples: [
220
+ 'buckyos pikg doctor',
221
+ 'buckyos pikg doctor ./dapp_meta',
222
+ 'buckyos pikg doctor ./dapp_meta --catalog ../publish/exttool',
223
+ ],
199
224
  handler: async (ctx, input) => await doctorCommand(ctx, input),
200
225
  },
201
226
  {
@@ -254,6 +279,7 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
254
279
  'buckyos --non-interactive pikg init . --owner did:bns:root --kind agent-skills --source ./skills',
255
280
  'buckyos --non-interactive pikg init . --owner did:bns:alice --kind aiws-controls --source ./aiws',
256
281
  'buckyos --non-interactive pikg init . --owner did:bns:alice --kind aiws-controls --source ./aiws --composite',
282
+ 'buckyos --non-interactive pikg init . --name ext-media --owner did:bns:root --kind ext-tools --source ./ext-media',
257
283
  ],
258
284
  handler: async (ctx, input) => await initCommand(ctx, input, docker),
259
285
  },
@@ -413,6 +439,10 @@ async function doctorCommand(
413
439
  status: missingSources.length === 0 ? 'ok' : 'missing',
414
440
  missing: missingSources,
415
441
  })
442
+ if (app.categories[0] === PATH_KINDS['ext-tools'].category) {
443
+ const catalog = optionalInputString(input, 'catalog')
444
+ checks.push(...await extToolsChecks(app, pikg, metaDir, catalog ? resolveFromCwd(ctx, catalog) : undefined))
445
+ }
416
446
  } catch (error) {
417
447
  checks.push({
418
448
  name: 'metadata_schema',
@@ -426,7 +456,7 @@ async function doctorCommand(
426
456
  const profile = profileName ? await ctx.configStore.readProfile(profileName) : undefined
427
457
  const host = getHost()
428
458
  return {
429
- healthy: checks.every((check) => check.status === 'ok'),
459
+ healthy: checks.every((check) => check.status === 'ok' || check.status === 'skipped'),
430
460
  tool_version: TOOL_VERSION,
431
461
  sdk_version: SDK_VERSION,
432
462
  protocol_version: PROTOCOL_VERSION,
@@ -480,7 +510,24 @@ async function initCommand(
480
510
  let sourceValue: Record<string, string>
481
511
  let selector: Record<string, string> | undefined
482
512
  let sourceSummary: Record<string, unknown>
483
- if (kind === 'docker') {
513
+ if (kind === 'ext-tools') {
514
+ const platform = hostExtToolsPlatform()
515
+ const absoluteSource = resolve(projectDir, source)
516
+ subpackageKey = platform.key
517
+ selector = platform.selector
518
+ sourceValue = {
519
+ type: 'path',
520
+ path: toPortablePath(relative(join(projectDir, 'dapp_meta'), join(absoluteSource, 'payload', platform.name))),
521
+ }
522
+ sourceSummary = { type: 'path', path: displaySource(projectDir, absoluteSource), platform: platform.name }
523
+ try {
524
+ await getHost().lstat(join(absoluteSource, EXT_TOOLS_MANIFEST_FILE))
525
+ } catch (error) {
526
+ if (!isHostError(error, 'NotFound')) throw error
527
+ await writeExtToolsSkeleton(absoluteSource, name, version, platform.name)
528
+ sourceSummary.skeleton = true
529
+ }
530
+ } else if (kind === 'docker') {
484
531
  const image = await docker.inspect(source)
485
532
  const arch = normalizeArch(image.architecture)
486
533
  subpackageKey = arch === 'x86_64'
@@ -592,6 +639,205 @@ async function writeAiwsSkeleton(dir: string, ownerDid: string, name: string, co
592
639
  await host.writeTextFile(join(dir, AIWS_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`)
593
640
  }
594
641
 
642
+ function hostExtToolsPlatform(): (typeof EXT_TOOLS_PLATFORMS)[number] {
643
+ const host = getHost()
644
+ const os = host.platform === 'darwin' ? 'macos' : host.platform === 'win32' ? 'windows' : host.platform
645
+ const arch = normalizeArch(host.arch)
646
+ const platform = EXT_TOOLS_PLATFORMS.find((known) => known.selector.os === os && known.selector.arch === arch)
647
+ if (!platform) {
648
+ throw new UsageError('UNSUPPORTED_PLATFORM', `extension packages have no payload for ${os}/${arch}`)
649
+ }
650
+ return platform
651
+ }
652
+
653
+ /** An extension package skeleton for this machine's platform: `ext-manifest.json` and one command that runs and has a
654
+ * smoke script, so the package builds from the first minute. Other platforms are added beside it. */
655
+ async function writeExtToolsSkeleton(dir: string, id: string, version: string, platform: string): Promise<void> {
656
+ const host = getHost()
657
+ const manifest = {
658
+ schema: EXT_TOOLS_MANIFEST_SCHEMA,
659
+ id,
660
+ version,
661
+ requires: [],
662
+ conflicts: [],
663
+ provides: [{ command: id, version_probe: `${id} --version`, smoke: `smoke/${id}.sh` }],
664
+ platforms: [platform],
665
+ license: '',
666
+ desc: '',
667
+ }
668
+ const text = `${JSON.stringify(manifest, null, 2)}\n`
669
+ const payload = join(dir, 'payload', platform)
670
+ for (const sub of ['bin', 'smoke', join('store', id)]) await host.mkdir(join(payload, sub), { recursive: true })
671
+ await host.writeTextFile(join(dir, EXT_TOOLS_MANIFEST_FILE), text)
672
+ await host.writeTextFile(join(payload, EXT_TOOLS_MANIFEST_FILE), text)
673
+ const windows = platform.startsWith('windows-')
674
+ await host.writeTextFile(
675
+ join(payload, 'bin', windows ? `${id}.cmd` : id),
676
+ windows ? `@echo off\r\necho ${id} ${version}\r\n` : `#!/bin/sh\necho "${id} ${version}"\n`,
677
+ { mode: 0o755 },
678
+ )
679
+ await host.writeTextFile(join(payload, 'smoke', `${id}.sh`), `#!/bin/sh\nset -eu\n${id} --version >/dev/null\n`, { mode: 0o755 })
680
+ }
681
+
682
+ async function readExtToolsManifestFile(path: string): Promise<{ text: string; manifest?: ExtToolsManifest; errors: string[] }> {
683
+ let text: string
684
+ try {
685
+ text = await getHost().readTextFile(path)
686
+ } catch (error) {
687
+ if (isHostError(error, 'NotFound')) return { text: '', errors: ['not found'] }
688
+ throw error
689
+ }
690
+ let value: unknown
691
+ try {
692
+ value = JSON.parse(text)
693
+ } catch {
694
+ return { text, errors: ['not valid JSON'] }
695
+ }
696
+ const errors = extToolsManifestErrors(value)
697
+ if (errors.length > 0) return { text, errors }
698
+ const manifest = value as ExtToolsManifest
699
+ manifest.requires ??= []
700
+ manifest.conflicts ??= []
701
+ return { text, manifest, errors }
702
+ }
703
+
704
+ /** A payload directory as `pikg build` would archive it: symlinks read through, and refused when they leave the tree. */
705
+ async function readExtToolsPayloadTree(root: string): Promise<{ tree: ExtToolsPayloadTree; errors: string[] }> {
706
+ const host = getHost()
707
+ const tree: ExtToolsPayloadTree = { files: new Map(), directories: new Set(), launchers: new Map() }
708
+ const errors: string[] = []
709
+ const realRoot = await host.realPath(root)
710
+ const walk = async (directory: string, prefix: string): Promise<void> => {
711
+ const children = await host.readDir(directory)
712
+ for (const child of children) {
713
+ const archivePath = prefix ? `${prefix}/${child.name}` : child.name
714
+ const path = join(directory, child.name)
715
+ const link = await host.lstat(path)
716
+ let physical = path
717
+ if (link.isSymlink) {
718
+ try {
719
+ physical = await host.realPath(path)
720
+ } catch (error) {
721
+ errors.push(`${archivePath}: ${isHostError(error, 'NotFound') ? 'a dangling symlink' : 'a symlink out of the payload'}`)
722
+ continue
723
+ }
724
+ const inside = relative(realRoot, physical)
725
+ if (inside === '..' || inside.startsWith(`..${host.path.sep}`) || isAbsolute(inside)) {
726
+ errors.push(`${archivePath}: a symlink out of the payload`)
727
+ continue
728
+ }
729
+ }
730
+ const info = link.isSymlink ? await host.stat(physical) : link
731
+ if (info.isDirectory) {
732
+ tree.directories.add(archivePath)
733
+ await walk(physical, archivePath)
734
+ } else if (info.isFile) {
735
+ tree.files.set(archivePath, { executable: ((info.mode ?? 0) & 0o111) !== 0 })
736
+ if (archivePath.startsWith('bin/') && info.size <= EXT_TOOLS_LAUNCHER_LIMIT) {
737
+ tree.launchers.set(archivePath, await host.readFile(physical))
738
+ }
739
+ } else {
740
+ errors.push(`${archivePath}: not a regular file`)
741
+ }
742
+ }
743
+ }
744
+ await walk(realRoot, '')
745
+ return { tree, errors }
746
+ }
747
+
748
+ /** `pikg doctor` of an extension package: the manifest against the App, every platform's payload, and, given a catalog
749
+ * of the other packages, `requires` and same-named commands. */
750
+ async function extToolsChecks(app: AppMeta, pikg: PikgMeta, metaDir: string, catalogDir: string | undefined): Promise<Array<Record<string, unknown>>> {
751
+ const checks: Array<Record<string, unknown>> = []
752
+ const platforms = new Map<string, string>()
753
+ const stray: string[] = []
754
+ for (const [key, subpackage] of Object.entries(pikg.sub_pkgs)) {
755
+ const platform = extToolsPlatformOfKey(key)
756
+ if (!platform || subpackage.source.type !== 'path') {
757
+ stray.push(key)
758
+ continue
759
+ }
760
+ platforms.set(platform.name, resolve(metaDir, subpackage.source.path))
761
+ }
762
+ const sources = [...platforms.values()]
763
+ // <source>/payload/<platform>: the package's manifest is two levels up
764
+ const source = sources.length > 0 ? dirname(dirname(sources[0])) : dirname(metaDir)
765
+ const master = await readExtToolsManifestFile(join(source, EXT_TOOLS_MANIFEST_FILE))
766
+ const manifestErrors = [...master.errors]
767
+ if (stray.length > 0) manifestErrors.push(`sub packages must be ext_tools_<os>_<arch> from a directory: ${stray.join(', ')}`)
768
+ const manifest = master.manifest
769
+ if (manifest) {
770
+ if (manifest.id !== app.name) manifestErrors.push(`id ${manifest.id} is not the App name ${app.name}`)
771
+ if (manifest.version !== app.version) manifestErrors.push(`version ${manifest.version} is not the App version ${app.version}`)
772
+ const without = manifest.platforms.filter((name) => !platforms.has(name))
773
+ if (without.length > 0) manifestErrors.push(`platforms without a sub package: ${without.join(', ')}`)
774
+ const unlisted = [...platforms.keys()].filter((name) => !manifest.platforms.includes(name))
775
+ if (unlisted.length > 0) manifestErrors.push(`sub packages of platforms it does not list: ${unlisted.join(', ')}`)
776
+ }
777
+ checks.push({
778
+ name: 'ext_tools_manifest',
779
+ status: manifestErrors.length === 0 ? 'ok' : 'error',
780
+ path: join(source, EXT_TOOLS_MANIFEST_FILE),
781
+ ...(manifestErrors.length > 0 ? { errors: manifestErrors } : {}),
782
+ })
783
+ if (!manifest) return checks
784
+ for (const [platform, dir] of platforms) {
785
+ const errors: string[] = []
786
+ try {
787
+ const { tree, errors: walkErrors } = await readExtToolsPayloadTree(dir)
788
+ errors.push(...walkErrors)
789
+ const copy = await readExtToolsManifestFile(join(dir, EXT_TOOLS_MANIFEST_FILE))
790
+ if (copy.errors.length > 0) errors.push(`${EXT_TOOLS_MANIFEST_FILE}: ${copy.errors.join('; ')}`)
791
+ else if (copy.text !== master.text) errors.push(`${EXT_TOOLS_MANIFEST_FILE}: differs from the package's manifest`)
792
+ errors.push(...extToolsPayloadErrors(manifest, platform, tree, [source, metaDir]))
793
+ } catch (error) {
794
+ if (!isHostError(error, 'NotFound')) throw error
795
+ errors.push(`${dir}: not found`)
796
+ }
797
+ checks.push({
798
+ name: 'ext_tools_payload',
799
+ platform,
800
+ status: errors.length === 0 ? 'ok' : 'error',
801
+ path: dir,
802
+ ...(errors.length > 0 ? { errors } : {}),
803
+ })
804
+ }
805
+ if (!catalogDir) {
806
+ checks.push({
807
+ name: 'ext_tools_catalog',
808
+ status: 'skipped',
809
+ detail: 'no --catalog: requires and commands other packages also provide are not checked',
810
+ })
811
+ return checks
812
+ }
813
+ const others = new Map<string, ExtToolsManifest>()
814
+ const catalogErrors: string[] = []
815
+ for (const entry of await getHost().readDir(catalogDir)) {
816
+ if (entry.isFile) continue
817
+ const candidate = await readExtToolsManifestFile(join(catalogDir, entry.name, EXT_TOOLS_MANIFEST_FILE))
818
+ if (candidate.errors.includes('not found')) continue
819
+ if (!candidate.manifest) catalogErrors.push(`${entry.name}: ${candidate.errors.join('; ')}`)
820
+ else if (candidate.manifest.id !== manifest.id) others.set(candidate.manifest.id, candidate.manifest)
821
+ }
822
+ for (const required of manifest.requires) {
823
+ if (!others.has(required.id)) catalogErrors.push(`requires ${required.id}: not in the catalog`)
824
+ }
825
+ const commands = new Set(manifest.provides.map((provided) => provided.command))
826
+ for (const other of others.values()) {
827
+ if (manifest.conflicts.includes(other.id) || other.conflicts.includes(manifest.id)) continue
828
+ const shared = other.provides.map((provided) => provided.command).filter((command) => commands.has(command))
829
+ if (shared.length > 0) catalogErrors.push(`${other.id} also provides ${shared.join(', ')}: one of the two must list the other in conflicts`)
830
+ }
831
+ checks.push({
832
+ name: 'ext_tools_catalog',
833
+ status: catalogErrors.length === 0 ? 'ok' : 'error',
834
+ path: catalogDir,
835
+ packages: [...others.keys()],
836
+ ...(catalogErrors.length > 0 ? { errors: catalogErrors } : {}),
837
+ })
838
+ return checks
839
+ }
840
+
595
841
  async function buildCommand(
596
842
  ctx: CommandContext,
597
843
  input: Record<string, unknown>,
@@ -144,51 +144,164 @@ export interface TarFileEntry {
144
144
 
145
145
  export async function readTarGz(bytes: Uint8Array, limit: number): Promise<TarFileEntry[]> {
146
146
  const tar = await gunzipBytes(bytes, limit)
147
- const decoder = new TextDecoder('utf-8', { fatal: true })
148
- const text = (offset: number, length: number) =>
149
- decoder.decode(tar.subarray(offset, offset + length)).replace(/\0.*$/s, '')
150
147
  const files: TarFileEntry[] = []
151
148
  const seen = new Set<string>()
152
149
  let offset = 0
153
150
  while (
154
151
  offset + BLOCK_SIZE <= tar.length && tar.subarray(offset, offset + BLOCK_SIZE).some(Boolean)
155
152
  ) {
156
- let raw: string
157
- let size: number
158
- let mode: number
159
- try {
160
- const prefix = text(offset + 257, 6) === 'ustar' ? text(offset + 345, 155) : ''
161
- raw = prefix ? `${prefix}/${text(offset, 100)}` : text(offset, 100)
162
- size = Number.parseInt(text(offset + 124, 12).trim() || '0', 8)
163
- mode = Number.parseInt(text(offset + 100, 8).trim() || '0', 8)
164
- } catch {
165
- throw invalidZip('payload', 'tar header is not valid UTF-8')
166
- }
167
- const type = String.fromCharCode(tar[offset + 156] || 0x30)
168
- const path = raw.replace(/^(?:\.\/)+/, '').replace(/\/$/, '')
153
+ const header = parseTarHeader(tar.subarray(offset, offset + BLOCK_SIZE))
169
154
  const start = offset + BLOCK_SIZE
170
- if (!Number.isSafeInteger(size) || size < 0 || start + size > tar.length) {
171
- throw invalidZip('payload', `tar entry is truncated: ${raw}`)
172
- }
173
- offset = start + Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE
174
- if (type === '5' && (path === '' || path === '.')) continue
175
- if (
176
- !path || path.startsWith('/') || /^[A-Za-z]:/.test(path) || path.includes('\\') ||
177
- path.split('/').some((segment) => !segment || segment === '.' || segment === '..')
178
- ) {
179
- throw invalidZip('payload', `unsafe tar entry path: ${JSON.stringify(raw)}`)
180
- }
181
- if (type === '5') continue
182
- if (type !== '0') {
183
- throw invalidZip('payload', `tar entry is not a regular file or directory: ${path}`)
155
+ if (start + header.size > tar.length) {
156
+ throw invalidZip('payload', `tar entry is truncated: ${header.path}`)
184
157
  }
185
- if (seen.has(path)) throw invalidZip('payload', `duplicate tar entry: ${path}`)
186
- seen.add(path)
187
- files.push({ path, executable: (mode & 0o111) !== 0, bytes: tar.slice(start, start + size) })
158
+ offset = start + Math.ceil(header.size / BLOCK_SIZE) * BLOCK_SIZE
159
+ if (header.kind !== 'file') continue
160
+ if (seen.has(header.path)) throw invalidZip('payload', `duplicate tar entry: ${header.path}`)
161
+ seen.add(header.path)
162
+ files.push({ path: header.path, executable: header.executable, bytes: tar.slice(start, start + header.size) })
188
163
  }
189
164
  return files
190
165
  }
191
166
 
167
+ interface TarHeader {
168
+ path: string
169
+ kind: 'file' | 'directory' | 'root'
170
+ size: number
171
+ executable: boolean
172
+ }
173
+
174
+ function parseTarHeader(block: Uint8Array): TarHeader {
175
+ const decoder = new TextDecoder('utf-8', { fatal: true })
176
+ const text = (offset: number, length: number) =>
177
+ decoder.decode(block.subarray(offset, offset + length)).replace(/\0.*$/s, '')
178
+ let raw: string
179
+ let size: number
180
+ let mode: number
181
+ try {
182
+ const prefix = text(257, 6) === 'ustar' ? text(345, 155) : ''
183
+ raw = prefix ? `${prefix}/${text(0, 100)}` : text(0, 100)
184
+ size = Number.parseInt(text(124, 12).trim() || '0', 8)
185
+ mode = Number.parseInt(text(100, 8).trim() || '0', 8)
186
+ } catch {
187
+ throw invalidZip('payload', 'tar header is not valid UTF-8')
188
+ }
189
+ if (!Number.isSafeInteger(size) || size < 0) {
190
+ throw invalidZip('payload', `tar entry is truncated: ${raw}`)
191
+ }
192
+ const type = String.fromCharCode(block[156] || 0x30)
193
+ const path = raw.replace(/^(?:\.\/)+/, '').replace(/\/$/, '')
194
+ if (type === '5' && (path === '' || path === '.')) {
195
+ return { path, kind: 'root', size, executable: false }
196
+ }
197
+ if (
198
+ !path || path.startsWith('/') || /^[A-Za-z]:/.test(path) || path.includes('\\') ||
199
+ path.split('/').some((segment) => !segment || segment === '.' || segment === '..')
200
+ ) {
201
+ throw invalidZip('payload', `unsafe tar entry path: ${JSON.stringify(raw)}`)
202
+ }
203
+ if (type !== '5' && type !== '0') {
204
+ throw invalidZip('payload', `tar entry is not a regular file or directory: ${path}`)
205
+ }
206
+ return { path, kind: type === '5' ? 'directory' : 'file', size, executable: (mode & 0o111) !== 0 }
207
+ }
208
+
209
+ export interface TarListing {
210
+ files: Map<string, { size: number; executable: boolean }>
211
+ directories: Set<string>
212
+ /** The contents of the files `capture` asked for. */
213
+ captured: Map<string, Uint8Array>
214
+ }
215
+
216
+ /** List a tar.gz too large to hold: a package's payload is gigabytes, and only its few small files need reading. */
217
+ export async function listTarGzStream(
218
+ gzip: ReadableStream<Uint8Array>,
219
+ limit: number,
220
+ capture: (path: string, size: number) => boolean,
221
+ ): Promise<TarListing> {
222
+ const listing: TarListing = { files: new Map(), directories: new Set(), captured: new Map() }
223
+ const header = new Uint8Array(BLOCK_SIZE)
224
+ let headerFill = 0
225
+ let skip = 0
226
+ let body: { path: string; bytes: Uint8Array; fill: number } | undefined
227
+ let ended = false
228
+ let total = 0
229
+ try {
230
+ const stream = gzip.pipeThrough(
231
+ new DecompressionStream('gzip') as unknown as ReadableWritablePair<Uint8Array, Uint8Array>,
232
+ )
233
+ for await (const chunk of stream) {
234
+ total += chunk.length
235
+ if (total > limit) throw invalidZip('limits', 'payload expands beyond the size limit')
236
+ let offset = 0
237
+ while (offset < chunk.length && !ended) {
238
+ if (skip > 0) {
239
+ const take = Math.min(skip, chunk.length - offset)
240
+ if (body && body.fill < body.bytes.length) {
241
+ const part = Math.min(take, body.bytes.length - body.fill)
242
+ body.bytes.set(chunk.subarray(offset, offset + part), body.fill)
243
+ body.fill += part
244
+ if (body.fill === body.bytes.length) {
245
+ listing.captured.set(body.path, body.bytes)
246
+ body = undefined
247
+ }
248
+ }
249
+ skip -= take
250
+ offset += take
251
+ continue
252
+ }
253
+ const take = Math.min(BLOCK_SIZE - headerFill, chunk.length - offset)
254
+ header.set(chunk.subarray(offset, offset + take), headerFill)
255
+ headerFill += take
256
+ offset += take
257
+ if (headerFill < BLOCK_SIZE) break
258
+ headerFill = 0
259
+ if (!header.some(Boolean)) {
260
+ ended = true
261
+ break
262
+ }
263
+ const entry = parseTarHeader(header)
264
+ skip = Math.ceil(entry.size / BLOCK_SIZE) * BLOCK_SIZE
265
+ if (entry.kind === 'directory') listing.directories.add(entry.path)
266
+ if (entry.kind !== 'file') continue
267
+ if (listing.files.has(entry.path)) {
268
+ throw invalidZip('payload', `duplicate tar entry: ${entry.path}`)
269
+ }
270
+ listing.files.set(entry.path, { size: entry.size, executable: entry.executable })
271
+ if (capture(entry.path, entry.size)) {
272
+ if (entry.size === 0) listing.captured.set(entry.path, new Uint8Array())
273
+ else body = { path: entry.path, bytes: new Uint8Array(entry.size), fill: 0 }
274
+ }
275
+ }
276
+ }
277
+ } catch (error) {
278
+ if (error instanceof ToolError) throw error
279
+ throw invalidZip('compression', 'payload is not a valid gzip stream')
280
+ }
281
+ if (skip > 0 || headerFill > 0 || body) throw invalidZip('payload', 'tar stream is truncated')
282
+ return listing
283
+ }
284
+
285
+ export function fileReadableStream(path: string): ReadableStream<Uint8Array> {
286
+ let file: HostFile | undefined
287
+ return new ReadableStream<Uint8Array>({
288
+ async pull(controller) {
289
+ file ??= await getHost().open(path, { read: true })
290
+ const buffer = new Uint8Array(256 * 1024)
291
+ const read = await file.read(buffer)
292
+ if (read === null) {
293
+ await file.close()
294
+ controller.close()
295
+ return
296
+ }
297
+ controller.enqueue(buffer.subarray(0, read))
298
+ },
299
+ async cancel() {
300
+ await file?.close()
301
+ },
302
+ })
303
+ }
304
+
192
305
  async function gunzipBytes(bytes: Uint8Array, limit: number): Promise<Uint8Array> {
193
306
  const stream = new ReadableStream<Uint8Array>({
194
307
  start(controller) {
@@ -628,7 +741,7 @@ function createEntryVerification(entry: ZipEntry): {
628
741
  }
629
742
  }
630
743
 
631
- function zipEntryStream(path: string, entry: ZipEntry): ReadableStream<Uint8Array> {
744
+ export function zipEntryStream(path: string, entry: ZipEntry): ReadableStream<Uint8Array> {
632
745
  let file: HostFile | undefined
633
746
  let remaining = entry.compressedSize
634
747
  const compressed = new ReadableStream<Uint8Array>({