buckyos 0.7.164 → 0.7.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/dist/cli.mjs +590 -45
- package/cli/dist/cli.mjs.map +1 -1
- package/cli/modules/app.ts +9 -2
- package/cli/modules/pikg.ts +251 -5
- package/cli/modules/pikg_archive.ts +147 -34
- package/cli/modules/pikg_protocol.ts +207 -3
- package/cli/runtime/host.ts +2 -0
- package/cli/version.ts +1 -1
- package/dist/aicc_client.d.ts +1 -1
- package/dist/aicc_client.d.ts.map +1 -1
- package/dist/browser.cjs +1 -1
- package/dist/browser.mjs +2 -2
- package/dist/bucket_client.d.ts +13 -1
- package/dist/bucket_client.d.ts.map +1 -1
- package/dist/{executor_client-86315a2f.mjs → executor_client-a32bc377.mjs} +28 -1
- package/dist/executor_client-a32bc377.mjs.map +1 -0
- package/dist/{executor_client-8a9add15.js → executor_client-e026942b.js} +2 -2
- package/dist/executor_client-e026942b.js.map +1 -0
- package/dist/index.cjs +1 -1
- package/dist/index.mjs +2 -2
- package/dist/node.cjs +1 -1
- package/dist/node.mjs +2 -2
- package/dist/object_url.d.ts +10 -0
- package/dist/object_url.d.ts.map +1 -1
- package/dist/workflow_client.d.ts +11 -2
- package/dist/workflow_client.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/executor_client-86315a2f.mjs.map +0 -1
- package/dist/executor_client-8a9add15.js.map +0 -1
package/cli/modules/app.ts
CHANGED
|
@@ -481,7 +481,7 @@ function publishCommand(): CommandDefinition {
|
|
|
481
481
|
property: 'app_type',
|
|
482
482
|
description: 'App type; defaults to the template app_type',
|
|
483
483
|
type: 'string',
|
|
484
|
-
enum: ['dapp', 'web', 'agent'],
|
|
484
|
+
enum: ['dapp', 'web', 'agent', 'ext_tools'],
|
|
485
485
|
},
|
|
486
486
|
],
|
|
487
487
|
inputSchema: {
|
|
@@ -489,7 +489,7 @@ function publishCommand(): CommandDefinition {
|
|
|
489
489
|
properties: {
|
|
490
490
|
local_dir: { type: 'string', minLength: 1 },
|
|
491
491
|
app_doc: { type: 'string', minLength: 1 },
|
|
492
|
-
app_type: { type: 'string', enum: ['dapp', 'web', 'agent'] },
|
|
492
|
+
app_type: { type: 'string', enum: ['dapp', 'web', 'agent', 'ext_tools'] },
|
|
493
493
|
},
|
|
494
494
|
required: ['local_dir', 'app_doc'],
|
|
495
495
|
additionalProperties: false,
|
|
@@ -502,6 +502,7 @@ function publishCommand(): CommandDefinition {
|
|
|
502
502
|
examples: [
|
|
503
503
|
'buckyos app publish /srv/build/demo-web --app-doc ./demo-web.appdoc.json',
|
|
504
504
|
'buckyos app publish /srv/build/demo-agent --app-doc ./demo-agent.appdoc.json --app-type agent',
|
|
505
|
+
'buckyos app publish /srv/build/ext-media --app-doc ./dapp_dist/APPDOC.json',
|
|
505
506
|
],
|
|
506
507
|
handler: async (ctx, input) => {
|
|
507
508
|
const localDir = expectString(input, 'local_dir')
|
|
@@ -591,6 +592,12 @@ async function installApp(
|
|
|
591
592
|
? await recomputePlan(ctx, source, input, submittedPlan)
|
|
592
593
|
: await inspectSource(ctx, source, input)
|
|
593
594
|
verifyPikgBinding(source, submittedPlan ?? inspection.plan)
|
|
595
|
+
if ((inspection.plan.app_doc as Record<string, unknown> | undefined)?.app_type === 'ext_tools') {
|
|
596
|
+
throw new ToolError(
|
|
597
|
+
'EXT_TOOLS_NOT_AN_APP',
|
|
598
|
+
'an extension tool package is not installed as an App: publish it to the Repo with `buckyos app publish`, then enable it for the Zone on the management page',
|
|
599
|
+
)
|
|
600
|
+
}
|
|
594
601
|
const app = expectObject(inspection.plan.app, 'inspection.plan.app')
|
|
595
602
|
const appDid = expectString(app, 'did')
|
|
596
603
|
const plannedOwner = submittedPlan ? expectString(submittedPlan, 'owner_user_id') : undefined
|
package/cli/modules/pikg.ts
CHANGED
|
@@ -14,6 +14,14 @@ import {
|
|
|
14
14
|
DIST_MANIFEST_NAME,
|
|
15
15
|
expectNonEmptyString,
|
|
16
16
|
expectObject,
|
|
17
|
+
EXT_TOOLS_MANIFEST_FILE,
|
|
18
|
+
EXT_TOOLS_MANIFEST_SCHEMA,
|
|
19
|
+
EXT_TOOLS_PLATFORMS,
|
|
20
|
+
extToolsManifestErrors,
|
|
21
|
+
extToolsPayloadErrors,
|
|
22
|
+
extToolsPlatformOfKey,
|
|
23
|
+
type ExtToolsManifest,
|
|
24
|
+
type ExtToolsPayloadTree,
|
|
17
25
|
inspectPikg,
|
|
18
26
|
normalizeArch,
|
|
19
27
|
PACKAGE_META_ENTRY,
|
|
@@ -56,14 +64,17 @@ const SHA256_ID = /^sha256:[0-9a-f]{64}$/
|
|
|
56
64
|
const OBJECT_ID = /^(?:appdoc|pkg):[0-9a-f]{64}$/
|
|
57
65
|
const SAFE_GENERATED_FILE = /^[A-Za-z0-9._-]+$/
|
|
58
66
|
const OBJECT_OUTPUT = { type: 'object' as const, additionalProperties: true }
|
|
59
|
-
const APP_KINDS = ['static-web', 'script', 'docker', 'agent-skills', 'aiws-controls']
|
|
67
|
+
const APP_KINDS = ['static-web', 'script', 'docker', 'agent-skills', 'aiws-controls', 'ext-tools']
|
|
60
68
|
const PATH_KINDS: Record<string, { key: string; category: string; selectorType: string }> = {
|
|
61
69
|
'static-web': { key: 'web', category: 'web', selectorType: 'static' },
|
|
62
70
|
script: { key: 'script', category: 'dapp', selectorType: 'single' },
|
|
63
71
|
'agent-skills': { key: 'agent_skills', category: 'agent_skills', selectorType: 'single' },
|
|
64
72
|
// AI Workspace controls: one bundle (aiws-manifest.json + controls/<dir>/), installed into the AI Workspace library
|
|
65
73
|
'aiws-controls': { key: 'aiws', category: 'aiws', selectorType: 'single' },
|
|
74
|
+
// Extension tools: ext-manifest.json + payload/<platform>/, one sub package per platform, enabled for the Zone
|
|
75
|
+
'ext-tools': { key: 'ext_tools', category: 'ext_tools', selectorType: 'single' },
|
|
66
76
|
}
|
|
77
|
+
const EXT_TOOLS_LAUNCHER_LIMIT = 1024 * 1024
|
|
67
78
|
|
|
68
79
|
export interface DockerImageInfo {
|
|
69
80
|
id: string
|
|
@@ -183,9 +194,19 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
|
|
|
183
194
|
positionals: [
|
|
184
195
|
{ name: 'meta_dir', description: 'dapp_meta directory', required: false },
|
|
185
196
|
],
|
|
197
|
+
options: [
|
|
198
|
+
{
|
|
199
|
+
name: 'catalog',
|
|
200
|
+
description: 'ext-tools: a directory of other packages (<id>/ext-manifest.json) to check requires and same-named commands against',
|
|
201
|
+
type: 'string',
|
|
202
|
+
},
|
|
203
|
+
],
|
|
186
204
|
inputSchema: {
|
|
187
205
|
type: 'object',
|
|
188
|
-
properties: {
|
|
206
|
+
properties: {
|
|
207
|
+
meta_dir: { type: 'string', minLength: 1 },
|
|
208
|
+
catalog: { type: 'string', minLength: 1 },
|
|
209
|
+
},
|
|
189
210
|
additionalProperties: false,
|
|
190
211
|
},
|
|
191
212
|
outputSchema: OBJECT_OUTPUT,
|
|
@@ -195,7 +216,11 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
|
|
|
195
216
|
requiresSession: false,
|
|
196
217
|
execution: 'local',
|
|
197
218
|
networkAccess: false,
|
|
198
|
-
examples: [
|
|
219
|
+
examples: [
|
|
220
|
+
'buckyos pikg doctor',
|
|
221
|
+
'buckyos pikg doctor ./dapp_meta',
|
|
222
|
+
'buckyos pikg doctor ./dapp_meta --catalog ../publish/exttool',
|
|
223
|
+
],
|
|
199
224
|
handler: async (ctx, input) => await doctorCommand(ctx, input),
|
|
200
225
|
},
|
|
201
226
|
{
|
|
@@ -254,6 +279,7 @@ export function createPikgModule(dependencies: PikgModuleDependencies = {}): Com
|
|
|
254
279
|
'buckyos --non-interactive pikg init . --owner did:bns:root --kind agent-skills --source ./skills',
|
|
255
280
|
'buckyos --non-interactive pikg init . --owner did:bns:alice --kind aiws-controls --source ./aiws',
|
|
256
281
|
'buckyos --non-interactive pikg init . --owner did:bns:alice --kind aiws-controls --source ./aiws --composite',
|
|
282
|
+
'buckyos --non-interactive pikg init . --name ext-media --owner did:bns:root --kind ext-tools --source ./ext-media',
|
|
257
283
|
],
|
|
258
284
|
handler: async (ctx, input) => await initCommand(ctx, input, docker),
|
|
259
285
|
},
|
|
@@ -413,6 +439,10 @@ async function doctorCommand(
|
|
|
413
439
|
status: missingSources.length === 0 ? 'ok' : 'missing',
|
|
414
440
|
missing: missingSources,
|
|
415
441
|
})
|
|
442
|
+
if (app.categories[0] === PATH_KINDS['ext-tools'].category) {
|
|
443
|
+
const catalog = optionalInputString(input, 'catalog')
|
|
444
|
+
checks.push(...await extToolsChecks(app, pikg, metaDir, catalog ? resolveFromCwd(ctx, catalog) : undefined))
|
|
445
|
+
}
|
|
416
446
|
} catch (error) {
|
|
417
447
|
checks.push({
|
|
418
448
|
name: 'metadata_schema',
|
|
@@ -426,7 +456,7 @@ async function doctorCommand(
|
|
|
426
456
|
const profile = profileName ? await ctx.configStore.readProfile(profileName) : undefined
|
|
427
457
|
const host = getHost()
|
|
428
458
|
return {
|
|
429
|
-
healthy: checks.every((check) => check.status === 'ok'),
|
|
459
|
+
healthy: checks.every((check) => check.status === 'ok' || check.status === 'skipped'),
|
|
430
460
|
tool_version: TOOL_VERSION,
|
|
431
461
|
sdk_version: SDK_VERSION,
|
|
432
462
|
protocol_version: PROTOCOL_VERSION,
|
|
@@ -480,7 +510,24 @@ async function initCommand(
|
|
|
480
510
|
let sourceValue: Record<string, string>
|
|
481
511
|
let selector: Record<string, string> | undefined
|
|
482
512
|
let sourceSummary: Record<string, unknown>
|
|
483
|
-
if (kind === '
|
|
513
|
+
if (kind === 'ext-tools') {
|
|
514
|
+
const platform = hostExtToolsPlatform()
|
|
515
|
+
const absoluteSource = resolve(projectDir, source)
|
|
516
|
+
subpackageKey = platform.key
|
|
517
|
+
selector = platform.selector
|
|
518
|
+
sourceValue = {
|
|
519
|
+
type: 'path',
|
|
520
|
+
path: toPortablePath(relative(join(projectDir, 'dapp_meta'), join(absoluteSource, 'payload', platform.name))),
|
|
521
|
+
}
|
|
522
|
+
sourceSummary = { type: 'path', path: displaySource(projectDir, absoluteSource), platform: platform.name }
|
|
523
|
+
try {
|
|
524
|
+
await getHost().lstat(join(absoluteSource, EXT_TOOLS_MANIFEST_FILE))
|
|
525
|
+
} catch (error) {
|
|
526
|
+
if (!isHostError(error, 'NotFound')) throw error
|
|
527
|
+
await writeExtToolsSkeleton(absoluteSource, name, version, platform.name)
|
|
528
|
+
sourceSummary.skeleton = true
|
|
529
|
+
}
|
|
530
|
+
} else if (kind === 'docker') {
|
|
484
531
|
const image = await docker.inspect(source)
|
|
485
532
|
const arch = normalizeArch(image.architecture)
|
|
486
533
|
subpackageKey = arch === 'x86_64'
|
|
@@ -592,6 +639,205 @@ async function writeAiwsSkeleton(dir: string, ownerDid: string, name: string, co
|
|
|
592
639
|
await host.writeTextFile(join(dir, AIWS_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`)
|
|
593
640
|
}
|
|
594
641
|
|
|
642
|
+
function hostExtToolsPlatform(): (typeof EXT_TOOLS_PLATFORMS)[number] {
|
|
643
|
+
const host = getHost()
|
|
644
|
+
const os = host.platform === 'darwin' ? 'macos' : host.platform === 'win32' ? 'windows' : host.platform
|
|
645
|
+
const arch = normalizeArch(host.arch)
|
|
646
|
+
const platform = EXT_TOOLS_PLATFORMS.find((known) => known.selector.os === os && known.selector.arch === arch)
|
|
647
|
+
if (!platform) {
|
|
648
|
+
throw new UsageError('UNSUPPORTED_PLATFORM', `extension packages have no payload for ${os}/${arch}`)
|
|
649
|
+
}
|
|
650
|
+
return platform
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/** An extension package skeleton for this machine's platform: `ext-manifest.json` and one command that runs and has a
|
|
654
|
+
* smoke script, so the package builds from the first minute. Other platforms are added beside it. */
|
|
655
|
+
async function writeExtToolsSkeleton(dir: string, id: string, version: string, platform: string): Promise<void> {
|
|
656
|
+
const host = getHost()
|
|
657
|
+
const manifest = {
|
|
658
|
+
schema: EXT_TOOLS_MANIFEST_SCHEMA,
|
|
659
|
+
id,
|
|
660
|
+
version,
|
|
661
|
+
requires: [],
|
|
662
|
+
conflicts: [],
|
|
663
|
+
provides: [{ command: id, version_probe: `${id} --version`, smoke: `smoke/${id}.sh` }],
|
|
664
|
+
platforms: [platform],
|
|
665
|
+
license: '',
|
|
666
|
+
desc: '',
|
|
667
|
+
}
|
|
668
|
+
const text = `${JSON.stringify(manifest, null, 2)}\n`
|
|
669
|
+
const payload = join(dir, 'payload', platform)
|
|
670
|
+
for (const sub of ['bin', 'smoke', join('store', id)]) await host.mkdir(join(payload, sub), { recursive: true })
|
|
671
|
+
await host.writeTextFile(join(dir, EXT_TOOLS_MANIFEST_FILE), text)
|
|
672
|
+
await host.writeTextFile(join(payload, EXT_TOOLS_MANIFEST_FILE), text)
|
|
673
|
+
const windows = platform.startsWith('windows-')
|
|
674
|
+
await host.writeTextFile(
|
|
675
|
+
join(payload, 'bin', windows ? `${id}.cmd` : id),
|
|
676
|
+
windows ? `@echo off\r\necho ${id} ${version}\r\n` : `#!/bin/sh\necho "${id} ${version}"\n`,
|
|
677
|
+
{ mode: 0o755 },
|
|
678
|
+
)
|
|
679
|
+
await host.writeTextFile(join(payload, 'smoke', `${id}.sh`), `#!/bin/sh\nset -eu\n${id} --version >/dev/null\n`, { mode: 0o755 })
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
async function readExtToolsManifestFile(path: string): Promise<{ text: string; manifest?: ExtToolsManifest; errors: string[] }> {
|
|
683
|
+
let text: string
|
|
684
|
+
try {
|
|
685
|
+
text = await getHost().readTextFile(path)
|
|
686
|
+
} catch (error) {
|
|
687
|
+
if (isHostError(error, 'NotFound')) return { text: '', errors: ['not found'] }
|
|
688
|
+
throw error
|
|
689
|
+
}
|
|
690
|
+
let value: unknown
|
|
691
|
+
try {
|
|
692
|
+
value = JSON.parse(text)
|
|
693
|
+
} catch {
|
|
694
|
+
return { text, errors: ['not valid JSON'] }
|
|
695
|
+
}
|
|
696
|
+
const errors = extToolsManifestErrors(value)
|
|
697
|
+
if (errors.length > 0) return { text, errors }
|
|
698
|
+
const manifest = value as ExtToolsManifest
|
|
699
|
+
manifest.requires ??= []
|
|
700
|
+
manifest.conflicts ??= []
|
|
701
|
+
return { text, manifest, errors }
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/** A payload directory as `pikg build` would archive it: symlinks read through, and refused when they leave the tree. */
|
|
705
|
+
async function readExtToolsPayloadTree(root: string): Promise<{ tree: ExtToolsPayloadTree; errors: string[] }> {
|
|
706
|
+
const host = getHost()
|
|
707
|
+
const tree: ExtToolsPayloadTree = { files: new Map(), directories: new Set(), launchers: new Map() }
|
|
708
|
+
const errors: string[] = []
|
|
709
|
+
const realRoot = await host.realPath(root)
|
|
710
|
+
const walk = async (directory: string, prefix: string): Promise<void> => {
|
|
711
|
+
const children = await host.readDir(directory)
|
|
712
|
+
for (const child of children) {
|
|
713
|
+
const archivePath = prefix ? `${prefix}/${child.name}` : child.name
|
|
714
|
+
const path = join(directory, child.name)
|
|
715
|
+
const link = await host.lstat(path)
|
|
716
|
+
let physical = path
|
|
717
|
+
if (link.isSymlink) {
|
|
718
|
+
try {
|
|
719
|
+
physical = await host.realPath(path)
|
|
720
|
+
} catch (error) {
|
|
721
|
+
errors.push(`${archivePath}: ${isHostError(error, 'NotFound') ? 'a dangling symlink' : 'a symlink out of the payload'}`)
|
|
722
|
+
continue
|
|
723
|
+
}
|
|
724
|
+
const inside = relative(realRoot, physical)
|
|
725
|
+
if (inside === '..' || inside.startsWith(`..${host.path.sep}`) || isAbsolute(inside)) {
|
|
726
|
+
errors.push(`${archivePath}: a symlink out of the payload`)
|
|
727
|
+
continue
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
const info = link.isSymlink ? await host.stat(physical) : link
|
|
731
|
+
if (info.isDirectory) {
|
|
732
|
+
tree.directories.add(archivePath)
|
|
733
|
+
await walk(physical, archivePath)
|
|
734
|
+
} else if (info.isFile) {
|
|
735
|
+
tree.files.set(archivePath, { executable: ((info.mode ?? 0) & 0o111) !== 0 })
|
|
736
|
+
if (archivePath.startsWith('bin/') && info.size <= EXT_TOOLS_LAUNCHER_LIMIT) {
|
|
737
|
+
tree.launchers.set(archivePath, await host.readFile(physical))
|
|
738
|
+
}
|
|
739
|
+
} else {
|
|
740
|
+
errors.push(`${archivePath}: not a regular file`)
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
await walk(realRoot, '')
|
|
745
|
+
return { tree, errors }
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/** `pikg doctor` of an extension package: the manifest against the App, every platform's payload, and, given a catalog
|
|
749
|
+
* of the other packages, `requires` and same-named commands. */
|
|
750
|
+
async function extToolsChecks(app: AppMeta, pikg: PikgMeta, metaDir: string, catalogDir: string | undefined): Promise<Array<Record<string, unknown>>> {
|
|
751
|
+
const checks: Array<Record<string, unknown>> = []
|
|
752
|
+
const platforms = new Map<string, string>()
|
|
753
|
+
const stray: string[] = []
|
|
754
|
+
for (const [key, subpackage] of Object.entries(pikg.sub_pkgs)) {
|
|
755
|
+
const platform = extToolsPlatformOfKey(key)
|
|
756
|
+
if (!platform || subpackage.source.type !== 'path') {
|
|
757
|
+
stray.push(key)
|
|
758
|
+
continue
|
|
759
|
+
}
|
|
760
|
+
platforms.set(platform.name, resolve(metaDir, subpackage.source.path))
|
|
761
|
+
}
|
|
762
|
+
const sources = [...platforms.values()]
|
|
763
|
+
// <source>/payload/<platform>: the package's manifest is two levels up
|
|
764
|
+
const source = sources.length > 0 ? dirname(dirname(sources[0])) : dirname(metaDir)
|
|
765
|
+
const master = await readExtToolsManifestFile(join(source, EXT_TOOLS_MANIFEST_FILE))
|
|
766
|
+
const manifestErrors = [...master.errors]
|
|
767
|
+
if (stray.length > 0) manifestErrors.push(`sub packages must be ext_tools_<os>_<arch> from a directory: ${stray.join(', ')}`)
|
|
768
|
+
const manifest = master.manifest
|
|
769
|
+
if (manifest) {
|
|
770
|
+
if (manifest.id !== app.name) manifestErrors.push(`id ${manifest.id} is not the App name ${app.name}`)
|
|
771
|
+
if (manifest.version !== app.version) manifestErrors.push(`version ${manifest.version} is not the App version ${app.version}`)
|
|
772
|
+
const without = manifest.platforms.filter((name) => !platforms.has(name))
|
|
773
|
+
if (without.length > 0) manifestErrors.push(`platforms without a sub package: ${without.join(', ')}`)
|
|
774
|
+
const unlisted = [...platforms.keys()].filter((name) => !manifest.platforms.includes(name))
|
|
775
|
+
if (unlisted.length > 0) manifestErrors.push(`sub packages of platforms it does not list: ${unlisted.join(', ')}`)
|
|
776
|
+
}
|
|
777
|
+
checks.push({
|
|
778
|
+
name: 'ext_tools_manifest',
|
|
779
|
+
status: manifestErrors.length === 0 ? 'ok' : 'error',
|
|
780
|
+
path: join(source, EXT_TOOLS_MANIFEST_FILE),
|
|
781
|
+
...(manifestErrors.length > 0 ? { errors: manifestErrors } : {}),
|
|
782
|
+
})
|
|
783
|
+
if (!manifest) return checks
|
|
784
|
+
for (const [platform, dir] of platforms) {
|
|
785
|
+
const errors: string[] = []
|
|
786
|
+
try {
|
|
787
|
+
const { tree, errors: walkErrors } = await readExtToolsPayloadTree(dir)
|
|
788
|
+
errors.push(...walkErrors)
|
|
789
|
+
const copy = await readExtToolsManifestFile(join(dir, EXT_TOOLS_MANIFEST_FILE))
|
|
790
|
+
if (copy.errors.length > 0) errors.push(`${EXT_TOOLS_MANIFEST_FILE}: ${copy.errors.join('; ')}`)
|
|
791
|
+
else if (copy.text !== master.text) errors.push(`${EXT_TOOLS_MANIFEST_FILE}: differs from the package's manifest`)
|
|
792
|
+
errors.push(...extToolsPayloadErrors(manifest, platform, tree, [source, metaDir]))
|
|
793
|
+
} catch (error) {
|
|
794
|
+
if (!isHostError(error, 'NotFound')) throw error
|
|
795
|
+
errors.push(`${dir}: not found`)
|
|
796
|
+
}
|
|
797
|
+
checks.push({
|
|
798
|
+
name: 'ext_tools_payload',
|
|
799
|
+
platform,
|
|
800
|
+
status: errors.length === 0 ? 'ok' : 'error',
|
|
801
|
+
path: dir,
|
|
802
|
+
...(errors.length > 0 ? { errors } : {}),
|
|
803
|
+
})
|
|
804
|
+
}
|
|
805
|
+
if (!catalogDir) {
|
|
806
|
+
checks.push({
|
|
807
|
+
name: 'ext_tools_catalog',
|
|
808
|
+
status: 'skipped',
|
|
809
|
+
detail: 'no --catalog: requires and commands other packages also provide are not checked',
|
|
810
|
+
})
|
|
811
|
+
return checks
|
|
812
|
+
}
|
|
813
|
+
const others = new Map<string, ExtToolsManifest>()
|
|
814
|
+
const catalogErrors: string[] = []
|
|
815
|
+
for (const entry of await getHost().readDir(catalogDir)) {
|
|
816
|
+
if (entry.isFile) continue
|
|
817
|
+
const candidate = await readExtToolsManifestFile(join(catalogDir, entry.name, EXT_TOOLS_MANIFEST_FILE))
|
|
818
|
+
if (candidate.errors.includes('not found')) continue
|
|
819
|
+
if (!candidate.manifest) catalogErrors.push(`${entry.name}: ${candidate.errors.join('; ')}`)
|
|
820
|
+
else if (candidate.manifest.id !== manifest.id) others.set(candidate.manifest.id, candidate.manifest)
|
|
821
|
+
}
|
|
822
|
+
for (const required of manifest.requires) {
|
|
823
|
+
if (!others.has(required.id)) catalogErrors.push(`requires ${required.id}: not in the catalog`)
|
|
824
|
+
}
|
|
825
|
+
const commands = new Set(manifest.provides.map((provided) => provided.command))
|
|
826
|
+
for (const other of others.values()) {
|
|
827
|
+
if (manifest.conflicts.includes(other.id) || other.conflicts.includes(manifest.id)) continue
|
|
828
|
+
const shared = other.provides.map((provided) => provided.command).filter((command) => commands.has(command))
|
|
829
|
+
if (shared.length > 0) catalogErrors.push(`${other.id} also provides ${shared.join(', ')}: one of the two must list the other in conflicts`)
|
|
830
|
+
}
|
|
831
|
+
checks.push({
|
|
832
|
+
name: 'ext_tools_catalog',
|
|
833
|
+
status: catalogErrors.length === 0 ? 'ok' : 'error',
|
|
834
|
+
path: catalogDir,
|
|
835
|
+
packages: [...others.keys()],
|
|
836
|
+
...(catalogErrors.length > 0 ? { errors: catalogErrors } : {}),
|
|
837
|
+
})
|
|
838
|
+
return checks
|
|
839
|
+
}
|
|
840
|
+
|
|
595
841
|
async function buildCommand(
|
|
596
842
|
ctx: CommandContext,
|
|
597
843
|
input: Record<string, unknown>,
|
|
@@ -144,51 +144,164 @@ export interface TarFileEntry {
|
|
|
144
144
|
|
|
145
145
|
export async function readTarGz(bytes: Uint8Array, limit: number): Promise<TarFileEntry[]> {
|
|
146
146
|
const tar = await gunzipBytes(bytes, limit)
|
|
147
|
-
const decoder = new TextDecoder('utf-8', { fatal: true })
|
|
148
|
-
const text = (offset: number, length: number) =>
|
|
149
|
-
decoder.decode(tar.subarray(offset, offset + length)).replace(/\0.*$/s, '')
|
|
150
147
|
const files: TarFileEntry[] = []
|
|
151
148
|
const seen = new Set<string>()
|
|
152
149
|
let offset = 0
|
|
153
150
|
while (
|
|
154
151
|
offset + BLOCK_SIZE <= tar.length && tar.subarray(offset, offset + BLOCK_SIZE).some(Boolean)
|
|
155
152
|
) {
|
|
156
|
-
|
|
157
|
-
let size: number
|
|
158
|
-
let mode: number
|
|
159
|
-
try {
|
|
160
|
-
const prefix = text(offset + 257, 6) === 'ustar' ? text(offset + 345, 155) : ''
|
|
161
|
-
raw = prefix ? `${prefix}/${text(offset, 100)}` : text(offset, 100)
|
|
162
|
-
size = Number.parseInt(text(offset + 124, 12).trim() || '0', 8)
|
|
163
|
-
mode = Number.parseInt(text(offset + 100, 8).trim() || '0', 8)
|
|
164
|
-
} catch {
|
|
165
|
-
throw invalidZip('payload', 'tar header is not valid UTF-8')
|
|
166
|
-
}
|
|
167
|
-
const type = String.fromCharCode(tar[offset + 156] || 0x30)
|
|
168
|
-
const path = raw.replace(/^(?:\.\/)+/, '').replace(/\/$/, '')
|
|
153
|
+
const header = parseTarHeader(tar.subarray(offset, offset + BLOCK_SIZE))
|
|
169
154
|
const start = offset + BLOCK_SIZE
|
|
170
|
-
if (
|
|
171
|
-
throw invalidZip('payload', `tar entry is truncated: ${
|
|
172
|
-
}
|
|
173
|
-
offset = start + Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE
|
|
174
|
-
if (type === '5' && (path === '' || path === '.')) continue
|
|
175
|
-
if (
|
|
176
|
-
!path || path.startsWith('/') || /^[A-Za-z]:/.test(path) || path.includes('\\') ||
|
|
177
|
-
path.split('/').some((segment) => !segment || segment === '.' || segment === '..')
|
|
178
|
-
) {
|
|
179
|
-
throw invalidZip('payload', `unsafe tar entry path: ${JSON.stringify(raw)}`)
|
|
180
|
-
}
|
|
181
|
-
if (type === '5') continue
|
|
182
|
-
if (type !== '0') {
|
|
183
|
-
throw invalidZip('payload', `tar entry is not a regular file or directory: ${path}`)
|
|
155
|
+
if (start + header.size > tar.length) {
|
|
156
|
+
throw invalidZip('payload', `tar entry is truncated: ${header.path}`)
|
|
184
157
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
158
|
+
offset = start + Math.ceil(header.size / BLOCK_SIZE) * BLOCK_SIZE
|
|
159
|
+
if (header.kind !== 'file') continue
|
|
160
|
+
if (seen.has(header.path)) throw invalidZip('payload', `duplicate tar entry: ${header.path}`)
|
|
161
|
+
seen.add(header.path)
|
|
162
|
+
files.push({ path: header.path, executable: header.executable, bytes: tar.slice(start, start + header.size) })
|
|
188
163
|
}
|
|
189
164
|
return files
|
|
190
165
|
}
|
|
191
166
|
|
|
167
|
+
interface TarHeader {
|
|
168
|
+
path: string
|
|
169
|
+
kind: 'file' | 'directory' | 'root'
|
|
170
|
+
size: number
|
|
171
|
+
executable: boolean
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function parseTarHeader(block: Uint8Array): TarHeader {
|
|
175
|
+
const decoder = new TextDecoder('utf-8', { fatal: true })
|
|
176
|
+
const text = (offset: number, length: number) =>
|
|
177
|
+
decoder.decode(block.subarray(offset, offset + length)).replace(/\0.*$/s, '')
|
|
178
|
+
let raw: string
|
|
179
|
+
let size: number
|
|
180
|
+
let mode: number
|
|
181
|
+
try {
|
|
182
|
+
const prefix = text(257, 6) === 'ustar' ? text(345, 155) : ''
|
|
183
|
+
raw = prefix ? `${prefix}/${text(0, 100)}` : text(0, 100)
|
|
184
|
+
size = Number.parseInt(text(124, 12).trim() || '0', 8)
|
|
185
|
+
mode = Number.parseInt(text(100, 8).trim() || '0', 8)
|
|
186
|
+
} catch {
|
|
187
|
+
throw invalidZip('payload', 'tar header is not valid UTF-8')
|
|
188
|
+
}
|
|
189
|
+
if (!Number.isSafeInteger(size) || size < 0) {
|
|
190
|
+
throw invalidZip('payload', `tar entry is truncated: ${raw}`)
|
|
191
|
+
}
|
|
192
|
+
const type = String.fromCharCode(block[156] || 0x30)
|
|
193
|
+
const path = raw.replace(/^(?:\.\/)+/, '').replace(/\/$/, '')
|
|
194
|
+
if (type === '5' && (path === '' || path === '.')) {
|
|
195
|
+
return { path, kind: 'root', size, executable: false }
|
|
196
|
+
}
|
|
197
|
+
if (
|
|
198
|
+
!path || path.startsWith('/') || /^[A-Za-z]:/.test(path) || path.includes('\\') ||
|
|
199
|
+
path.split('/').some((segment) => !segment || segment === '.' || segment === '..')
|
|
200
|
+
) {
|
|
201
|
+
throw invalidZip('payload', `unsafe tar entry path: ${JSON.stringify(raw)}`)
|
|
202
|
+
}
|
|
203
|
+
if (type !== '5' && type !== '0') {
|
|
204
|
+
throw invalidZip('payload', `tar entry is not a regular file or directory: ${path}`)
|
|
205
|
+
}
|
|
206
|
+
return { path, kind: type === '5' ? 'directory' : 'file', size, executable: (mode & 0o111) !== 0 }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export interface TarListing {
|
|
210
|
+
files: Map<string, { size: number; executable: boolean }>
|
|
211
|
+
directories: Set<string>
|
|
212
|
+
/** The contents of the files `capture` asked for. */
|
|
213
|
+
captured: Map<string, Uint8Array>
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** List a tar.gz too large to hold: a package's payload is gigabytes, and only its few small files need reading. */
|
|
217
|
+
export async function listTarGzStream(
|
|
218
|
+
gzip: ReadableStream<Uint8Array>,
|
|
219
|
+
limit: number,
|
|
220
|
+
capture: (path: string, size: number) => boolean,
|
|
221
|
+
): Promise<TarListing> {
|
|
222
|
+
const listing: TarListing = { files: new Map(), directories: new Set(), captured: new Map() }
|
|
223
|
+
const header = new Uint8Array(BLOCK_SIZE)
|
|
224
|
+
let headerFill = 0
|
|
225
|
+
let skip = 0
|
|
226
|
+
let body: { path: string; bytes: Uint8Array; fill: number } | undefined
|
|
227
|
+
let ended = false
|
|
228
|
+
let total = 0
|
|
229
|
+
try {
|
|
230
|
+
const stream = gzip.pipeThrough(
|
|
231
|
+
new DecompressionStream('gzip') as unknown as ReadableWritablePair<Uint8Array, Uint8Array>,
|
|
232
|
+
)
|
|
233
|
+
for await (const chunk of stream) {
|
|
234
|
+
total += chunk.length
|
|
235
|
+
if (total > limit) throw invalidZip('limits', 'payload expands beyond the size limit')
|
|
236
|
+
let offset = 0
|
|
237
|
+
while (offset < chunk.length && !ended) {
|
|
238
|
+
if (skip > 0) {
|
|
239
|
+
const take = Math.min(skip, chunk.length - offset)
|
|
240
|
+
if (body && body.fill < body.bytes.length) {
|
|
241
|
+
const part = Math.min(take, body.bytes.length - body.fill)
|
|
242
|
+
body.bytes.set(chunk.subarray(offset, offset + part), body.fill)
|
|
243
|
+
body.fill += part
|
|
244
|
+
if (body.fill === body.bytes.length) {
|
|
245
|
+
listing.captured.set(body.path, body.bytes)
|
|
246
|
+
body = undefined
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
skip -= take
|
|
250
|
+
offset += take
|
|
251
|
+
continue
|
|
252
|
+
}
|
|
253
|
+
const take = Math.min(BLOCK_SIZE - headerFill, chunk.length - offset)
|
|
254
|
+
header.set(chunk.subarray(offset, offset + take), headerFill)
|
|
255
|
+
headerFill += take
|
|
256
|
+
offset += take
|
|
257
|
+
if (headerFill < BLOCK_SIZE) break
|
|
258
|
+
headerFill = 0
|
|
259
|
+
if (!header.some(Boolean)) {
|
|
260
|
+
ended = true
|
|
261
|
+
break
|
|
262
|
+
}
|
|
263
|
+
const entry = parseTarHeader(header)
|
|
264
|
+
skip = Math.ceil(entry.size / BLOCK_SIZE) * BLOCK_SIZE
|
|
265
|
+
if (entry.kind === 'directory') listing.directories.add(entry.path)
|
|
266
|
+
if (entry.kind !== 'file') continue
|
|
267
|
+
if (listing.files.has(entry.path)) {
|
|
268
|
+
throw invalidZip('payload', `duplicate tar entry: ${entry.path}`)
|
|
269
|
+
}
|
|
270
|
+
listing.files.set(entry.path, { size: entry.size, executable: entry.executable })
|
|
271
|
+
if (capture(entry.path, entry.size)) {
|
|
272
|
+
if (entry.size === 0) listing.captured.set(entry.path, new Uint8Array())
|
|
273
|
+
else body = { path: entry.path, bytes: new Uint8Array(entry.size), fill: 0 }
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
} catch (error) {
|
|
278
|
+
if (error instanceof ToolError) throw error
|
|
279
|
+
throw invalidZip('compression', 'payload is not a valid gzip stream')
|
|
280
|
+
}
|
|
281
|
+
if (skip > 0 || headerFill > 0 || body) throw invalidZip('payload', 'tar stream is truncated')
|
|
282
|
+
return listing
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export function fileReadableStream(path: string): ReadableStream<Uint8Array> {
|
|
286
|
+
let file: HostFile | undefined
|
|
287
|
+
return new ReadableStream<Uint8Array>({
|
|
288
|
+
async pull(controller) {
|
|
289
|
+
file ??= await getHost().open(path, { read: true })
|
|
290
|
+
const buffer = new Uint8Array(256 * 1024)
|
|
291
|
+
const read = await file.read(buffer)
|
|
292
|
+
if (read === null) {
|
|
293
|
+
await file.close()
|
|
294
|
+
controller.close()
|
|
295
|
+
return
|
|
296
|
+
}
|
|
297
|
+
controller.enqueue(buffer.subarray(0, read))
|
|
298
|
+
},
|
|
299
|
+
async cancel() {
|
|
300
|
+
await file?.close()
|
|
301
|
+
},
|
|
302
|
+
})
|
|
303
|
+
}
|
|
304
|
+
|
|
192
305
|
async function gunzipBytes(bytes: Uint8Array, limit: number): Promise<Uint8Array> {
|
|
193
306
|
const stream = new ReadableStream<Uint8Array>({
|
|
194
307
|
start(controller) {
|
|
@@ -628,7 +741,7 @@ function createEntryVerification(entry: ZipEntry): {
|
|
|
628
741
|
}
|
|
629
742
|
}
|
|
630
743
|
|
|
631
|
-
function zipEntryStream(path: string, entry: ZipEntry): ReadableStream<Uint8Array> {
|
|
744
|
+
export function zipEntryStream(path: string, entry: ZipEntry): ReadableStream<Uint8Array> {
|
|
632
745
|
let file: HostFile | undefined
|
|
633
746
|
let remaining = entry.compressedSize
|
|
634
747
|
const compressed = new ReadableStream<Uint8Array>({
|