@zhuoyuezs/ml-platform 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CI-ARTIFACT.md +2 -2
- package/checksums.json +74 -54
- package/package.json +1 -1
- package/release.json +19 -19
- package/runtime/business-client/README.md +64 -4
- package/runtime/business-client/package-lock.json +2 -2
- package/runtime/business-client/package.json +1 -1
- package/runtime/business-client/src/catalog.js +25 -6
- package/runtime/business-client/src/cli.js +177 -7
- package/runtime/business-client/src/http.js +192 -32
- package/skills/data-source-exploration/SKILL.md +12 -1
- package/skills/data-source-exploration/agents/openai.yaml +2 -2
- package/skills/data-source-exploration/references/data-platform.md +83 -0
- package/skills/feature-management/SKILL.md +29 -11
- package/skills/feature-management/agents/openai.yaml +2 -2
- package/skills/feature-management/references/commands.md +64 -7
- package/skills/feature-management/references/contracts.md +124 -6
- package/skills/feature-management/references/data-platform-mapping.md +104 -0
- package/skills/feature-management/references/operator-authoring.md +116 -8
- package/skills/feature-management/references/platform-capability-guide.md +6 -0
- package/skills/feature-management/references/supervised-datasets.md +13 -0
- package/skills/model-deployment-management/SKILL.md +7 -0
- package/skills/model-deployment-management/references/contracts.md +6 -0
- package/skills/model-deployment-management/references/operations.md +28 -1
- package/skills/model-deployment-management/references/troubleshooting.md +8 -0
- package/skills/model-lifecycle-management/SKILL.md +27 -5
- package/skills/model-lifecycle-management/agents/openai.yaml +2 -2
- package/skills/model-lifecycle-management/references/ablation.md +111 -0
- package/skills/model-lifecycle-management/references/discovery.md +18 -3
- package/skills/model-lifecycle-management/references/evaluation.md +188 -1
- package/skills/model-lifecycle-management/references/experiment-runs.md +59 -0
- package/skills/model-lifecycle-management/references/model-comparison.md +11 -9
- package/skills/model-lifecycle-management/references/packaging.md +46 -0
- package/skills/model-lifecycle-management/references/training-contracts.md +172 -1
- package/skills/model-lifecycle-management/references/training.md +60 -0
- package/skills/prediction-report-management/SKILL.md +43 -8
|
@@ -7,7 +7,7 @@ const { configPath, expandHome, loadApiUrl, normalizeApiUrl, saveApiUrl } = requ
|
|
|
7
7
|
const { PlatformApiClient } = require("./http");
|
|
8
8
|
const { applyCatalog, loadCatalog, normalizeDataset, normalizeRegistryDataset } = require("./catalog");
|
|
9
9
|
const { version: CLIENT_VERSION } = require("../package.json");
|
|
10
|
-
const LIFECYCLE_COLLECTIONS = new Set(["training-runs", "training-jobs", "model-artifacts", "evaluation-configs", "evaluation-runs", "evaluation-jobs", "metric-definitions", "executable-packages", "model-packages"]);
|
|
10
|
+
const LIFECYCLE_COLLECTIONS = new Set(["training-runs", "training-jobs", "model-artifacts", "evaluation-configs", "evaluation-runs", "evaluation-jobs", "metric-definitions", "executable-packages", "model-packages", "experiment-runs"]);
|
|
11
11
|
const BUSINESS_COMMANDS = new Set([
|
|
12
12
|
...[...LIFECYCLE_COLLECTIONS].map((name) => `list-${name}`),
|
|
13
13
|
"get-model-runtime-release", "get-training-run", "get-model-artifact", "get-executable-package",
|
|
@@ -15,6 +15,9 @@ const BUSINESS_COMMANDS = new Set([
|
|
|
15
15
|
"version", "create-project", "list-projects", "get-project", "delete-project",
|
|
16
16
|
"get-dataset-contract-file", "validate-evaluation-comparison",
|
|
17
17
|
"configure", "show-config", "health",
|
|
18
|
+
"data-platform-status", "list-data-platform-connections", "get-data-platform-connection",
|
|
19
|
+
"list-data-platform-objects", "list-data-platform-mappings", "list-data-platform-query-candidates", "get-data-platform-mapping-revision",
|
|
20
|
+
"bind-data-platform-parameters", "query-data-platform", "get-data-platform-query", "get-data-platform-snapshot",
|
|
18
21
|
"list-datasource-connections", "list-datasource-namespaces", "list-datasource-objects",
|
|
19
22
|
"list-datasource-fields", "query-datasource-object", "create-datasource-connection",
|
|
20
23
|
"get-datasource-connection", "update-datasource-connection", "delete-datasource-connection",
|
|
@@ -25,7 +28,7 @@ const BUSINESS_COMMANDS = new Set([
|
|
|
25
28
|
"get-parameter", "get-feature", "get-feature-set", "get-operator",
|
|
26
29
|
"delete-parameter", "delete-feature", "delete-feature-set",
|
|
27
30
|
"restore-parameter", "restore-feature", "restore-feature-set",
|
|
28
|
-
"get-parameter-lifecycle", "get-feature-lifecycle", "get-feature-set-lifecycle",
|
|
31
|
+
"get-parameter-lifecycle", "get-feature-lifecycle", "get-feature-set-lifecycle", "get-parameter-freshness",
|
|
29
32
|
"list-parameter-references", "list-feature-references", "list-feature-set-references",
|
|
30
33
|
"list-parameter-versions", "list-feature-versions", "list-feature-set-versions", "list-operator-versions",
|
|
31
34
|
"add-parameter", "add-feature", "add-feature-set", "add-operator", "publish-operator",
|
|
@@ -34,10 +37,12 @@ const BUSINESS_COMMANDS = new Set([
|
|
|
34
37
|
"get-job", "wait-job", "retry-job", "cancel-job", "get-dataset-artifact", "download-dataset-artifact", "fetch-inference-data", "fetch-inference-context", "predict-model", "apply",
|
|
35
38
|
"list-foundation-models", "get-foundation-model", "predict-foundation", "predict-foundation-csv",
|
|
36
39
|
"list-system-models", "get-system-model", "register-training-run", "validate-training-run", "submit-training-job", "get-training-job", "retry-training-job", "cancel-training-job",
|
|
40
|
+
"create-experiment-run", "validate-experiment-run", "submit-experiment-run", "get-experiment-run", "list-experiment-candidates", "reconcile-experiment-run", "cancel-experiment-run",
|
|
37
41
|
"register-evaluation-config", "get-evaluation-config", "register-metric-definition", "get-metric-definition",
|
|
38
42
|
"register-executable-package", "verify-executable-package", "publish-executable-package",
|
|
39
43
|
"validate-evaluation-run", "submit-evaluation", "get-evaluation-run", "get-evaluation-job", "retry-evaluation-job", "cancel-evaluation-job", "list-evaluation-attempts",
|
|
40
44
|
"get-evaluation-result", "get-evaluation-result-metrics", "get-evaluation-result-artifacts", "summarize-evaluation-run", "get-evaluation-summary",
|
|
45
|
+
"download-model-artifact", "download-evaluation-result-artifact",
|
|
41
46
|
"create-model-package", "get-model-package", "cancel-model-package", "validate-model-artifact", "register-model-artifact",
|
|
42
47
|
"list-business-domains", "list-scenarios", "create-scenario", "put-deployment-metadata",
|
|
43
48
|
"list-deployment-targets", "validate-deployment", "create-deployment", "list-deployments",
|
|
@@ -61,6 +66,17 @@ const COMMAND_USAGE = {
|
|
|
61
66
|
configure: "configure --api-url URL",
|
|
62
67
|
"show-config": "show-config",
|
|
63
68
|
health: "health",
|
|
69
|
+
"data-platform-status": "data-platform-status",
|
|
70
|
+
"list-data-platform-connections": "list-data-platform-connections",
|
|
71
|
+
"get-data-platform-connection": "get-data-platform-connection CONNECTION_ID",
|
|
72
|
+
"list-data-platform-objects": "list-data-platform-objects CONNECTION_ID",
|
|
73
|
+
"list-data-platform-mappings": "list-data-platform-mappings [--page N] [--page-size N] [--scope-code CODE]",
|
|
74
|
+
"list-data-platform-query-candidates": "list-data-platform-query-candidates",
|
|
75
|
+
"get-data-platform-mapping-revision": "get-data-platform-mapping-revision MAPPING_ID REVISION_ID",
|
|
76
|
+
"bind-data-platform-parameters": "bind-data-platform-parameters SPEC_JSON",
|
|
77
|
+
"query-data-platform": "query-data-platform SPEC_JSON",
|
|
78
|
+
"get-data-platform-query": "get-data-platform-query RUN_ID",
|
|
79
|
+
"get-data-platform-snapshot": "get-data-platform-snapshot SNAPSHOT_ID [--offset N] [--limit N]",
|
|
64
80
|
"list-datasource-connections": "list-datasource-connections [--limit N] [--cursor CURSOR]",
|
|
65
81
|
"list-datasource-namespaces": "list-datasource-namespaces CONNECTION_ID [--limit N] [--cursor CURSOR]",
|
|
66
82
|
"list-datasource-objects": "list-datasource-objects CONNECTION_ID NAMESPACE_ID [--limit N] [--cursor CURSOR]",
|
|
@@ -91,6 +107,7 @@ const COMMAND_USAGE = {
|
|
|
91
107
|
"restore-feature": "restore-feature NAME VERSION [--project PROJECT] [--actor ACTOR] [--reason TEXT] [--idempotency-key KEY]",
|
|
92
108
|
"restore-feature-set": "restore-feature-set NAME VERSION [--project PROJECT] [--actor ACTOR] [--reason TEXT] [--idempotency-key KEY]",
|
|
93
109
|
"get-parameter-lifecycle": "get-parameter-lifecycle NAME VERSION [--project PROJECT]",
|
|
110
|
+
"get-parameter-freshness": "get-parameter-freshness NAME VERSION [--project PROJECT] [--lookback DURATION] [--end TIMESTAMP] [--online]",
|
|
94
111
|
"get-feature-lifecycle": "get-feature-lifecycle NAME VERSION [--project PROJECT]",
|
|
95
112
|
"get-feature-set-lifecycle": "get-feature-set-lifecycle NAME VERSION [--project PROJECT]",
|
|
96
113
|
"list-parameter-references": "list-parameter-references NAME VERSION [--project PROJECT] [--direct]",
|
|
@@ -159,11 +176,20 @@ const COMMAND_USAGE = {
|
|
|
159
176
|
"get-evaluation-result": "get-evaluation-result RESULT_ID",
|
|
160
177
|
"get-evaluation-result-metrics": "get-evaluation-result-metrics RESULT_ID",
|
|
161
178
|
"get-evaluation-result-artifacts": "get-evaluation-result-artifacts RESULT_ID",
|
|
179
|
+
"download-model-artifact": "download-model-artifact ARTIFACT_ID --out-dir PATH [--file NAME] [--project PROJECT] [--force]",
|
|
180
|
+
"download-evaluation-result-artifact": "download-evaluation-result-artifact RESULT_ID --out-dir PATH [--file NAME] [--project PROJECT] [--force]",
|
|
162
181
|
"summarize-evaluation-run": "summarize-evaluation-run RUN_ID",
|
|
163
182
|
"get-evaluation-summary": "get-evaluation-summary SUMMARY_ID",
|
|
164
183
|
"create-model-package": "create-model-package REQUEST_JSON [--idempotency-key KEY]",
|
|
165
184
|
"get-model-package": "get-model-package PACKAGE_ID [--package-version VERSION]",
|
|
166
185
|
"cancel-model-package": "cancel-model-package PACKAGE_ID [--package-version VERSION]",
|
|
186
|
+
"create-experiment-run": "create-experiment-run SPEC_JSON",
|
|
187
|
+
"validate-experiment-run": "validate-experiment-run RUN_ID",
|
|
188
|
+
"submit-experiment-run": "submit-experiment-run RUN_ID",
|
|
189
|
+
"get-experiment-run": "get-experiment-run RUN_ID",
|
|
190
|
+
"list-experiment-candidates": "list-experiment-candidates RUN_ID [--limit N] [--offset N]",
|
|
191
|
+
"reconcile-experiment-run": "reconcile-experiment-run RUN_ID",
|
|
192
|
+
"cancel-experiment-run": "cancel-experiment-run RUN_ID",
|
|
167
193
|
"validate-model-artifact": "validate-model-artifact ARTIFACT_ID",
|
|
168
194
|
"register-model-artifact": "register-model-artifact ARTIFACT_ID --evaluation-id EVALUATION_ID",
|
|
169
195
|
"list-business-domains": "list-business-domains",
|
|
@@ -173,7 +199,7 @@ const COMMAND_USAGE = {
|
|
|
173
199
|
"list-deployment-targets": "list-deployment-targets",
|
|
174
200
|
"validate-deployment": "validate-deployment SPEC_JSON --evaluation-id ID",
|
|
175
201
|
"create-deployment": "create-deployment SPEC_JSON --evaluation-id ID",
|
|
176
|
-
"list-deployments": "list-deployments [--service
|
|
202
|
+
"list-deployments": "list-deployments [--service TEXT] [--environment ENV] [--package-id ID] [--package-version VERSION] [--model-artifact-id ID] [--state STATE] [--business-domain-id ID] [--scenario-id ID] [-q QUERY] [--sort-by deployment_id|created_at|updated_at] [--sort-order asc|desc] [--limit N] [--offset N]",
|
|
177
203
|
"get-deployment": "get-deployment DEPLOYMENT_ID",
|
|
178
204
|
"wait-deployment": "wait-deployment DEPLOYMENT_ID [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
179
205
|
"reconcile-deployment": "reconcile-deployment DEPLOYMENT_ID",
|
|
@@ -200,10 +226,22 @@ const COMMAND_USAGE = {
|
|
|
200
226
|
"get-prediction-report-run": "get-prediction-report-run RUN_ID",
|
|
201
227
|
apply: "apply CATALOG_DIRECTORY [--dry-run] [--project PROJECT]",
|
|
202
228
|
};
|
|
229
|
+
const IDEMPOTENCY_HEADER_NOTE = "--idempotency-key is sent as the Idempotency-Key header; the current server does not enforce it, so inspect platform state before retrying an uncertain submission.";
|
|
230
|
+
const BUILD_WAIT_NOTE = "With --wait, a succeeded result adds artifact.manifest_hash: the DatasetArtifact key for get/download-dataset-artifact and training/evaluation dataset bindings. The top-level manifest_hash identifies the submitted manifest and is not an artifact key.";
|
|
231
|
+
// Help-only notes. Keep them out of COMMAND_USAGE, which is also the argv grammar.
|
|
232
|
+
const COMMAND_NOTES = {
|
|
233
|
+
"download-dataset-artifact": "MANIFEST_HASH is the artifact manifest hash (list-dataset-artifacts, or artifact.manifest_hash from build-registered-dataset --wait), not the registered manifest hash.",
|
|
234
|
+
"build-dataset": BUILD_WAIT_NOTE,
|
|
235
|
+
"build-registered-dataset": BUILD_WAIT_NOTE,
|
|
236
|
+
"wait-job": "A succeeded build Job result adds artifact.manifest_hash, the DatasetArtifact key.",
|
|
237
|
+
"download-model-artifact": "Downloads every indexed ModelArtifact file, or only each --file NAME, and verifies each sha256 against the registry record. --project fails closed when the artifact belongs to another project.",
|
|
238
|
+
"download-evaluation-result-artifact": "Downloads every indexed EvaluationResult file (for example predictions.parquet, metrics.json, runtime.json), or only each --file NAME, and verifies each sha256 against the registry record.",
|
|
239
|
+
...Object.fromEntries(["create-project", "register-training-run", "submit-training-job", "register-evaluation-config", "submit-evaluation", "create-model-package"].map((name) => [name, IDEMPOTENCY_HEADER_NOTE])),
|
|
240
|
+
};
|
|
203
241
|
function usage(command) {
|
|
204
242
|
const prefix = "ml-platform [--profile server] [--api-url URL] [--request-timeout-seconds SECONDS]";
|
|
205
243
|
if (command)
|
|
206
|
-
return `usage: ${prefix} ${COMMAND_USAGE[command]}\n`;
|
|
244
|
+
return `usage: ${prefix} ${COMMAND_USAGE[command]}\n${COMMAND_NOTES[command] ? `\n${COMMAND_NOTES[command]}\n` : ""}`;
|
|
207
245
|
return `usage: ${prefix} COMMAND [ARGS]\n\ncommands:\n ${[...BUSINESS_COMMANDS].join("\n ")}\n`;
|
|
208
246
|
}
|
|
209
247
|
function parse(args) {
|
|
@@ -363,12 +401,33 @@ async function submitBuild(api, manifest, build) {
|
|
|
363
401
|
if (build.wait) {
|
|
364
402
|
if (!result || typeof result !== "object" || !result.job_id)
|
|
365
403
|
throw new Error("server build did not return a job_id to wait for");
|
|
366
|
-
result = await waitForJob(api, String(result.job_id), build.pollInterval, build.waitTimeout);
|
|
404
|
+
result = withBuildArtifact(await waitForJob(api, String(result.job_id), build.pollInterval, build.waitTimeout));
|
|
367
405
|
if (["failed", "cancelled"].includes(result.status))
|
|
368
406
|
exitCode = 1;
|
|
369
407
|
}
|
|
370
408
|
return { result, exitCode };
|
|
371
409
|
}
|
|
410
|
+
// A build Job's top-level manifest_hash identifies the submitted (registered)
|
|
411
|
+
// manifest. The published DatasetArtifact is keyed by the resolved contract hash,
|
|
412
|
+
// so surface it additively as `artifact` for downloads and dataset bindings.
|
|
413
|
+
function withBuildArtifact(job) {
|
|
414
|
+
if (!job || typeof job !== "object" || Array.isArray(job) || job.status !== "succeeded")
|
|
415
|
+
return job;
|
|
416
|
+
const resolved = job.resolved_manifest;
|
|
417
|
+
if (!resolved || typeof resolved !== "object" || !resolved.manifest_hash)
|
|
418
|
+
return job;
|
|
419
|
+
return {
|
|
420
|
+
...job,
|
|
421
|
+
artifact: {
|
|
422
|
+
project: resolved.project ?? job.manifest?.project ?? "default",
|
|
423
|
+
dataset_id: resolved.dataset_id ?? job.manifest?.dataset_id ?? null,
|
|
424
|
+
dataset_version: resolved.dataset_version ?? job.manifest?.dataset_version ?? null,
|
|
425
|
+
manifest_hash: resolved.manifest_hash,
|
|
426
|
+
artifact_uri: resolved.artifact_uri ?? null,
|
|
427
|
+
row_count: resolved.row_count ?? null,
|
|
428
|
+
},
|
|
429
|
+
};
|
|
430
|
+
}
|
|
372
431
|
async function waitForJob(api, jobId, pollInterval, timeoutSeconds) {
|
|
373
432
|
if (!(pollInterval > 0))
|
|
374
433
|
throw new Error("job poll interval must be positive");
|
|
@@ -453,6 +512,42 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
453
512
|
}
|
|
454
513
|
else if (options.command === "health")
|
|
455
514
|
result = await client(options).get("/health");
|
|
515
|
+
else if (options.command === "data-platform-status")
|
|
516
|
+
result = await client(options).get("/data-platform/status");
|
|
517
|
+
else if (options.command === "list-data-platform-connections")
|
|
518
|
+
result = await client(options).get("/data-platform/connections");
|
|
519
|
+
else if (options.command === "list-data-platform-query-candidates")
|
|
520
|
+
result = await client(options).get("/data-platform/query-candidates");
|
|
521
|
+
else if (["get-data-platform-connection", "list-data-platform-objects"].includes(options.command)) {
|
|
522
|
+
const id = encodeURIComponent(positional(rest, "connection id"));
|
|
523
|
+
const suffix = options.command === "list-data-platform-objects" ? "/objects" : "";
|
|
524
|
+
result = await client(options).get(`/data-platform/connections/${id}${suffix}`);
|
|
525
|
+
}
|
|
526
|
+
else if (options.command === "list-data-platform-mappings") {
|
|
527
|
+
result = await client(options).get("/data-platform/mappings", {
|
|
528
|
+
page: integer(take(rest, "--page", "1"), "page"),
|
|
529
|
+
page_size: integer(take(rest, "--page-size", "50"), "page size"),
|
|
530
|
+
scope_code: take(rest, "--scope-code"),
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
else if (options.command === "get-data-platform-mapping-revision") {
|
|
534
|
+
const mappingId = encodeURIComponent(positional(rest, "mapping id"));
|
|
535
|
+
const revisionId = encodeURIComponent(positional(rest, "revision id"));
|
|
536
|
+
result = await client(options).get(`/data-platform/mappings/${mappingId}/revisions/${revisionId}`);
|
|
537
|
+
}
|
|
538
|
+
else if (options.command === "bind-data-platform-parameters")
|
|
539
|
+
result = await client(options).post("/data-platform/parameters", readSpec(positional(rest, "binding spec JSON")));
|
|
540
|
+
else if (options.command === "query-data-platform")
|
|
541
|
+
result = await client(options).post("/data-platform/query-runs", readSpec(positional(rest, "query spec JSON")));
|
|
542
|
+
else if (options.command === "get-data-platform-query")
|
|
543
|
+
result = await client(options).get(`/data-platform/query-runs/${encodeURIComponent(positional(rest, "run id"))}`);
|
|
544
|
+
else if (options.command === "get-data-platform-snapshot") {
|
|
545
|
+
const id = encodeURIComponent(positional(rest, "snapshot id"));
|
|
546
|
+
result = await client(options).get(`/data-platform/snapshots/${id}/rows`, {
|
|
547
|
+
offset: integer(take(rest, "--offset", "0"), "offset"),
|
|
548
|
+
limit: integer(take(rest, "--limit", "200"), "limit"),
|
|
549
|
+
});
|
|
550
|
+
}
|
|
456
551
|
else if (options.command === "list-datasource-connections") {
|
|
457
552
|
result = await client(options).get("/datasource-connections", datasourcePageParams(rest));
|
|
458
553
|
}
|
|
@@ -559,6 +654,21 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
559
654
|
actor: take(rest, "--actor") || "anonymous", reason: take(rest, "--reason", ""), idempotency_key: take(rest, "--idempotency-key"),
|
|
560
655
|
});
|
|
561
656
|
}
|
|
657
|
+
else if (options.command === "get-parameter-freshness") {
|
|
658
|
+
const name = positional(rest, "parameter name");
|
|
659
|
+
const version = positional(rest, "parameter version");
|
|
660
|
+
const project = take(rest, "--project", "default");
|
|
661
|
+
const params = {};
|
|
662
|
+
const lookback = take(rest, "--lookback");
|
|
663
|
+
const end = take(rest, "--end");
|
|
664
|
+
if (lookback !== undefined)
|
|
665
|
+
params.lookback = lookback;
|
|
666
|
+
if (end !== undefined)
|
|
667
|
+
params.end = end;
|
|
668
|
+
if (boolean(rest, "--online"))
|
|
669
|
+
params.online = true;
|
|
670
|
+
result = await client(options).get(`/parameters/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}/freshness`, params);
|
|
671
|
+
}
|
|
562
672
|
else if (/^get-(parameter|feature|feature-set)-lifecycle$/.test(options.command)) {
|
|
563
673
|
const name = positional(rest, "resource name");
|
|
564
674
|
const version = positional(rest, "resource version");
|
|
@@ -730,7 +840,7 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
730
840
|
const jobId = positional(rest, "job id");
|
|
731
841
|
const poll = finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval");
|
|
732
842
|
const rawTimeout = take(rest, "--wait-timeout-seconds");
|
|
733
|
-
result = await waitForJob(client(options), jobId, poll, rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds"));
|
|
843
|
+
result = withBuildArtifact(await waitForJob(client(options), jobId, poll, rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds")));
|
|
734
844
|
if (["failed", "cancelled"].includes(result.status))
|
|
735
845
|
exitCode = 1;
|
|
736
846
|
}
|
|
@@ -868,6 +978,8 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
868
978
|
business_domain_id: take(rest, "--business-domain-id"),
|
|
869
979
|
scenario_id: take(rest, "--scenario-id"),
|
|
870
980
|
q: take(rest, "--q", take(rest, "-q")),
|
|
981
|
+
sort_by: take(rest, "--sort-by"),
|
|
982
|
+
sort_order: take(rest, "--sort-order"),
|
|
871
983
|
};
|
|
872
984
|
boolean(rest, "--all");
|
|
873
985
|
result = await listCollection(client(options), "/deployments", params, rest);
|
|
@@ -997,6 +1109,34 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
997
1109
|
else if (options.command === "get-model-runtime-release") {
|
|
998
1110
|
result = await client(options).get("/model-runtime-release");
|
|
999
1111
|
}
|
|
1112
|
+
else if (options.command === "create-experiment-run") {
|
|
1113
|
+
result = await client(options).post("/experiment-runs", readSpec(positional(rest, "experiment JSON")));
|
|
1114
|
+
}
|
|
1115
|
+
else if (/^(validate|submit|reconcile|cancel)-experiment-run$/.test(options.command)) {
|
|
1116
|
+
const runId = positional(rest, "experiment run id");
|
|
1117
|
+
const action = options.command.split("-")[0];
|
|
1118
|
+
result = await client(options).post(`/experiment-runs/${encodeURIComponent(runId)}/${action}`);
|
|
1119
|
+
}
|
|
1120
|
+
else if (options.command === "get-experiment-run") {
|
|
1121
|
+
result = await client(options).get(`/experiment-runs/${encodeURIComponent(positional(rest, "experiment run id"))}`);
|
|
1122
|
+
}
|
|
1123
|
+
else if (options.command === "list-experiment-candidates") {
|
|
1124
|
+
const runId = positional(rest, "experiment run id");
|
|
1125
|
+
const params = {};
|
|
1126
|
+
const limit = take(rest, "--limit");
|
|
1127
|
+
const offset = take(rest, "--offset");
|
|
1128
|
+
if (limit !== undefined) {
|
|
1129
|
+
params.limit = integer(limit, "--limit");
|
|
1130
|
+
if (params.limit < 1 || params.limit > 500)
|
|
1131
|
+
throw new Error("--limit must be between 1 and 500");
|
|
1132
|
+
}
|
|
1133
|
+
if (offset !== undefined) {
|
|
1134
|
+
params.offset = integer(offset, "--offset");
|
|
1135
|
+
if (params.offset < 0)
|
|
1136
|
+
throw new Error("--offset must be non-negative");
|
|
1137
|
+
}
|
|
1138
|
+
result = await client(options).get(`/experiment-runs/${encodeURIComponent(runId)}/candidates`, params);
|
|
1139
|
+
}
|
|
1000
1140
|
else if (options.command.startsWith("list-") && LIFECYCLE_COLLECTIONS.has(options.command.slice(5))) {
|
|
1001
1141
|
const params = {};
|
|
1002
1142
|
const limit = take(rest, "--limit");
|
|
@@ -1074,6 +1214,36 @@ async function runBusinessCli(argv, output = { write: text => process.stdout.wri
|
|
|
1074
1214
|
const suffix = options.command.endsWith("-metrics") ? "/metrics" : options.command.endsWith("-artifacts") ? "/artifacts" : "";
|
|
1075
1215
|
result = await client(options).get(`/evaluation-results/${encodeURIComponent(resultId)}${suffix}`);
|
|
1076
1216
|
}
|
|
1217
|
+
else if (options.command === "download-model-artifact" || options.command === "download-evaluation-result-artifact") {
|
|
1218
|
+
const model = options.command === "download-model-artifact";
|
|
1219
|
+
const resourceId = positional(rest, model ? "model artifact id" : "evaluation result id");
|
|
1220
|
+
const outDir = take(rest, "--out-dir");
|
|
1221
|
+
const project = take(rest, "--project");
|
|
1222
|
+
const force = boolean(rest, "--force");
|
|
1223
|
+
if (!outDir)
|
|
1224
|
+
throw new Error("--out-dir is required");
|
|
1225
|
+
const files = [];
|
|
1226
|
+
while (rest.includes("--file"))
|
|
1227
|
+
files.push(take(rest, "--file"));
|
|
1228
|
+
const api = client(options);
|
|
1229
|
+
let index;
|
|
1230
|
+
let owner = project;
|
|
1231
|
+
if (model) {
|
|
1232
|
+
const record = await api.get(`/model-artifacts/${encodeURIComponent(resourceId)}`);
|
|
1233
|
+
if (project !== undefined && record.project !== project)
|
|
1234
|
+
throw new Error(`model artifact ${resourceId} belongs to project ${JSON.stringify(record.project)}, not ${JSON.stringify(project)}`);
|
|
1235
|
+
index = record.files;
|
|
1236
|
+
owner = record.project;
|
|
1237
|
+
}
|
|
1238
|
+
else {
|
|
1239
|
+
const listing = await api.get(`/evaluation-results/${encodeURIComponent(resourceId)}/artifacts`);
|
|
1240
|
+
if (!Array.isArray(listing.items))
|
|
1241
|
+
throw new Error("evaluation result artifact listing has no items");
|
|
1242
|
+
index = Object.fromEntries(listing.items.map((item) => [String(item.name), String(item.sha256)]));
|
|
1243
|
+
}
|
|
1244
|
+
const downloaded = await api.downloadLifecycleArtifactFiles(model ? "model-artifacts" : "evaluation-results", resourceId, index, files, resolvedPath(outDir), { force, project });
|
|
1245
|
+
result = { [model ? "model_artifact_id" : "result_id"]: resourceId, ...downloaded, project: owner ?? null };
|
|
1246
|
+
}
|
|
1077
1247
|
else if (/^(get-evaluation-config|get-metric-definition)$/.test(options.command)) {
|
|
1078
1248
|
const name = positional(rest, "resource name");
|
|
1079
1249
|
const version = positional(rest, "resource version");
|
|
@@ -1137,7 +1307,7 @@ async function listCollection(api, endpoint, params, rest) {
|
|
|
1137
1307
|
throw new Error("offset must be >= 0");
|
|
1138
1308
|
return api.get(endpoint, { ...params, limit, offset });
|
|
1139
1309
|
}
|
|
1140
|
-
module.exports = { BUSINESS_COMMANDS, fetchAll, listCollection, parse, runBusinessCli, usage, waitForDeployment, waitForJob };
|
|
1310
|
+
module.exports = { BUSINESS_COMMANDS, COMMAND_NOTES, withBuildArtifact, fetchAll, listCollection, parse, runBusinessCli, usage, waitForDeployment, waitForJob };
|
|
1141
1311
|
if (require.main === module) {
|
|
1142
1312
|
runBusinessCli(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
|
|
1143
1313
|
process.stderr.write(`error: ${error.message}\n`);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const crypto = require("crypto");
|
|
3
4
|
const fs = require("fs");
|
|
4
5
|
const os = require("os");
|
|
5
6
|
const path = require("path");
|
|
@@ -16,14 +17,120 @@ function apiErrorDetail(result, fallback) {
|
|
|
16
17
|
return result.detail;
|
|
17
18
|
return JSON.stringify(result, null, 2);
|
|
18
19
|
}
|
|
20
|
+
// Render the body of a failed binary download with the same rules as JSON calls,
|
|
21
|
+
// so structured FastAPI `detail` objects never collapse to "[object Object]".
|
|
22
|
+
async function responseErrorDetail(response, signal) {
|
|
23
|
+
let text = "";
|
|
24
|
+
try {
|
|
25
|
+
text = String(await (signal ? abortable(response.text(), signal) : response.text()));
|
|
26
|
+
}
|
|
27
|
+
catch (_) { /* fall back to status text */ }
|
|
28
|
+
let parsed;
|
|
29
|
+
try {
|
|
30
|
+
parsed = JSON.parse(text);
|
|
31
|
+
}
|
|
32
|
+
catch (_) {
|
|
33
|
+
parsed = undefined;
|
|
34
|
+
}
|
|
35
|
+
return apiErrorDetail(parsed, text.trim() || response.statusText || "request failed");
|
|
36
|
+
}
|
|
37
|
+
function safeRelativeArtifactPath(relative) {
|
|
38
|
+
const normalized = String(relative).replace(/\\/g, "/");
|
|
39
|
+
const parts = normalized.split("/");
|
|
40
|
+
if (!normalized || normalized.startsWith("/") || parts.some((part) => part === "" || part === "." || part === "..")) {
|
|
41
|
+
throw new Error(`artifact path escapes output directory: ${JSON.stringify(String(relative))}`);
|
|
42
|
+
}
|
|
43
|
+
return { normalized, parts };
|
|
44
|
+
}
|
|
45
|
+
// Connection-failure policy. A failure is retried only when it happened before
|
|
46
|
+
// any response headers arrived AND either
|
|
47
|
+
// - the method is idempotent (GET/HEAD): transient connection errors and the
|
|
48
|
+
// per-attempt timeout are retried with bounded backoff; or
|
|
49
|
+
// - any method: the TCP connection was never established (connection refused
|
|
50
|
+
// or connect timeout), which proves no request bytes reached the server.
|
|
51
|
+
// Every other failure of a non-idempotent request (for example a reset after
|
|
52
|
+
// the request was written) is reported once, because the server may already
|
|
53
|
+
// have applied it.
|
|
54
|
+
const IDEMPOTENT_METHODS = new Set(["GET", "HEAD"]);
|
|
55
|
+
const UNSENT_CONNECT_CODES = new Set(["ECONNREFUSED", "UND_ERR_CONNECT_TIMEOUT"]);
|
|
56
|
+
const TRANSIENT_CONNECTION_CODES = new Set([
|
|
57
|
+
...UNSENT_CONNECT_CODES, "UND_ERR_SOCKET", "UND_ERR_CLOSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "ECONNABORTED",
|
|
58
|
+
"EHOSTUNREACH", "ENETUNREACH", "ENETDOWN", "EAI_AGAIN",
|
|
59
|
+
]);
|
|
60
|
+
const DEFAULT_RETRY_DELAYS_MS = [500, 1000, 2000];
|
|
61
|
+
// Error codes along the `cause` chain (undici wraps socket errors as
|
|
62
|
+
// `TypeError: fetch failed` with the real error in `cause`; happy-eyeballs
|
|
63
|
+
// connects may report an AggregateError with per-address errors).
|
|
64
|
+
function errorCodes(error, seen = new Set()) {
|
|
65
|
+
if (!error || typeof error !== "object" || seen.has(error))
|
|
66
|
+
return [];
|
|
67
|
+
seen.add(error);
|
|
68
|
+
const own = typeof error.code === "string" ? [error.code] : [];
|
|
69
|
+
const nested = Array.isArray(error.errors) ? error.errors.flatMap((item) => errorCodes(item, seen)) : [];
|
|
70
|
+
return [...own, ...nested, ...errorCodes(error.cause, seen)];
|
|
71
|
+
}
|
|
72
|
+
function describeError(error) {
|
|
73
|
+
if (!(error instanceof Error))
|
|
74
|
+
return String(error);
|
|
75
|
+
const details = [];
|
|
76
|
+
const seen = new Set([error]);
|
|
77
|
+
let cause = error.cause;
|
|
78
|
+
while (cause && typeof cause === "object" && !seen.has(cause)) {
|
|
79
|
+
seen.add(cause);
|
|
80
|
+
const message = cause.message || (Array.isArray(cause.errors) ? cause.errors.map((item) => item?.message).filter(Boolean).join("; ") : "");
|
|
81
|
+
details.push([cause.code, message].filter(Boolean).join(": "));
|
|
82
|
+
cause = cause.cause;
|
|
83
|
+
}
|
|
84
|
+
const suffix = details.filter(Boolean);
|
|
85
|
+
return suffix.length ? `${error.message} (${suffix.join(" <- ")})` : error.message;
|
|
86
|
+
}
|
|
87
|
+
function retryableFailure(method, error, timedOut) {
|
|
88
|
+
if (IDEMPOTENT_METHODS.has(method))
|
|
89
|
+
return timedOut || errorCodes(error).some((code) => TRANSIENT_CONNECTION_CODES.has(code));
|
|
90
|
+
if (timedOut)
|
|
91
|
+
return false;
|
|
92
|
+
const codes = errorCodes(error);
|
|
93
|
+
return codes.length > 0 && codes.every((code) => UNSENT_CONNECT_CODES.has(code));
|
|
94
|
+
}
|
|
19
95
|
class PlatformApiClient {
|
|
20
|
-
constructor(baseUrl, timeoutSeconds = 30) {
|
|
96
|
+
constructor(baseUrl, timeoutSeconds = 30, { retryDelaysMs = DEFAULT_RETRY_DELAYS_MS } = {}) {
|
|
21
97
|
this.baseUrl = String(baseUrl).trim().replace(/\/+$/, "");
|
|
22
98
|
if (!/^https?:\/\//.test(this.baseUrl))
|
|
23
99
|
throw new Error("server profile api_url must start with http:// or https://");
|
|
24
100
|
if (!(timeoutSeconds > 0))
|
|
25
101
|
throw new Error("request timeout must be positive");
|
|
26
102
|
this.timeoutMs = timeoutSeconds * 1000;
|
|
103
|
+
this.retryDelaysMs = [...retryDelaysMs];
|
|
104
|
+
}
|
|
105
|
+
// Send one request and return once response headers arrive. The caller owns
|
|
106
|
+
// the returned per-attempt timeout and must call `done()` after reading the
|
|
107
|
+
// body. Each request asks for `Connection: close`, so no pooled keep-alive
|
|
108
|
+
// socket is ever reused after the server (uvicorn closes idle sockets after
|
|
109
|
+
// 5 s) or a VPN/NAT hop has dropped it.
|
|
110
|
+
async open(method, url, { headers = {}, body } = {}) {
|
|
111
|
+
for (let attempt = 0;; attempt += 1) {
|
|
112
|
+
const controller = new AbortController();
|
|
113
|
+
let timedOut = false;
|
|
114
|
+
const timer = setTimeout(() => { timedOut = true; controller.abort(); }, this.timeoutMs);
|
|
115
|
+
try {
|
|
116
|
+
const response = await fetch(url, { method, headers: { ...headers, Connection: "close" }, body, signal: controller.signal });
|
|
117
|
+
return { response, signal: controller.signal, done: () => clearTimeout(timer) };
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
clearTimeout(timer);
|
|
121
|
+
const detail = timedOut ? `request timed out after ${this.timeoutMs / 1000}s (${describeError(error)})` : describeError(error);
|
|
122
|
+
const delay = this.retryDelaysMs[attempt];
|
|
123
|
+
if (delay !== undefined && retryableFailure(method, error, timedOut)) {
|
|
124
|
+
process.stderr.write(`platform API ${method} ${url}: ${detail}; retrying in ${delay}ms (attempt ${attempt + 2}/${this.retryDelaysMs.length + 1})\n`);
|
|
125
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
const attempts = attempt ? ` after ${attempt + 1} attempts` : "";
|
|
129
|
+
const ambiguous = !IDEMPOTENT_METHODS.has(method) && !retryableFailure(method, error, timedOut)
|
|
130
|
+
? "; not retried because the server may have received it; inspect platform state before resubmitting" : "";
|
|
131
|
+
throw new Error(`platform API request failed: ${method} ${url}: ${detail}${attempts}${ambiguous}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
27
134
|
}
|
|
28
135
|
url(endpoint, params = {}) {
|
|
29
136
|
const url = new URL(endpoint.replace(/^\//, ""), `${this.baseUrl}/`);
|
|
@@ -34,8 +141,6 @@ class PlatformApiClient {
|
|
|
34
141
|
return url;
|
|
35
142
|
}
|
|
36
143
|
async request(method, endpoint, { params, payload, body, contentType, requestHeaders } = {}) {
|
|
37
|
-
const controller = new AbortController();
|
|
38
|
-
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
39
144
|
const headers = { Accept: "application/json" };
|
|
40
145
|
Object.assign(headers, requestHeaders || {});
|
|
41
146
|
let requestBody = body;
|
|
@@ -45,17 +150,17 @@ class PlatformApiClient {
|
|
|
45
150
|
}
|
|
46
151
|
else if (contentType)
|
|
47
152
|
headers["Content-Type"] = contentType;
|
|
48
|
-
|
|
153
|
+
const { response, done } = await this.open(method, this.url(endpoint, params), { headers, body: requestBody });
|
|
154
|
+
let text;
|
|
49
155
|
try {
|
|
50
|
-
|
|
156
|
+
text = await response.text();
|
|
51
157
|
}
|
|
52
158
|
catch (error) {
|
|
53
|
-
throw new Error(`platform API request failed: ${method} ${this.url(endpoint, params)}: ${error
|
|
159
|
+
throw new Error(`platform API request failed: ${method} ${this.url(endpoint, params)}: reading response: ${describeError(error)}`);
|
|
54
160
|
}
|
|
55
161
|
finally {
|
|
56
|
-
|
|
162
|
+
done();
|
|
57
163
|
}
|
|
58
|
-
const text = await response.text();
|
|
59
164
|
let result;
|
|
60
165
|
try {
|
|
61
166
|
result = JSON.parse(text);
|
|
@@ -115,20 +220,15 @@ class PlatformApiClient {
|
|
|
115
220
|
fs.rmSync(target, { recursive: true, force: true });
|
|
116
221
|
}
|
|
117
222
|
fs.mkdirSync(target, { recursive: true });
|
|
118
|
-
const controller = new AbortController();
|
|
119
|
-
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
120
223
|
const temporaryDir = fs.mkdtempSync(path.join(os.tmpdir(), "data-platform-artifact-download-"));
|
|
121
224
|
const temporary = path.join(temporaryDir, "artifact.zip");
|
|
122
|
-
let
|
|
225
|
+
let done = () => { };
|
|
123
226
|
try {
|
|
124
|
-
|
|
227
|
+
const opened = await this.open("GET", this.url(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/download`), { headers: { Accept: "application/zip" } });
|
|
228
|
+
const { response, signal } = opened;
|
|
229
|
+
done = opened.done;
|
|
125
230
|
if (!response.ok) {
|
|
126
|
-
|
|
127
|
-
try {
|
|
128
|
-
detail = JSON.parse(await abortable(response.text(), controller.signal))?.detail ?? detail;
|
|
129
|
-
}
|
|
130
|
-
catch (_) { /* use status text */ }
|
|
131
|
-
throw new Error(`artifact download failed: HTTP ${response.status}: ${detail}`);
|
|
231
|
+
throw new Error(`artifact download failed: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
|
|
132
232
|
}
|
|
133
233
|
const total = contentLength(response);
|
|
134
234
|
let lastReported = -1;
|
|
@@ -139,7 +239,7 @@ class PlatformApiClient {
|
|
|
139
239
|
}
|
|
140
240
|
};
|
|
141
241
|
report(0);
|
|
142
|
-
await writeResponseBody(response, temporary,
|
|
242
|
+
await writeResponseBody(response, temporary, signal, report);
|
|
143
243
|
const files = await extractArchive(temporary, target);
|
|
144
244
|
return { dataset_id: datasetId, manifest_hash: manifestHash, output_dir: target, file_count: files.length, files };
|
|
145
245
|
}
|
|
@@ -147,10 +247,10 @@ class PlatformApiClient {
|
|
|
147
247
|
fs.rmSync(target, { recursive: true, force: true });
|
|
148
248
|
if (error instanceof Error && error.message.startsWith("artifact download failed:"))
|
|
149
249
|
throw error;
|
|
150
|
-
throw new Error(`artifact download failed: ${error
|
|
250
|
+
throw new Error(`artifact download failed: ${describeError(error)}`);
|
|
151
251
|
}
|
|
152
252
|
finally {
|
|
153
|
-
|
|
253
|
+
done();
|
|
154
254
|
fs.rmSync(temporaryDir, { recursive: true, force: true });
|
|
155
255
|
}
|
|
156
256
|
}
|
|
@@ -183,22 +283,15 @@ class PlatformApiClient {
|
|
|
183
283
|
}
|
|
184
284
|
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
185
285
|
const endpoint = `/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/files/${parts.map(encodeURIComponent).join("/")}`;
|
|
186
|
-
const
|
|
187
|
-
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
286
|
+
const { response, signal, done } = await this.open("GET", this.url(endpoint), { headers: { Accept: "application/octet-stream" } });
|
|
188
287
|
try {
|
|
189
|
-
const response = await fetch(this.url(endpoint), { headers: { Accept: "application/octet-stream" }, signal: controller.signal });
|
|
190
288
|
if (!response.ok) {
|
|
191
|
-
|
|
192
|
-
try {
|
|
193
|
-
detail = JSON.parse(await response.text())?.detail ?? detail;
|
|
194
|
-
}
|
|
195
|
-
catch (_) { /* status text */ }
|
|
196
|
-
throw new Error(`artifact file download failed: HTTP ${response.status}: ${detail}`);
|
|
289
|
+
throw new Error(`artifact file download failed: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
|
|
197
290
|
}
|
|
198
291
|
fs.writeFileSync(destination, Buffer.from(await response.arrayBuffer()), { flag: "wx" });
|
|
199
292
|
}
|
|
200
293
|
finally {
|
|
201
|
-
|
|
294
|
+
done();
|
|
202
295
|
}
|
|
203
296
|
}
|
|
204
297
|
}
|
|
@@ -208,6 +301,71 @@ class PlatformApiClient {
|
|
|
208
301
|
}
|
|
209
302
|
return { project, dataset_id: datasetId, manifest_hash: manifestHash, output_dir: targetRoot, file_count: files.length, files: files.sort() };
|
|
210
303
|
}
|
|
304
|
+
// Download indexed ModelArtifact/EvaluationResult files one by one and verify
|
|
305
|
+
// each against the sha256 recorded in the registry index before keeping it.
|
|
306
|
+
async downloadLifecycleArtifactFiles(collection, resourceId, index, relativePaths, outputDir, { force = false, project = undefined } = {}) {
|
|
307
|
+
if (!["model-artifacts", "evaluation-results"].includes(collection))
|
|
308
|
+
throw new Error(`unsupported artifact collection: ${collection}`);
|
|
309
|
+
if (!index || typeof index !== "object" || Array.isArray(index))
|
|
310
|
+
throw new Error(`${collection} record has no file index`);
|
|
311
|
+
const available = Object.keys(index).sort();
|
|
312
|
+
const requested = relativePaths && relativePaths.length ? [...new Set(relativePaths.map((value) => String(value)))] : available;
|
|
313
|
+
if (!requested.length)
|
|
314
|
+
throw new Error(`${collection} record indexes no files: ${resourceId}`);
|
|
315
|
+
const plan = requested.map((relative) => safeRelativeArtifactPath(relative));
|
|
316
|
+
const missing = plan.filter((item) => !Object.prototype.hasOwnProperty.call(index, item.normalized)).map((item) => item.normalized);
|
|
317
|
+
if (missing.length)
|
|
318
|
+
throw new Error(`file is not indexed by ${collection}/${resourceId}: ${missing.join(", ")}; available: ${available.join(", ")}`);
|
|
319
|
+
const targetRoot = path.resolve(expandHome(outputDir));
|
|
320
|
+
if (fs.existsSync(targetRoot) && fs.readdirSync(targetRoot).length) {
|
|
321
|
+
if (!force)
|
|
322
|
+
throw new Error(`download output directory is not empty: ${targetRoot}`);
|
|
323
|
+
fs.rmSync(targetRoot, { recursive: true, force: true });
|
|
324
|
+
}
|
|
325
|
+
fs.mkdirSync(targetRoot, { recursive: true });
|
|
326
|
+
const downloaded = [];
|
|
327
|
+
try {
|
|
328
|
+
for (const { normalized, parts } of plan) {
|
|
329
|
+
const destination = path.resolve(targetRoot, ...parts);
|
|
330
|
+
if (!destination.startsWith(`${targetRoot}${path.sep}`))
|
|
331
|
+
throw new Error(`artifact path escapes output directory: ${JSON.stringify(normalized)}`);
|
|
332
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
333
|
+
const endpoint = `/${collection}/${encodeURIComponent(resourceId)}/files/${parts.map(encodeURIComponent).join("/")}`;
|
|
334
|
+
const partial = `${destination}.partial`;
|
|
335
|
+
let done = () => { };
|
|
336
|
+
try {
|
|
337
|
+
const opened = await this.open("GET", this.url(endpoint, { project }), { headers: { Accept: "application/octet-stream" } });
|
|
338
|
+
const { response, signal } = opened;
|
|
339
|
+
done = opened.done;
|
|
340
|
+
if (!response.ok)
|
|
341
|
+
throw new Error(`artifact file download failed: ${normalized}: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
|
|
342
|
+
const hash = crypto.createHash("sha256");
|
|
343
|
+
let size = 0;
|
|
344
|
+
await writeResponseBody(response, partial, signal, () => { }, (chunk) => { hash.update(chunk); size += chunk.length; });
|
|
345
|
+
const actual = `sha256:${hash.digest("hex")}`;
|
|
346
|
+
const expected = String(index[normalized]);
|
|
347
|
+
const served = response.headers?.get?.("x-artifact-sha256");
|
|
348
|
+
if (actual !== expected || (served && served !== expected)) {
|
|
349
|
+
throw new Error(`artifact file checksum mismatch: ${normalized}: registry ${expected}, downloaded ${actual}${served && served !== expected ? `, server header ${served}` : ""}`);
|
|
350
|
+
}
|
|
351
|
+
fs.renameSync(partial, destination);
|
|
352
|
+
downloaded.push({ name: normalized, sha256: actual, size_bytes: size });
|
|
353
|
+
}
|
|
354
|
+
finally {
|
|
355
|
+
done();
|
|
356
|
+
fs.rmSync(partial, { force: true });
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
catch (error) {
|
|
361
|
+
fs.rmSync(targetRoot, { recursive: true, force: true });
|
|
362
|
+
if (error instanceof Error && /^artifact (file|path)/.test(error.message))
|
|
363
|
+
throw error;
|
|
364
|
+
throw new Error(`artifact file download failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
365
|
+
}
|
|
366
|
+
downloaded.sort((left, right) => left.name.localeCompare(right.name));
|
|
367
|
+
return { output_dir: targetRoot, file_count: downloaded.length, verified: true, files: downloaded };
|
|
368
|
+
}
|
|
211
369
|
}
|
|
212
370
|
function contentLength(response) {
|
|
213
371
|
const raw = response.headers?.get?.("content-length") ?? response.headers?.["content-length"];
|
|
@@ -228,13 +386,15 @@ function abortable(value, signal) {
|
|
|
228
386
|
Promise.resolve(value).then((result) => { cleanup(); resolve(result); }, (error) => { cleanup(); reject(error); });
|
|
229
387
|
});
|
|
230
388
|
}
|
|
231
|
-
async function writeResponseBody(response, outputPath, signal, onProgress) {
|
|
389
|
+
async function writeResponseBody(response, outputPath, signal, onProgress, onChunk = undefined) {
|
|
232
390
|
const handle = fs.openSync(outputPath, "w");
|
|
233
391
|
let received = 0;
|
|
234
392
|
try {
|
|
235
393
|
const write = (value) => {
|
|
236
394
|
const chunk = Buffer.from(value);
|
|
237
395
|
fs.writeSync(handle, chunk);
|
|
396
|
+
if (onChunk)
|
|
397
|
+
onChunk(chunk);
|
|
238
398
|
received += chunk.length;
|
|
239
399
|
onProgress(received);
|
|
240
400
|
};
|