@assethub/cli 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +349 -5
- package/dist/autopilot.d.ts +13 -0
- package/dist/autopilot.d.ts.map +1 -0
- package/dist/autopilot.js +158 -0
- package/dist/canvas.d.ts +27 -0
- package/dist/canvas.d.ts.map +1 -0
- package/dist/canvas.js +82 -0
- package/dist/canvasAssetImport.d.ts +45 -0
- package/dist/canvasAssetImport.d.ts.map +1 -0
- package/dist/canvasAssetImport.js +154 -0
- package/dist/canvasComparison.d.ts +21 -0
- package/dist/canvasComparison.d.ts.map +1 -0
- package/dist/canvasComparison.js +58 -0
- package/dist/execution.d.ts +175 -0
- package/dist/execution.d.ts.map +1 -0
- package/dist/execution.js +234 -0
- package/dist/index.d.ts +94 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1579 -189
- package/dist/projectSources.d.ts +65 -0
- package/dist/projectSources.d.ts.map +1 -0
- package/dist/projectSources.js +369 -0
- package/dist/projectSourcesWorkbook.d.ts +3 -0
- package/dist/projectSourcesWorkbook.d.ts.map +1 -0
- package/dist/projectSourcesWorkbook.js +143 -0
- package/dist/runsUpload/artifactGraphModel.d.ts +48 -0
- package/dist/runsUpload/artifactGraphModel.d.ts.map +1 -0
- package/dist/runsUpload/artifactGraphModel.js +127 -0
- package/dist/runsUpload/blobRefs.d.ts +13 -0
- package/dist/runsUpload/blobRefs.d.ts.map +1 -0
- package/dist/runsUpload/blobRefs.js +73 -0
- package/dist/runsUpload/buildRunUpload.d.ts +63 -0
- package/dist/runsUpload/buildRunUpload.d.ts.map +1 -0
- package/dist/runsUpload/buildRunUpload.js +428 -0
- package/dist/runsUpload/canonicalJson.d.ts +5 -0
- package/dist/runsUpload/canonicalJson.d.ts.map +1 -0
- package/dist/runsUpload/canonicalJson.js +108 -0
- package/dist/runsUpload/controlEndpoints.d.ts +63 -0
- package/dist/runsUpload/controlEndpoints.d.ts.map +1 -0
- package/dist/runsUpload/controlEndpoints.js +64 -0
- package/dist/runsUpload/graphFolder.d.ts +25 -0
- package/dist/runsUpload/graphFolder.d.ts.map +1 -0
- package/dist/runsUpload/graphFolder.js +147 -0
- package/dist/runsUpload/runUploadError.d.ts +23 -0
- package/dist/runsUpload/runUploadError.d.ts.map +1 -0
- package/dist/runsUpload/runUploadError.js +24 -0
- package/dist/runsUpload/uploadRun.d.ts +43 -0
- package/dist/runsUpload/uploadRun.d.ts.map +1 -0
- package/dist/runsUpload/uploadRun.js +217 -0
- package/package.json +11 -8
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE ONE PLACE the Production Control run-upload wire contract lives.
|
|
3
|
+
*
|
|
4
|
+
* The backend for these routes is being built in parallel. Everything the CLI
|
|
5
|
+
* knows about *where* to send bytes and *how big* they may be is in this file,
|
|
6
|
+
* so a late change on the server side is a one-line edit here rather than a
|
|
7
|
+
* sweep through the transport.
|
|
8
|
+
*
|
|
9
|
+
* Route shape mirrors the Cloudflare ag-registry Worker
|
|
10
|
+
* (assethub-ml research/artifact-graph/cloud/registry-worker/src/worker.js)
|
|
11
|
+
* because the payloads are the same `ag.registry.v1` documents. Only the prefix
|
|
12
|
+
* and the transport envelope differ: these live on AssetHub's public v2 API
|
|
13
|
+
* surface, behind `withExposedApi()` + API-key auth, so they speak AssetHub's
|
|
14
|
+
* `{success, data}` / `{success, error}` envelope rather than
|
|
15
|
+
* `ag.registry-error.v1`.
|
|
16
|
+
*/
|
|
17
|
+
/** Change this one line if the backend lands on a different prefix. */
|
|
18
|
+
export const CONTROL_RUN_UPLOAD_API_PREFIX = '/api/v2/production/run-uploads';
|
|
19
|
+
const trimTrailingSlash = (value) => value.replace(/\/+$/, '');
|
|
20
|
+
export const controlRunUploadUrls = (baseUrl) => {
|
|
21
|
+
// Matches @assethub/api-client's own URL construction: trim the trailing
|
|
22
|
+
// slash and append, so a base URL carrying a path prefix keeps it.
|
|
23
|
+
const prefix = `${trimTrailingSlash(baseUrl)}${CONTROL_RUN_UPLOAD_API_PREFIX}`;
|
|
24
|
+
return {
|
|
25
|
+
graphs: `${prefix}/graphs`,
|
|
26
|
+
snapshot: graphId => `${prefix}/graphs/${encodeURIComponent(graphId)}/snapshot`,
|
|
27
|
+
blob: (graphId, blobKey) => `${prefix}/blobs/${encodeURIComponent(graphId)}/${encodeURIComponent(blobKey)}`,
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Server-enforced ceilings, checked locally so a doomed run is rejected before
|
|
32
|
+
* the first byte leaves rather than after N blobs have already landed.
|
|
33
|
+
*
|
|
34
|
+
* The `ag.registry.v1` format allows far more (64 MiB graph arrays, 2 MiB
|
|
35
|
+
* manifest). These smaller numbers are the real ceiling: the routes run on
|
|
36
|
+
* Vercel Serverless, whose request-body limit bites first.
|
|
37
|
+
*/
|
|
38
|
+
export const CONTROL_MAX_BLOB_BYTES = 4 * 1024 * 1024;
|
|
39
|
+
export const CONTROL_MAX_PUSH_BYTES = 4 * 1024 * 1024;
|
|
40
|
+
/** Format-level ceilings from the registry contract, checked for completeness. */
|
|
41
|
+
export const REGISTRY_MAX_MANIFEST_BYTES = 2 * 1024 * 1024;
|
|
42
|
+
export const REGISTRY_MAX_GRAPH_ARRAY_BYTES = 64 * 1024 * 1024;
|
|
43
|
+
export const REGISTRY_MAX_METADATA_BYTES = 256 * 1024;
|
|
44
|
+
/**
|
|
45
|
+
* Error codes the routes answer with. Values follow AssetHub's public-API
|
|
46
|
+
* SCREAMING_SNAKE convention; the semantics follow the registry's.
|
|
47
|
+
*/
|
|
48
|
+
export const CONTROL_ERROR_CODES = {
|
|
49
|
+
/** 409 — the snapshot references bytes the pool does not hold yet. */
|
|
50
|
+
missingBlobs: 'MISSING_BLOBS',
|
|
51
|
+
/** 409 — same rev, different graphHash. A genuine divergence. */
|
|
52
|
+
snapshotConflict: 'SNAPSHOT_CONFLICT',
|
|
53
|
+
/** 409 — the server already holds a newer rev. Desired end state, not a failure. */
|
|
54
|
+
staleSnapshot: 'STALE_SNAPSHOT',
|
|
55
|
+
payloadTooLarge: 'PAYLOAD_TOO_LARGE',
|
|
56
|
+
/** 404 — the graph was never registered by this org. Distinct from "no access". */
|
|
57
|
+
notRegistered: 'NOT_REGISTERED',
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* These routes are internal-only. An API key whose owner is not an internal
|
|
61
|
+
* user, or whose org lacks the Production Control entitlement, gets 404 rather
|
|
62
|
+
* than 403 — the path's existence is itself not disclosed.
|
|
63
|
+
*/
|
|
64
|
+
export const CONTROL_NOT_ENTITLED_STATUS = 404;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type ArtifactGraph, type ArtifactNode } from './artifactGraphModel.js';
|
|
2
|
+
export declare const GRAPH_FOLDER_SCHEMA_VERSION = "ag.graph-folder.v1";
|
|
3
|
+
export declare const MANIFEST_FILE_NAME = "manifest.json";
|
|
4
|
+
export declare const NODES_FILE_NAME = "nodes.jsonl";
|
|
5
|
+
export declare const EDGES_FILE_NAME = "edges.jsonl";
|
|
6
|
+
export type GraphFolder = {
|
|
7
|
+
/** Absolute-or-as-given folder path, used verbatim in error messages. */
|
|
8
|
+
path: string;
|
|
9
|
+
/** The folder's own name, the default source for a graphId. */
|
|
10
|
+
name: string;
|
|
11
|
+
manifest: Record<string, unknown>;
|
|
12
|
+
/** Normalized + canonically ordered, ready to hash. */
|
|
13
|
+
graph: ArtifactGraph;
|
|
14
|
+
/** blobKey -> the file inside the folder that holds its bytes. */
|
|
15
|
+
blobFiles: Map<string, string>;
|
|
16
|
+
};
|
|
17
|
+
/** Port of format.py `manifest_flows`: metadata.flow.id ∪ `flow:*` tag suffixes. */
|
|
18
|
+
export declare const deriveFlows: (graph: ArtifactGraph) => string[];
|
|
19
|
+
/** Port of format.py `node_state`. */
|
|
20
|
+
export declare const nodeState: (node: ArtifactNode) => string;
|
|
21
|
+
/** Port of format.py `manifest_states`: `state:*` tag counts, key-sorted. */
|
|
22
|
+
export declare const deriveStates: (graph: ArtifactGraph) => Record<string, number>;
|
|
23
|
+
/** Port of `read_graph_folder`. */
|
|
24
|
+
export declare const readGraphFolder: (path: string) => Promise<GraphFolder>;
|
|
25
|
+
//# sourceMappingURL=graphFolder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graphFolder.d.ts","sourceRoot":"","sources":["../../src/runsUpload/graphFolder.ts"],"names":[],"mappings":"AAgBA,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,yBAAyB,CAAA;AAIhC,eAAO,MAAM,2BAA2B,uBAAuB,CAAA;AAC/D,eAAO,MAAM,kBAAkB,kBAAkB,CAAA;AACjD,eAAO,MAAM,eAAe,gBAAgB,CAAA;AAC5C,eAAO,MAAM,eAAe,gBAAgB,CAAA;AAE5C,MAAM,MAAM,WAAW,GAAG;IACxB,yEAAyE;IACzE,IAAI,EAAE,MAAM,CAAA;IACZ,+DAA+D;IAC/D,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAA;IACpB,kEAAkE;IAClE,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC/B,CAAA;AAED,oFAAoF;AACpF,eAAO,MAAM,WAAW,GAAI,OAAO,aAAa,KAAG,MAAM,EAYxD,CAAA;AAED,sCAAsC;AACtC,eAAO,MAAM,SAAS,GAAI,MAAM,YAAY,KAAG,MAK9C,CAAA;AAED,6EAA6E;AAC7E,eAAO,MAAM,YAAY,GAAI,OAAO,aAAa,KAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAOxE,CAAA;AAED,mCAAmC;AACnC,eAAO,MAAM,eAAe,GAAU,MAAM,MAAM,KAAG,OAAO,CAAC,WAAW,CAevE,CAAA"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// Reading an `ag.graph-folder.v1` folder off local disk.
|
|
2
|
+
//
|
|
3
|
+
// Port of assethub-ml research/artifact-graph/artifact-graph-py/
|
|
4
|
+
// artifact_graph/graph_folder/ (`format.py` parse_manifest / parse_jsonl /
|
|
5
|
+
// manifest_flows / manifest_states / node_state, `load.py` read_graph_folder).
|
|
6
|
+
//
|
|
7
|
+
// A folder written by `save_graph_folder` is already byte-canonical: its
|
|
8
|
+
// nodes.jsonl / edges.jsonl lines are the `normalize_graph_state` output, one
|
|
9
|
+
// canonical JSON line each, in canonical order. Re-normalizing them here is
|
|
10
|
+
// therefore a no-op for a healthy folder and a repair for a hand-edited one —
|
|
11
|
+
// and because the manifest's graphHash was taken over that same output,
|
|
12
|
+
// re-hashing is what proves the folder still describes itself.
|
|
13
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
14
|
+
import { basename, join } from 'node:path';
|
|
15
|
+
import { codePointCompare, normalizeArtifactGraph, } from './artifactGraphModel.js';
|
|
16
|
+
import { isPlainObject } from './canonicalJson.js';
|
|
17
|
+
import { RunUploadError } from './runUploadError.js';
|
|
18
|
+
export const GRAPH_FOLDER_SCHEMA_VERSION = 'ag.graph-folder.v1';
|
|
19
|
+
export const MANIFEST_FILE_NAME = 'manifest.json';
|
|
20
|
+
export const NODES_FILE_NAME = 'nodes.jsonl';
|
|
21
|
+
export const EDGES_FILE_NAME = 'edges.jsonl';
|
|
22
|
+
/** Port of format.py `manifest_flows`: metadata.flow.id ∪ `flow:*` tag suffixes. */
|
|
23
|
+
export const deriveFlows = (graph) => {
|
|
24
|
+
const flows = new Set();
|
|
25
|
+
for (const node of graph.nodes ?? []) {
|
|
26
|
+
const metadata = isPlainObject(node.metadata) ? node.metadata : {};
|
|
27
|
+
const flow = isPlainObject(metadata.flow) ? metadata.flow : {};
|
|
28
|
+
const flowId = String(flow.id ?? '').trim();
|
|
29
|
+
if (flowId)
|
|
30
|
+
flows.add(flowId);
|
|
31
|
+
for (const tag of Array.isArray(node.tags) ? node.tags : []) {
|
|
32
|
+
if (String(tag).startsWith('flow:'))
|
|
33
|
+
flows.add(String(tag).slice('flow:'.length));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return [...flows].sort(codePointCompare);
|
|
37
|
+
};
|
|
38
|
+
/** Port of format.py `node_state`. */
|
|
39
|
+
export const nodeState = (node) => {
|
|
40
|
+
for (const tag of Array.isArray(node.tags) ? node.tags : []) {
|
|
41
|
+
if (String(tag).startsWith('state:'))
|
|
42
|
+
return String(tag).slice('state:'.length);
|
|
43
|
+
}
|
|
44
|
+
return '';
|
|
45
|
+
};
|
|
46
|
+
/** Port of format.py `manifest_states`: `state:*` tag counts, key-sorted. */
|
|
47
|
+
export const deriveStates = (graph) => {
|
|
48
|
+
const states = {};
|
|
49
|
+
for (const node of graph.nodes ?? []) {
|
|
50
|
+
const state = nodeState(node);
|
|
51
|
+
if (state)
|
|
52
|
+
states[state] = (states[state] ?? 0) + 1;
|
|
53
|
+
}
|
|
54
|
+
return Object.fromEntries(Object.entries(states).sort(([a], [b]) => codePointCompare(a, b)));
|
|
55
|
+
};
|
|
56
|
+
/** Port of `read_graph_folder`. */
|
|
57
|
+
export const readGraphFolder = async (path) => {
|
|
58
|
+
const manifest = await parseManifest(join(path, MANIFEST_FILE_NAME), path);
|
|
59
|
+
const nodes = await parseJsonl(join(path, NODES_FILE_NAME), NODES_FILE_NAME);
|
|
60
|
+
const edges = await parseJsonl(join(path, EDGES_FILE_NAME), EDGES_FILE_NAME);
|
|
61
|
+
const graph = normalizeArtifactGraph({ nodes, edges });
|
|
62
|
+
const blobFiles = new Map();
|
|
63
|
+
for (const entry of Array.isArray(manifest.blobs) ? manifest.blobs : []) {
|
|
64
|
+
if (!isPlainObject(entry))
|
|
65
|
+
continue;
|
|
66
|
+
const key = typeof entry.key === 'string' ? entry.key : '';
|
|
67
|
+
const file = typeof entry.file === 'string' ? entry.file : '';
|
|
68
|
+
if (key && file)
|
|
69
|
+
blobFiles.set(key, join(path, file));
|
|
70
|
+
}
|
|
71
|
+
return { path, name: basename(path), manifest, graph, blobFiles };
|
|
72
|
+
};
|
|
73
|
+
const parseManifest = async (file, folderPath) => {
|
|
74
|
+
let raw;
|
|
75
|
+
try {
|
|
76
|
+
raw = await readFile(file, 'utf8');
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (isNotFound(error)) {
|
|
80
|
+
const exists = await pathExists(folderPath);
|
|
81
|
+
throw new RunUploadError('folder_not_a_graph_folder', exists
|
|
82
|
+
? `${folderPath} is not an artifact graph run folder: ${MANIFEST_FILE_NAME} is missing. ` +
|
|
83
|
+
`Point the command at the folder that holds ${MANIFEST_FILE_NAME}, ${NODES_FILE_NAME} and ${EDGES_FILE_NAME}.`
|
|
84
|
+
: `No such run folder: ${folderPath}. Expected a folder containing ${MANIFEST_FILE_NAME}.`, { cause: error });
|
|
85
|
+
}
|
|
86
|
+
throw new RunUploadError('folder_unreadable', `Could not read ${MANIFEST_FILE_NAME} in ${folderPath}: ${messageOf(error)}`, { cause: error });
|
|
87
|
+
}
|
|
88
|
+
let parsed;
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(raw);
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
throw new RunUploadError('manifest_invalid_json', `${file} is not valid JSON: ${messageOf(error)}`, { cause: error });
|
|
94
|
+
}
|
|
95
|
+
if (!isPlainObject(parsed)) {
|
|
96
|
+
throw new RunUploadError('manifest_invalid_json', `${file} must contain a JSON object.`);
|
|
97
|
+
}
|
|
98
|
+
if (parsed.schemaVersion !== GRAPH_FOLDER_SCHEMA_VERSION) {
|
|
99
|
+
throw new RunUploadError('manifest_unsupported_schema', `Unsupported run folder schemaVersion ${JSON.stringify(parsed.schemaVersion ?? null)} in ${file}; ` +
|
|
100
|
+
`this CLI uploads ${GRAPH_FOLDER_SCHEMA_VERSION} folders. Re-export the run with a current artifact-graph build.`);
|
|
101
|
+
}
|
|
102
|
+
return parsed;
|
|
103
|
+
};
|
|
104
|
+
const parseJsonl = async (file, label) => {
|
|
105
|
+
let raw;
|
|
106
|
+
try {
|
|
107
|
+
raw = await readFile(file, 'utf8');
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (isNotFound(error)) {
|
|
111
|
+
throw new RunUploadError('folder_not_a_graph_folder', `Run folder is missing ${label}: ${file}.`, { cause: error });
|
|
112
|
+
}
|
|
113
|
+
throw new RunUploadError('folder_unreadable', `Could not read ${file}: ${messageOf(error)}`, {
|
|
114
|
+
cause: error,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
const items = [];
|
|
118
|
+
const lines = raw.split('\n');
|
|
119
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
120
|
+
const line = lines[index].trim();
|
|
121
|
+
if (!line)
|
|
122
|
+
continue;
|
|
123
|
+
let item;
|
|
124
|
+
try {
|
|
125
|
+
item = JSON.parse(line);
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
throw new RunUploadError('jsonl_invalid', `Invalid JSONL in ${label} line ${index + 1}: ${messageOf(error)}`, { cause: error });
|
|
129
|
+
}
|
|
130
|
+
if (!isPlainObject(item)) {
|
|
131
|
+
throw new RunUploadError('jsonl_invalid', `Invalid JSONL in ${label} line ${index + 1}: expected an object.`);
|
|
132
|
+
}
|
|
133
|
+
items.push(item);
|
|
134
|
+
}
|
|
135
|
+
return items;
|
|
136
|
+
};
|
|
137
|
+
const pathExists = async (path) => {
|
|
138
|
+
try {
|
|
139
|
+
await stat(path);
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
const isNotFound = (error) => error != null && typeof error === 'object' && 'code' in error && error.code === 'ENOENT';
|
|
147
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every failure the run-upload path reports.
|
|
3
|
+
*
|
|
4
|
+
* `message` is what the operator reads on stderr, so it always answers both
|
|
5
|
+
* questions: what happened, and what to do next. `code` stays stable for tests
|
|
6
|
+
* and for anyone scripting around the command.
|
|
7
|
+
*
|
|
8
|
+
* Nothing here ever carries the API key. Request URLs are safe to print (the
|
|
9
|
+
* key rides in the Authorization header, never in a query string) but response
|
|
10
|
+
* bodies and header dumps are not echoed wholesale — only the server's own
|
|
11
|
+
* `error.code` / `error.message` / `error.details.blobKeys`.
|
|
12
|
+
*/
|
|
13
|
+
export declare class RunUploadError extends Error {
|
|
14
|
+
readonly code: string;
|
|
15
|
+
readonly status?: number;
|
|
16
|
+
readonly details?: Record<string, unknown>;
|
|
17
|
+
constructor(code: string, message: string, options?: {
|
|
18
|
+
status?: number;
|
|
19
|
+
details?: Record<string, unknown>;
|
|
20
|
+
cause?: unknown;
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=runUploadError.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runUploadError.d.ts","sourceRoot":"","sources":["../../src/runsUpload/runUploadError.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;gBAGxC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAM;CAQtF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every failure the run-upload path reports.
|
|
3
|
+
*
|
|
4
|
+
* `message` is what the operator reads on stderr, so it always answers both
|
|
5
|
+
* questions: what happened, and what to do next. `code` stays stable for tests
|
|
6
|
+
* and for anyone scripting around the command.
|
|
7
|
+
*
|
|
8
|
+
* Nothing here ever carries the API key. Request URLs are safe to print (the
|
|
9
|
+
* key rides in the Authorization header, never in a query string) but response
|
|
10
|
+
* bodies and header dumps are not echoed wholesale — only the server's own
|
|
11
|
+
* `error.code` / `error.message` / `error.details.blobKeys`.
|
|
12
|
+
*/
|
|
13
|
+
export class RunUploadError extends Error {
|
|
14
|
+
code;
|
|
15
|
+
status;
|
|
16
|
+
details;
|
|
17
|
+
constructor(code, message, options = {}) {
|
|
18
|
+
super(message, options.cause === undefined ? undefined : { cause: options.cause });
|
|
19
|
+
this.name = 'RunUploadError';
|
|
20
|
+
this.code = code;
|
|
21
|
+
this.status = options.status;
|
|
22
|
+
this.details = options.details;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type PlannedBlob, type RunUploadPlan } from './buildRunUpload.js';
|
|
2
|
+
export type RunUploadProgressEvent = {
|
|
3
|
+
kind: 'register';
|
|
4
|
+
graphId: string;
|
|
5
|
+
} | {
|
|
6
|
+
kind: 'blob';
|
|
7
|
+
index: number;
|
|
8
|
+
total: number;
|
|
9
|
+
blob: PlannedBlob;
|
|
10
|
+
status: 'uploaded' | 'skipped';
|
|
11
|
+
} | {
|
|
12
|
+
kind: 'push';
|
|
13
|
+
graphId: string;
|
|
14
|
+
rev: number;
|
|
15
|
+
};
|
|
16
|
+
export type RunUploadResult = {
|
|
17
|
+
graphId: string;
|
|
18
|
+
streamId: string;
|
|
19
|
+
rev: number;
|
|
20
|
+
graphHash: string;
|
|
21
|
+
registered: boolean;
|
|
22
|
+
blobsUploaded: number;
|
|
23
|
+
blobsSkipped: number;
|
|
24
|
+
bytesUploaded: number;
|
|
25
|
+
/**
|
|
26
|
+
* `published` — the snapshot became the graph's current revision.
|
|
27
|
+
* `unchanged` — an exact re-push of what the server already holds.
|
|
28
|
+
* `stale` — the server already holds a NEWER revision, so this upload is
|
|
29
|
+
* a no-op rather than a failure.
|
|
30
|
+
*/
|
|
31
|
+
status: 'published' | 'unchanged' | 'stale';
|
|
32
|
+
data: unknown;
|
|
33
|
+
};
|
|
34
|
+
export type UploadRunInput = {
|
|
35
|
+
plan: RunUploadPlan;
|
|
36
|
+
baseUrl: string;
|
|
37
|
+
apiKey: string;
|
|
38
|
+
fetchImpl?: typeof fetch;
|
|
39
|
+
skipRegister?: boolean;
|
|
40
|
+
onProgress?: (event: RunUploadProgressEvent) => void;
|
|
41
|
+
};
|
|
42
|
+
export declare const uploadRun: ({ plan, baseUrl, apiKey, fetchImpl, skipRegister, onProgress, }: UploadRunInput) => Promise<RunUploadResult>;
|
|
43
|
+
//# sourceMappingURL=uploadRun.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uploadRun.d.ts","sourceRoot":"","sources":["../../src/runsUpload/uploadRun.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAc,KAAK,WAAW,EAAE,KAAK,aAAa,EAAC,MAAM,qBAAqB,CAAA;AAGrF,MAAM,MAAM,sBAAsB,GAC9B;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACnC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,UAAU,GAAG,SAAS,CAAA;CAAC,GAC/F;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAC,CAAA;AAEhD,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,MAAM,CAAA;IACX,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,OAAO,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB;;;;;OAKG;IACH,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,OAAO,CAAA;IAC3C,IAAI,EAAE,OAAO,CAAA;CACd,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,aAAa,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,CAAA;CACrD,CAAA;AAED,eAAO,MAAM,SAAS,GAAU,iEAO7B,cAAc,KAAG,OAAO,CAAC,eAAe,CA0F1C,CAAA"}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
// Sending a planned run to Production Control.
|
|
2
|
+
//
|
|
3
|
+
// Order is the registry's: register -> blobs -> snapshot. The snapshot is
|
|
4
|
+
// rejected while any blob it references is absent from the pool, so the bytes
|
|
5
|
+
// have to land first.
|
|
6
|
+
//
|
|
7
|
+
// Resumability is the reason blobs go one at a time behind a HEAD. Content
|
|
8
|
+
// addressing makes an upload idempotent — the same key always holds the same
|
|
9
|
+
// bytes — so a run that dies at blob 150 of 300 is finished by re-running the
|
|
10
|
+
// exact same command: the first 150 HEAD as present and are skipped. Nothing
|
|
11
|
+
// records progress locally, because the server already knows it.
|
|
12
|
+
//
|
|
13
|
+
// Memory: exactly one blob is resident at a time, and `buildRunUploadPlan` has
|
|
14
|
+
// already refused any blob over the per-request ceiling by stat'ing it, so an
|
|
15
|
+
// oversized file is never read at all.
|
|
16
|
+
import { readFile } from 'node:fs/promises';
|
|
17
|
+
import { createHash } from 'node:crypto';
|
|
18
|
+
import { CONTROL_ERROR_CODES, CONTROL_NOT_ENTITLED_STATUS, controlRunUploadUrls, } from './controlEndpoints.js';
|
|
19
|
+
import { RunUploadError } from './runUploadError.js';
|
|
20
|
+
export const uploadRun = async ({ plan, baseUrl, apiKey, fetchImpl = globalThis.fetch.bind(globalThis), skipRegister = false, onProgress = () => { }, }) => {
|
|
21
|
+
const urls = controlRunUploadUrls(baseUrl);
|
|
22
|
+
// Built once, never logged, never placed in a URL or a query string.
|
|
23
|
+
const authHeaders = { authorization: `Bearer ${apiKey}` };
|
|
24
|
+
let registered = false;
|
|
25
|
+
if (!skipRegister) {
|
|
26
|
+
onProgress({ kind: 'register', graphId: plan.graphId });
|
|
27
|
+
await request(fetchImpl, urls.graphs, {
|
|
28
|
+
method: 'POST',
|
|
29
|
+
headers: { ...authHeaders, 'content-type': 'application/json' },
|
|
30
|
+
body: JSON.stringify(plan.registration),
|
|
31
|
+
});
|
|
32
|
+
registered = true;
|
|
33
|
+
}
|
|
34
|
+
let blobsUploaded = 0;
|
|
35
|
+
let blobsSkipped = 0;
|
|
36
|
+
let bytesUploaded = 0;
|
|
37
|
+
for (let index = 0; index < plan.blobs.length; index += 1) {
|
|
38
|
+
const blob = plan.blobs[index];
|
|
39
|
+
const url = urls.blob(plan.graphId, blob.blobKey);
|
|
40
|
+
if (await blobExists(fetchImpl, url, authHeaders)) {
|
|
41
|
+
blobsSkipped += 1;
|
|
42
|
+
onProgress({ kind: 'blob', index, total: plan.blobs.length, blob, status: 'skipped' });
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const bytes = await readVerifiedBlob(blob);
|
|
46
|
+
await request(fetchImpl, url, {
|
|
47
|
+
method: 'PUT',
|
|
48
|
+
headers: {
|
|
49
|
+
...authHeaders,
|
|
50
|
+
'content-type': blob.mime,
|
|
51
|
+
'x-artifact-blob-name': encodeURIComponent(blob.name),
|
|
52
|
+
'x-artifact-blob-sha256': blob.sha256,
|
|
53
|
+
'x-artifact-blob-size': String(blob.size),
|
|
54
|
+
},
|
|
55
|
+
body: bytes,
|
|
56
|
+
});
|
|
57
|
+
blobsUploaded += 1;
|
|
58
|
+
bytesUploaded += blob.size;
|
|
59
|
+
onProgress({ kind: 'blob', index, total: plan.blobs.length, blob, status: 'uploaded' });
|
|
60
|
+
}
|
|
61
|
+
onProgress({ kind: 'push', graphId: plan.graphId, rev: plan.rev });
|
|
62
|
+
let payload;
|
|
63
|
+
try {
|
|
64
|
+
payload = await request(fetchImpl, urls.snapshot(plan.graphId), {
|
|
65
|
+
method: 'PUT',
|
|
66
|
+
headers: { ...authHeaders, 'content-type': 'application/json' },
|
|
67
|
+
body: plan.pushBody,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
// The server already holds a newer revision. That is the end state the
|
|
72
|
+
// caller wanted, so it is reported, not thrown.
|
|
73
|
+
if (error instanceof RunUploadError && error.code === CONTROL_ERROR_CODES.staleSnapshot) {
|
|
74
|
+
return {
|
|
75
|
+
graphId: plan.graphId,
|
|
76
|
+
streamId: plan.streamId,
|
|
77
|
+
rev: plan.rev,
|
|
78
|
+
graphHash: plan.graphHash,
|
|
79
|
+
registered,
|
|
80
|
+
blobsUploaded,
|
|
81
|
+
blobsSkipped,
|
|
82
|
+
bytesUploaded,
|
|
83
|
+
status: 'stale',
|
|
84
|
+
data: error.details ?? null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
const data = payload.data ?? null;
|
|
90
|
+
const serverStatus = data != null && typeof data === 'object' && 'status' in data
|
|
91
|
+
? String(data.status)
|
|
92
|
+
: '';
|
|
93
|
+
return {
|
|
94
|
+
graphId: plan.graphId,
|
|
95
|
+
streamId: plan.streamId,
|
|
96
|
+
rev: plan.rev,
|
|
97
|
+
graphHash: plan.graphHash,
|
|
98
|
+
registered,
|
|
99
|
+
blobsUploaded,
|
|
100
|
+
blobsSkipped,
|
|
101
|
+
bytesUploaded,
|
|
102
|
+
status: serverStatus === 'unchanged' ? 'unchanged' : 'published',
|
|
103
|
+
data,
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
const blobExists = async (fetchImpl, url, authHeaders) => {
|
|
107
|
+
const response = await send(fetchImpl, url, { method: 'HEAD', headers: authHeaders });
|
|
108
|
+
if (response.ok)
|
|
109
|
+
return true;
|
|
110
|
+
if (response.status === 404)
|
|
111
|
+
return false;
|
|
112
|
+
throw await errorFrom(response, url, 'HEAD');
|
|
113
|
+
};
|
|
114
|
+
const readVerifiedBlob = async (blob) => {
|
|
115
|
+
const bytes = await readFile(blob.file);
|
|
116
|
+
const digest = createHash('sha256').update(bytes).digest('hex');
|
|
117
|
+
// The server recomputes this and rejects a mismatch; catching it here names
|
|
118
|
+
// the local file instead of surfacing an opaque 400.
|
|
119
|
+
if (digest !== blob.sha256) {
|
|
120
|
+
throw new RunUploadError('blob_digest_mismatch', `${blob.file} hashes to sha256:${digest} but the run folder declares ${blob.blobKey}. ` +
|
|
121
|
+
'The file changed since the folder was exported. Nothing further was sent; re-export the run.', { details: { blobKey: blob.blobKey, file: blob.file } });
|
|
122
|
+
}
|
|
123
|
+
if (bytes.byteLength !== blob.size) {
|
|
124
|
+
throw new RunUploadError('blob_size_mismatch', `${blob.file} is ${bytes.byteLength} bytes but the run folder declares ${blob.size}. ` +
|
|
125
|
+
'The file changed since the folder was exported. Nothing further was sent; re-export the run.', { details: { blobKey: blob.blobKey, file: blob.file } });
|
|
126
|
+
}
|
|
127
|
+
return bytes;
|
|
128
|
+
};
|
|
129
|
+
const request = async (fetchImpl, url, init) => {
|
|
130
|
+
const method = String(init.method ?? 'GET');
|
|
131
|
+
const response = await send(fetchImpl, url, init);
|
|
132
|
+
const payload = await readJson(response);
|
|
133
|
+
if (!response.ok || payload?.success !== true) {
|
|
134
|
+
throw await errorFrom(response, url, method, payload);
|
|
135
|
+
}
|
|
136
|
+
return payload;
|
|
137
|
+
};
|
|
138
|
+
const send = async (fetchImpl, url, init) => {
|
|
139
|
+
try {
|
|
140
|
+
return await fetchImpl(url, init);
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
// Nothing here echoes the request headers, so the API key cannot leak into
|
|
144
|
+
// a transport error.
|
|
145
|
+
throw new RunUploadError('network_unreachable', `Could not reach Production Control at ${originOf(url)}: ${causeText(error)}. ` +
|
|
146
|
+
'Check your network and --base-url (or ASSETHUB_API_BASE_URL), then run the same command again — ' +
|
|
147
|
+
'blobs that already landed are detected and skipped, so the upload continues where it stopped.', { cause: error });
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
const readJson = async (response) => {
|
|
151
|
+
try {
|
|
152
|
+
return (await response.json());
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
// HEAD, 204, or a proxy's HTML error page.
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
const errorFrom = async (response, url, method, parsed) => {
|
|
160
|
+
const payload = parsed === undefined ? await readJson(response) : parsed;
|
|
161
|
+
const code = payload?.error?.code ?? `HTTP_${response.status}`;
|
|
162
|
+
const message = payload?.error?.message ?? `${method} ${originOf(url)} failed with ${response.status}`;
|
|
163
|
+
const details = payload?.error?.details;
|
|
164
|
+
return new RunUploadError(code, `${message}${hintFor(code, response.status, details)}`, {
|
|
165
|
+
status: response.status,
|
|
166
|
+
details,
|
|
167
|
+
});
|
|
168
|
+
};
|
|
169
|
+
/** The "what do I do now" half of every server-side failure. */
|
|
170
|
+
const hintFor = (code, status, details) => {
|
|
171
|
+
if (code === CONTROL_ERROR_CODES.missingBlobs) {
|
|
172
|
+
const keys = Array.isArray(details?.blobKeys) ? details.blobKeys.map(String) : [];
|
|
173
|
+
return (`\nProduction Control is missing ${keys.length || 'some'} of this run's blobs` +
|
|
174
|
+
(keys.length > 0 ? `: ${keys.slice(0, 5).join(', ')}${keys.length > 5 ? ', …' : ''}` : '') +
|
|
175
|
+
'.\nRun the same command again — it re-checks every blob and sends only the absent ones.');
|
|
176
|
+
}
|
|
177
|
+
if (code === CONTROL_ERROR_CODES.snapshotConflict) {
|
|
178
|
+
return ('\nProduction Control already holds a different snapshot at this revision.' +
|
|
179
|
+
'\nUpload as a new revision with --rev <n> higher than the stored one, or --graph-id to publish it as a separate run.');
|
|
180
|
+
}
|
|
181
|
+
if (code === CONTROL_ERROR_CODES.payloadTooLarge || status === 413) {
|
|
182
|
+
return '\nThe request exceeded the size Production Control accepts. Re-export the run with smaller assets, or split it.';
|
|
183
|
+
}
|
|
184
|
+
// Both of these are 404, so the code is what separates them. Reading a
|
|
185
|
+
// NOT_REGISTERED as "you have no access" would send someone chasing an
|
|
186
|
+
// entitlement they already have.
|
|
187
|
+
if (code === CONTROL_ERROR_CODES.notRegistered) {
|
|
188
|
+
return ('\nThat graph is not registered for your org, so there is nothing to upload into.' +
|
|
189
|
+
'\nDrop --skip-register (registration is part of the normal upload), or pass the --graph-id of a run you have already registered.');
|
|
190
|
+
}
|
|
191
|
+
if (status === CONTROL_NOT_ENTITLED_STATUS) {
|
|
192
|
+
return ('\nThis API key cannot upload runs to Production Control. That needs an internal account whose org has the Production Control entitlement.' +
|
|
193
|
+
'\nCheck `assethub auth status` is using the key you meant, then ask for access.');
|
|
194
|
+
}
|
|
195
|
+
if (status === 401 || status === 403) {
|
|
196
|
+
return '\nThe API key was rejected. Re-run `assethub auth login --api-key-stdin`, or pass --api-key.';
|
|
197
|
+
}
|
|
198
|
+
if (status >= 500) {
|
|
199
|
+
return '\nThat is a server-side failure. Run the same command again — already-uploaded blobs are skipped, so a retry is cheap.';
|
|
200
|
+
}
|
|
201
|
+
return '';
|
|
202
|
+
};
|
|
203
|
+
const originOf = (url) => {
|
|
204
|
+
try {
|
|
205
|
+
return new URL(url).origin;
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return url;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
const causeText = (error) => {
|
|
212
|
+
if (!(error instanceof Error))
|
|
213
|
+
return String(error);
|
|
214
|
+
const cause = error.cause;
|
|
215
|
+
const causeMessage = cause instanceof Error ? cause.message : '';
|
|
216
|
+
return causeMessage ? `${error.message} (${causeMessage})` : error.message;
|
|
217
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@assethub/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Command line interface for AssetHub.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -20,19 +20,22 @@
|
|
|
20
20
|
"README.md",
|
|
21
21
|
"LICENSE"
|
|
22
22
|
],
|
|
23
|
-
"scripts": {
|
|
24
|
-
"build": "tsc -p tsconfig.json && chmod +x dist/index.js",
|
|
25
|
-
"typecheck": "tsc -p tsconfig.typecheck.json --noEmit"
|
|
26
|
-
},
|
|
27
23
|
"license": "MIT",
|
|
28
24
|
"publishConfig": {
|
|
29
25
|
"access": "public"
|
|
30
26
|
},
|
|
31
27
|
"dependencies": {
|
|
32
|
-
"@assethub/api-client": "^0.1.
|
|
28
|
+
"@assethub/api-client": "^0.1.2"
|
|
33
29
|
},
|
|
34
30
|
"devDependencies": {
|
|
35
31
|
"@types/node": "24.10.1",
|
|
36
|
-
"typescript": "5.9.3"
|
|
32
|
+
"typescript": "5.9.3",
|
|
33
|
+
"vitest": "^4.1.5",
|
|
34
|
+
"@assethub/autopilot-core": "0.0.1"
|
|
35
|
+
},
|
|
36
|
+
"scripts": {
|
|
37
|
+
"build": "tsc -p tsconfig.json && chmod +x dist/index.js",
|
|
38
|
+
"test": "vitest run",
|
|
39
|
+
"typecheck": "tsc -p tsconfig.typecheck.json --noEmit"
|
|
37
40
|
}
|
|
38
|
-
}
|
|
41
|
+
}
|