@pipelex/sdk 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/artifacts.d.ts +92 -27
- package/dist/artifacts.js +308 -100
- package/dist/artifacts.js.map +1 -1
- package/dist/client.d.ts +9 -4
- package/dist/client.js +18 -6
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +30 -5
- package/dist/errors.js +9 -5
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/dist/runs.d.ts +10 -1
- package/dist/runs.js +17 -1
- package/dist/runs.js.map +1 -1
- package/dist/timers.d.ts +11 -0
- package/dist/timers.js +14 -0
- package/dist/timers.js.map +1 -0
- package/dist/upload-grant.d.ts +38 -15
- package/dist/upload-grant.js +200 -67
- package/dist/upload-grant.js.map +1 -1
- package/dist/upload.js +9 -2
- package/dist/upload.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +7 -6
- package/docs/artifact-download.md +50 -15
- package/docs/input-preparation.md +14 -8
- package/docs/run-results.md +1 -1
- package/package.json +2 -2
package/dist/artifacts.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* The artifact stack — the download twin of `prepareInputs`, in layers so each
|
|
3
3
|
* operation is usable without the next:
|
|
4
4
|
*
|
|
5
|
-
* - {@link
|
|
6
|
-
* ARE `pipelex-storage://` references
|
|
5
|
+
* - {@link locateArtifacts} — a pure walk of any JSON value for the strings that
|
|
6
|
+
* ARE `pipelex-storage://` references, each with every path it sits at.
|
|
7
|
+
* {@link collectArtifacts} is the same walk's bare references. No network, no key.
|
|
7
8
|
* - {@link resolveArtifacts} — the platform's bulk resolve route over a whole
|
|
8
9
|
* list, chunked at the route's bound, one verdict per reference.
|
|
9
10
|
* - {@link fetchArtifact} — a bounded `Response` for one reference: resolved
|
|
@@ -20,12 +21,13 @@
|
|
|
20
21
|
*
|
|
21
22
|
* `node:fs/promises`, `node:path` and `undici` are imported dynamically, behind
|
|
22
23
|
* `isNodeRuntime()`, so the module's static import graph stays free of Node
|
|
23
|
-
* builtins:
|
|
24
|
+
* builtins: the walk and `resolveArtifacts` run anywhere, `fetchArtifact`
|
|
24
25
|
* runs anywhere the global `fetch` does (the dispatcher is a Node refinement), and
|
|
25
26
|
* only `downloadArtifacts` touches a filesystem.
|
|
26
27
|
*/
|
|
27
28
|
import { ApiResponseError, ArtifactAuthenticationError, ArtifactFetchError, ArtifactOperationError, RunFailedError, RunStillRunningError, ScopeUnavailableError, } from "./errors.js";
|
|
28
29
|
import { isNodeRuntime } from "./upload.js";
|
|
30
|
+
import { MAX_TIMER_DELAY_MS, isTimerDelay } from "./timers.js";
|
|
29
31
|
// ── Constants ────────────────────────────────────────────────────────
|
|
30
32
|
/** The scheme of a durable storage reference. */
|
|
31
33
|
export const PIPELEX_STORAGE_SCHEME = "pipelex-storage://";
|
|
@@ -56,12 +58,24 @@ export const DEFAULT_DOWNLOAD_CONCURRENCY = 4;
|
|
|
56
58
|
const EXPIRY_MARGIN_MS = 10_000;
|
|
57
59
|
/** Longest filename `downloadArtifacts` writes, extension included. */
|
|
58
60
|
const MAX_FILENAME_LENGTH = 128;
|
|
61
|
+
/**
|
|
62
|
+
* Longest extension taken from a storage key, dot excluded. Anything longer after
|
|
63
|
+
* the key's last dot is read as part of a name rather than as an extension, and
|
|
64
|
+
* the content type's extension is used instead.
|
|
65
|
+
*/
|
|
66
|
+
const MAX_EXTENSION_LENGTH = 10;
|
|
67
|
+
/**
|
|
68
|
+
* Stems Windows reserves for a device, in any case and whatever the extension:
|
|
69
|
+
* `aux.png` there names the auxiliary device, not a file. A stem is a field name
|
|
70
|
+
* the method author chose, so `$.aux.url` would otherwise reach one.
|
|
71
|
+
*/
|
|
72
|
+
const WINDOWS_DEVICE_STEM = /^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
|
|
59
73
|
/** Ceiling on collision suffixes before the never-overwrite rule gives up. */
|
|
60
74
|
const MAX_UNIQUE_ATTEMPTS = 10_000;
|
|
61
75
|
/**
|
|
62
|
-
* The extension
|
|
63
|
-
* type is one of the artifact types a run produces. Deliberately
|
|
64
|
-
* unknown type simply gets no extension, never a guessed one.
|
|
76
|
+
* The extension a saved file takes when its storage key carries none and the
|
|
77
|
+
* resolved content type is one of the artifact types a run produces. Deliberately
|
|
78
|
+
* short: an unknown type simply gets no extension, never a guessed one.
|
|
65
79
|
*/
|
|
66
80
|
const EXTENSION_BY_CONTENT_TYPE = {
|
|
67
81
|
"image/png": ".png",
|
|
@@ -76,37 +90,138 @@ const EXTENSION_BY_CONTENT_TYPE = {
|
|
|
76
90
|
"text/csv": ".csv",
|
|
77
91
|
"application/json": ".json",
|
|
78
92
|
};
|
|
79
|
-
|
|
93
|
+
/** An object key rendered as `.key` in a path; any other key is rendered as `["…"]`. */
|
|
94
|
+
const IDENTIFIER_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
95
|
+
/**
|
|
96
|
+
* Every `pipelex-storage://` reference inside a JSON-shaped value, each with
|
|
97
|
+
* every path at which it occurs. The references are deduplicated and kept in
|
|
98
|
+
* discovery order (the order of their first sighting), and each one's `found_at`
|
|
99
|
+
* lists its paths in walk order, so `found_at[0]` is where it was first seen.
|
|
100
|
+
*
|
|
101
|
+
* The string test is {@link collectArtifacts}'s: a string counts only when it IS
|
|
102
|
+
* a reference. A path is rooted at `$`, the walked value itself; an object key
|
|
103
|
+
* matching `^[A-Za-z_][A-Za-z0-9_]*$` is written `.key`, any other key `["…"]` in
|
|
104
|
+
* JSON string escaping, and an array index `[n]`. The runtime serializes a
|
|
105
|
+
* produced image or document as content carrying its reference in `url`, so a
|
|
106
|
+
* typical path ends there: `$.rooms[3].staged_photo.url`, `$.items[0].url`, or
|
|
107
|
+
* `$.url` for an output that is one image. Pure — no network, no key.
|
|
108
|
+
*/
|
|
109
|
+
export function locateArtifacts(value) {
|
|
110
|
+
return walkReferences(value).map(renderLocation);
|
|
111
|
+
}
|
|
80
112
|
/**
|
|
81
113
|
* Every `pipelex-storage://` reference inside a JSON-shaped value, deduplicated,
|
|
82
|
-
* in discovery order
|
|
83
|
-
* string
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
114
|
+
* in discovery order — the references of {@link locateArtifacts}, without their
|
|
115
|
+
* paths. A string counts only when it IS a reference — the whole string, scheme
|
|
116
|
+
* first, with something after the scheme; text that merely contains one does
|
|
117
|
+
* not. The scheme is unambiguous, so this walk is a contract rather than a
|
|
118
|
+
* heuristic: the runtime serializes a produced image or document as content
|
|
119
|
+
* carrying its reference in `url`, beside an expiring `public_url` this walk
|
|
120
|
+
* ignores. Pure — no network, no key — so a consumer can count or list a run's
|
|
121
|
+
* produced files without resolving any of them.
|
|
89
122
|
*/
|
|
90
123
|
export function collectArtifacts(value) {
|
|
91
|
-
|
|
92
|
-
walkForReferences(value, found);
|
|
93
|
-
return [...found];
|
|
124
|
+
return walkReferences(value, false).map((located) => located.uri);
|
|
94
125
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
126
|
+
/**
|
|
127
|
+
* The walk both public functions share: depth first, keys in `Object.entries`
|
|
128
|
+
* order. With `withPaths` off it records each reference once and no path at
|
|
129
|
+
* all, which is what `collectArtifacts` needs: copying the trail for every
|
|
130
|
+
* occurrence costs memory in proportion to occurrences times depth, where the
|
|
131
|
+
* deduplicated list needs only one entry per unique reference.
|
|
132
|
+
*/
|
|
133
|
+
function walkReferences(value, withPaths = true) {
|
|
134
|
+
// A Map keeps insertion order, which is the order of first sighting.
|
|
135
|
+
const byUri = new Map();
|
|
136
|
+
const trail = [];
|
|
137
|
+
const visit = (node) => {
|
|
138
|
+
if (typeof node === "string") {
|
|
139
|
+
if (!isStorageReference(node))
|
|
140
|
+
return;
|
|
141
|
+
const known = byUri.get(node);
|
|
142
|
+
if (!withPaths) {
|
|
143
|
+
if (known === undefined)
|
|
144
|
+
byUri.set(node, { uri: node, paths: [] });
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const path = [...trail];
|
|
148
|
+
if (known === undefined)
|
|
149
|
+
byUri.set(node, { uri: node, paths: [path] });
|
|
150
|
+
else
|
|
151
|
+
known.paths.push(path);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
if (Array.isArray(node)) {
|
|
155
|
+
for (let index = 0; index < node.length; index += 1) {
|
|
156
|
+
trail.push(index);
|
|
157
|
+
visit(node[index]);
|
|
158
|
+
trail.pop();
|
|
159
|
+
}
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
if (typeof node === "object" && node !== null) {
|
|
163
|
+
for (const [key, entry] of Object.entries(node)) {
|
|
164
|
+
trail.push(key);
|
|
165
|
+
visit(entry);
|
|
166
|
+
trail.pop();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
visit(value);
|
|
171
|
+
return [...byUri.values()];
|
|
172
|
+
}
|
|
173
|
+
function renderLocation(located) {
|
|
174
|
+
return { uri: located.uri, found_at: located.paths.map(renderPath) };
|
|
175
|
+
}
|
|
176
|
+
/** Segments to the `$`-rooted notation `found_at` carries. */
|
|
177
|
+
function renderPath(segments) {
|
|
178
|
+
let path = "$";
|
|
179
|
+
for (const segment of segments) {
|
|
180
|
+
if (typeof segment === "number")
|
|
181
|
+
path += `[${segment}]`;
|
|
182
|
+
else if (IDENTIFIER_KEY.test(segment))
|
|
183
|
+
path += `.${segment}`;
|
|
184
|
+
else
|
|
185
|
+
path += `[${JSON.stringify(segment)}]`;
|
|
105
186
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
187
|
+
return path;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The `$`-rooted notation back to segments, for a location that reaches
|
|
191
|
+
* {@link artifactFilename} from outside the walk. The rendering is lossless, so
|
|
192
|
+
* this is exact for every path the walk produced; anything else is `undefined`.
|
|
193
|
+
*/
|
|
194
|
+
function parsePath(path) {
|
|
195
|
+
if (!path.startsWith("$"))
|
|
196
|
+
return undefined;
|
|
197
|
+
const segments = [];
|
|
198
|
+
let at = 1;
|
|
199
|
+
while (at < path.length) {
|
|
200
|
+
const rest = path.slice(at);
|
|
201
|
+
const key = /^\.([A-Za-z_][A-Za-z0-9_]*)/.exec(rest);
|
|
202
|
+
if (key !== null) {
|
|
203
|
+
segments.push(key[1]);
|
|
204
|
+
at += key[0].length;
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const index = /^\[(\d+)\]/.exec(rest);
|
|
208
|
+
if (index !== null) {
|
|
209
|
+
segments.push(Number(index[1]));
|
|
210
|
+
at += index[0].length;
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
const quoted = /^\[("(?:[^"\\]|\\.)*")\]/s.exec(rest);
|
|
214
|
+
if (quoted === null)
|
|
215
|
+
return undefined;
|
|
216
|
+
try {
|
|
217
|
+
segments.push(JSON.parse(quoted[1]));
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
at += quoted[0].length;
|
|
109
223
|
}
|
|
224
|
+
return segments;
|
|
110
225
|
}
|
|
111
226
|
/** A string that is a storage reference: the scheme, then at least one character. */
|
|
112
227
|
export function isStorageReference(value) {
|
|
@@ -114,18 +229,95 @@ export function isStorageReference(value) {
|
|
|
114
229
|
}
|
|
115
230
|
// ── artifactFilename ─────────────────────────────────────────────────
|
|
116
231
|
/**
|
|
117
|
-
* The bare filename a
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
232
|
+
* The bare filename a reference is saved under, named after the field it fills:
|
|
233
|
+
* the path in `location.found_at[0]`, where the reference was first seen.
|
|
234
|
+
*
|
|
235
|
+
* 1. A final object key `url` is dropped, since the runtime's image and
|
|
236
|
+
* document contents carry their reference there; a reference under any other
|
|
237
|
+
* key keeps that key, and a `url` key that is not final is kept.
|
|
238
|
+
* 2. Each key is reduced to `[A-Za-z0-9_]`, every other character (`-` and `.`
|
|
239
|
+
* included) becoming `_`; an index stays its decimal digits.
|
|
240
|
+
* 3. The segments are joined with `-`. An empty result — the reference is the
|
|
241
|
+
* walked value itself, or its `url` — becomes the scope's name.
|
|
242
|
+
* 4. Over the filename length cap (`MAX_FILENAME_LENGTH`, extension included),
|
|
243
|
+
* the tail is kept: whole leading segments are dropped first, since the last
|
|
244
|
+
* ones are the specific ones, and a single segment still too long is cut to
|
|
245
|
+
* fit.
|
|
246
|
+
* 5. A stem Windows reserves for a device (`con`, `prn`, `aux`, `nul`,
|
|
247
|
+
* `com0`–`com9`, `lpt0`–`lpt9`, in any case) gets a trailing `_`, so
|
|
248
|
+
* `$.aux.url` is saved as `aux_.png`.
|
|
249
|
+
* 6. The extension is the storage key's own, reduced to `[A-Za-z0-9]`, when it
|
|
250
|
+
* has a short one; otherwise the content type's, for the types a run
|
|
251
|
+
* produces; otherwise there is none.
|
|
252
|
+
*
|
|
253
|
+
* So `$.rooms[3].staged_photo.url` is saved as `rooms-3-staged_photo.png`, and
|
|
254
|
+
* `$.url` in `main_stuff` as `main_stuff.png`. The stem is ASCII letters, digits,
|
|
255
|
+
* `_` and the `-` joins, never empty and never a device name, so the name can
|
|
256
|
+
* only ever be a regular file directly inside the target directory. A collision on disk is not this
|
|
125
257
|
* function's concern: `downloadArtifacts` suffixes the stem (`name-1.ext`) on
|
|
126
258
|
* exclusive creation, so a file is never overwritten.
|
|
259
|
+
*
|
|
260
|
+
* Throws `ArtifactOperationError` for a location whose `found_at[0]` is not a path
|
|
261
|
+
* in the notation {@link locateArtifacts} writes, or for an unknown scope.
|
|
127
262
|
*/
|
|
128
|
-
export function artifactFilename(
|
|
263
|
+
export function artifactFilename(location, contentType, scope) {
|
|
264
|
+
requireScope(scope);
|
|
265
|
+
// A JavaScript caller can hand anything here — the old signature's bare uri
|
|
266
|
+
// string among them — and every shape must reach the documented refusal
|
|
267
|
+
// rather than a TypeError, or a string's first character.
|
|
268
|
+
const loose = location;
|
|
269
|
+
const uri = loose?.uri;
|
|
270
|
+
if (typeof uri !== "string") {
|
|
271
|
+
throw new ArtifactOperationError(`artifactFilename needs a location carrying its reference as a string "uri"; got ${String(uri)}.`);
|
|
272
|
+
}
|
|
273
|
+
const foundAt = loose?.found_at;
|
|
274
|
+
const first = Array.isArray(foundAt) ? foundAt[0] : undefined;
|
|
275
|
+
const segments = typeof first === "string" ? parsePath(first) : undefined;
|
|
276
|
+
if (segments === undefined) {
|
|
277
|
+
throw new ArtifactOperationError(`artifactFilename needs a location whose first "found_at" entry is a path such as ` +
|
|
278
|
+
`"$.items[0].url"; got ${typeof first === "string" ? JSON.stringify(first) : String(first)}.`);
|
|
279
|
+
}
|
|
280
|
+
return filenameFor(segments, uri, contentType, scope);
|
|
281
|
+
}
|
|
282
|
+
/** The naming rule of {@link artifactFilename}, over the walk's own segments. */
|
|
283
|
+
function filenameFor(segments, uri, contentType, scope) {
|
|
284
|
+
const named = segments.at(-1) === "url" ? segments.slice(0, -1) : segments;
|
|
285
|
+
const words = named
|
|
286
|
+
.map((segment) => typeof segment === "number" ? String(segment) : segment.replace(/[^A-Za-z0-9_]/gu, "_"))
|
|
287
|
+
// Only the empty key reduces to nothing, and it says nothing about the field.
|
|
288
|
+
.filter((word) => word !== "");
|
|
289
|
+
const extension = extensionFor(uri, contentType);
|
|
290
|
+
const stem = fitStem(words.length > 0 ? words : [scope], MAX_FILENAME_LENGTH - extension.length);
|
|
291
|
+
// A device stem is at most four characters, so the `_` cannot overrun the cap.
|
|
292
|
+
return (WINDOWS_DEVICE_STEM.test(stem) ? `${stem}_` : stem) + extension;
|
|
293
|
+
}
|
|
294
|
+
/** The words joined with `-` within `budget` characters, keeping the tail. */
|
|
295
|
+
function fitStem(words, budget) {
|
|
296
|
+
let start = 0;
|
|
297
|
+
let length = words.reduce((sum, word) => sum + word.length, 0) + words.length - 1;
|
|
298
|
+
while (length > budget && start < words.length - 1) {
|
|
299
|
+
length -= words[start].length + 1;
|
|
300
|
+
start += 1;
|
|
301
|
+
}
|
|
302
|
+
const stem = words.slice(start).join("-");
|
|
303
|
+
return stem.length > budget ? stem.slice(0, budget) : stem;
|
|
304
|
+
}
|
|
305
|
+
/** `.ext` for the saved file — the storage key's own, else the content type's — or `""`. */
|
|
306
|
+
function extensionFor(uri, contentType) {
|
|
307
|
+
const fromKey = storageKeyExtension(uri);
|
|
308
|
+
if (fromKey !== "")
|
|
309
|
+
return `.${fromKey}`;
|
|
310
|
+
if (contentType == null)
|
|
311
|
+
return "";
|
|
312
|
+
return EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()] ?? "";
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* The extension the storage key's last segment carries, without its dot, reduced
|
|
316
|
+
* to `[A-Za-z0-9]` — or `""` when it has none, or none that short. The segment is
|
|
317
|
+
* what follows the last `/` or `\` once the scheme, query and fragment are gone,
|
|
318
|
+
* percent-decoded when it decodes; a leading dot is not an extension.
|
|
319
|
+
*/
|
|
320
|
+
function storageKeyExtension(uri) {
|
|
129
321
|
const key = uri.startsWith(PIPELEX_STORAGE_SCHEME)
|
|
130
322
|
? uri.slice(PIPELEX_STORAGE_SCHEME.length)
|
|
131
323
|
: uri;
|
|
@@ -138,41 +330,24 @@ export function artifactFilename(uri, contentType, index) {
|
|
|
138
330
|
decoded = decodeURIComponent(segment);
|
|
139
331
|
}
|
|
140
332
|
catch {
|
|
141
|
-
// A malformed escape sequence is kept as typed;
|
|
142
|
-
}
|
|
143
|
-
let name = decoded
|
|
144
|
-
.replace(/[^A-Za-z0-9._-]/g, "_")
|
|
145
|
-
.replace(/^[._-]+/, "")
|
|
146
|
-
.replace(/[._-]+$/, "");
|
|
147
|
-
if (name === "") {
|
|
148
|
-
name = `artifact-${index + 1}`;
|
|
149
|
-
}
|
|
150
|
-
if (name.length > MAX_FILENAME_LENGTH) {
|
|
151
|
-
const ext = extensionOf(name);
|
|
152
|
-
// The extension is kept only if there is room left for a stem. An extension
|
|
153
|
-
// at least as long as the cap would give `slice` a negative start, which
|
|
154
|
-
// counts from the END and yields a name LONGER than the cap — so a
|
|
155
|
-
// pathological extension is dropped rather than preserved.
|
|
156
|
-
name =
|
|
157
|
-
ext.length < MAX_FILENAME_LENGTH
|
|
158
|
-
? name.slice(0, MAX_FILENAME_LENGTH - ext.length) + ext
|
|
159
|
-
: name.slice(0, MAX_FILENAME_LENGTH);
|
|
160
|
-
}
|
|
161
|
-
if (extensionOf(name) === "") {
|
|
162
|
-
const ext = contentType == null
|
|
163
|
-
? undefined
|
|
164
|
-
: EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()];
|
|
165
|
-
if (ext !== undefined) {
|
|
166
|
-
name += ext;
|
|
167
|
-
}
|
|
333
|
+
// A malformed escape sequence is kept as typed; the reduction handles it.
|
|
168
334
|
}
|
|
169
|
-
|
|
335
|
+
const dot = decoded.lastIndexOf(".");
|
|
336
|
+
if (dot <= 0)
|
|
337
|
+
return "";
|
|
338
|
+
const extension = decoded.slice(dot + 1).replace(/[^A-Za-z0-9]/g, "");
|
|
339
|
+
return extension.length <= MAX_EXTENSION_LENGTH ? extension : "";
|
|
170
340
|
}
|
|
171
341
|
/** `path.extname` for a bare filename, without the `node:path` import: `.ext`, or `""` (a leading dot is not an extension). */
|
|
172
342
|
function extensionOf(name) {
|
|
173
343
|
const dot = name.lastIndexOf(".");
|
|
174
344
|
return dot > 0 ? name.slice(dot) : "";
|
|
175
345
|
}
|
|
346
|
+
function requireScope(scope) {
|
|
347
|
+
if (scope !== "main_stuff" && scope !== "working_memory") {
|
|
348
|
+
throw new ArtifactOperationError(`"scope" must be "main_stuff" or "working_memory", got ${String(scope)}.`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
176
351
|
// ── resolveArtifacts ─────────────────────────────────────────────────
|
|
177
352
|
/**
|
|
178
353
|
* Resolve a list of references through the bulk route, chunked at
|
|
@@ -237,7 +412,11 @@ function fetchBounds(options) {
|
|
|
237
412
|
const maxBytes = options.maxBytes ?? DEFAULT_ARTIFACT_MAX_BYTES;
|
|
238
413
|
const timeoutMs = options.timeoutMs ?? DEFAULT_ARTIFACT_TIMEOUT_MS;
|
|
239
414
|
requirePositive("maxBytes", maxBytes);
|
|
240
|
-
|
|
415
|
+
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
416
|
+
if (!isTimerDelay(timeoutMs)) {
|
|
417
|
+
throw new ArtifactOperationError(`"timeoutMs" must be a positive number no larger than ${MAX_TIMER_DELAY_MS}, got ` +
|
|
418
|
+
`${String(timeoutMs)}.`);
|
|
419
|
+
}
|
|
241
420
|
return { maxBytes, timeoutMs, allowHttp: options.allowHttp ?? false, signal: options.signal };
|
|
242
421
|
}
|
|
243
422
|
function requirePositive(name, value) {
|
|
@@ -545,8 +724,15 @@ class ArtifactWriteError extends Error {
|
|
|
545
724
|
const SKIPPED_ABORTED = "The download was aborted before this artifact was fetched.";
|
|
546
725
|
const SKIPPED_CREDENTIAL = "The download stopped on a credential failure before this artifact was fetched.";
|
|
547
726
|
const SKIPPED_TOTAL = "Skipped: the download's total byte limit was reached by an earlier artifact.";
|
|
548
|
-
function itemError(
|
|
549
|
-
return {
|
|
727
|
+
function itemError(location, contentType, code, detail) {
|
|
728
|
+
return {
|
|
729
|
+
uri: location.uri,
|
|
730
|
+
found_at: location.found_at,
|
|
731
|
+
path: null,
|
|
732
|
+
content_type: contentType,
|
|
733
|
+
size: null,
|
|
734
|
+
error: { code, detail },
|
|
735
|
+
};
|
|
550
736
|
}
|
|
551
737
|
function isExpired(expiresAt) {
|
|
552
738
|
const at = Date.parse(expiresAt);
|
|
@@ -571,19 +757,20 @@ async function readCompletedResults(client, runId, signal) {
|
|
|
571
757
|
* Save a run's produced files under `dir` — Node-only, like the path-string arm
|
|
572
758
|
* of `uploadFile`. Takes a `run_id` (the results are re-read, so a run is
|
|
573
759
|
* downloadable days later) or a `RunResults` in hand; walks the requested scope
|
|
574
|
-
* with {@link
|
|
760
|
+
* with {@link locateArtifacts}; resolves the whole set through the bulk route
|
|
575
761
|
* ahead of the workers; then a bounded pool of workers each fetch → open with
|
|
576
762
|
* `wx` → write, re-resolving any link that has expired by the time a worker
|
|
577
|
-
* reaches it. The embedded `public_url` is never used.
|
|
578
|
-
* {@link artifactFilename} and never overwritten;
|
|
579
|
-
* unlinks its partial file.
|
|
763
|
+
* reaches it. The embedded `public_url` is never used. Each file is named after
|
|
764
|
+
* the field it fills, by {@link artifactFilename}'s rule, and never overwritten;
|
|
765
|
+
* a failed or aborted download unlinks its partial file.
|
|
580
766
|
*
|
|
581
767
|
* Returns a produced verdict: one entry per reference, errors as values. It
|
|
582
768
|
* throws only when no verdict can be produced — `RunStillRunningError` or
|
|
583
769
|
* `RunFailedError` for a run that has not completed, `ScopeUnavailableError`
|
|
584
770
|
* when the scope's artifact is null or missing, `ArtifactAuthenticationError`
|
|
585
771
|
* (carrying the verdict so far) when the resolve route refuses the credential,
|
|
586
|
-
* `ArtifactOperationError` outside Node
|
|
772
|
+
* `ArtifactOperationError` outside Node, for an unusable `dir` or for an
|
|
773
|
+
* unknown `scope`, and the
|
|
587
774
|
* transport and lifecycle errors of the reads it makes (`ApiResponseError` for
|
|
588
775
|
* a deployment without the bulk route, `RunLifecycleUnavailableError` for a
|
|
589
776
|
* bare runner asked by id, `ApiUnreachableError`).
|
|
@@ -594,6 +781,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
594
781
|
"with resolveArtifacts or stream with fetchArtifact instead.");
|
|
595
782
|
}
|
|
596
783
|
const scope = request.scope ?? "main_stuff";
|
|
784
|
+
requireScope(scope);
|
|
597
785
|
const concurrency = request.concurrency ?? DEFAULT_DOWNLOAD_CONCURRENCY;
|
|
598
786
|
const maxTotalBytes = request.maxTotalBytes ?? DEFAULT_DOWNLOAD_MAX_TOTAL_BYTES;
|
|
599
787
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
@@ -619,10 +807,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
619
807
|
if (walked == null) {
|
|
620
808
|
throw new ScopeUnavailableError(scope, runId);
|
|
621
809
|
}
|
|
622
|
-
|
|
623
|
-
|
|
810
|
+
// The walk's own record names the files; `locations` is what the verdict reports.
|
|
811
|
+
const located = walkReferences(walked);
|
|
812
|
+
if (located.length === 0) {
|
|
624
813
|
return { scope, artifacts: [], saved_paths: [], all_saved: true };
|
|
625
814
|
}
|
|
815
|
+
const locations = located.map(renderLocation);
|
|
816
|
+
const uris = located.map((reference) => reference.uri);
|
|
626
817
|
const fs = (await import("node:fs/promises"));
|
|
627
818
|
const path = (await import("node:path"));
|
|
628
819
|
const dir = path.resolve(request.dir);
|
|
@@ -636,7 +827,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
636
827
|
// must name every reference, reached or not.
|
|
637
828
|
const outcomes = Array.from({ length: uris.length }, () => undefined);
|
|
638
829
|
const verdictSoFar = () => {
|
|
639
|
-
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(
|
|
830
|
+
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_CREDENTIAL));
|
|
640
831
|
return assembleVerdict(scope, artifacts, request.signal?.aborted === true);
|
|
641
832
|
};
|
|
642
833
|
let resolved;
|
|
@@ -673,13 +864,15 @@ export async function downloadArtifacts(client, request) {
|
|
|
673
864
|
let savedBytes = 0;
|
|
674
865
|
let limitReached = false;
|
|
675
866
|
let next = 0;
|
|
676
|
-
const saveOne = async (index,
|
|
867
|
+
const saveOne = async (index, url, contentType) => {
|
|
868
|
+
const location = locations[index];
|
|
869
|
+
const uri = location.uri;
|
|
677
870
|
let response;
|
|
678
871
|
try {
|
|
679
872
|
response = await fetchResolvedUrl(uri, url, workerBounds, dispatcher, () => undefined);
|
|
680
873
|
}
|
|
681
874
|
catch (err) {
|
|
682
|
-
return itemError(
|
|
875
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
683
876
|
}
|
|
684
877
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
685
878
|
const reserved = Number.isFinite(declaredLength) ? declaredLength : 0;
|
|
@@ -687,7 +880,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
687
880
|
if (savedBytes + reserved > maxTotalBytes)
|
|
688
881
|
limitReached = true;
|
|
689
882
|
await discard(response);
|
|
690
|
-
return itemError(
|
|
883
|
+
return itemError(location, contentType, "total_limit_exceeded", `Saving this ${formatMiB(reserved)} artifact would take the download past its ` +
|
|
691
884
|
`${formatMiB(maxTotalBytes)} total limit.`);
|
|
692
885
|
}
|
|
693
886
|
committedBytes += reserved;
|
|
@@ -695,12 +888,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
695
888
|
const share = () => Math.max(written, reserved);
|
|
696
889
|
let target;
|
|
697
890
|
try {
|
|
698
|
-
|
|
891
|
+
const filename = filenameFor(located[index].paths[0], uri, contentType, scope);
|
|
892
|
+
target = await openUniqueFile(fs, path, dir, filename);
|
|
699
893
|
}
|
|
700
894
|
catch (err) {
|
|
701
895
|
committedBytes -= share();
|
|
702
896
|
await discard(response);
|
|
703
|
-
return itemError(
|
|
897
|
+
return itemError(location, contentType, "write_failed", `The file could not be created: ${err instanceof Error ? err.message : String(err)}.`);
|
|
704
898
|
}
|
|
705
899
|
const removePartial = async () => {
|
|
706
900
|
committedBytes -= share();
|
|
@@ -713,10 +907,17 @@ export async function downloadArtifacts(client, request) {
|
|
|
713
907
|
}
|
|
714
908
|
catch (err) {
|
|
715
909
|
await removePartial();
|
|
716
|
-
return itemError(
|
|
910
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
717
911
|
}
|
|
718
912
|
committedBytes -= share();
|
|
719
|
-
return {
|
|
913
|
+
return {
|
|
914
|
+
uri,
|
|
915
|
+
found_at: location.found_at,
|
|
916
|
+
path: target.path,
|
|
917
|
+
content_type: contentType,
|
|
918
|
+
size: 0,
|
|
919
|
+
error: null,
|
|
920
|
+
};
|
|
720
921
|
}
|
|
721
922
|
const reader = response.body.getReader();
|
|
722
923
|
try {
|
|
@@ -731,7 +932,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
731
932
|
limitReached = true;
|
|
732
933
|
await reader.cancel().catch(() => undefined);
|
|
733
934
|
await removePartial();
|
|
734
|
-
return itemError(
|
|
935
|
+
return itemError(location, contentType, "total_limit_exceeded", `This artifact took the download past its ${formatMiB(maxTotalBytes)} total limit.`);
|
|
735
936
|
}
|
|
736
937
|
committedBytes += growth;
|
|
737
938
|
written += value.byteLength;
|
|
@@ -747,21 +948,28 @@ export async function downloadArtifacts(client, request) {
|
|
|
747
948
|
await reader.cancel().catch(() => undefined);
|
|
748
949
|
await removePartial();
|
|
749
950
|
if (err instanceof ArtifactWriteError) {
|
|
750
|
-
return itemError(
|
|
951
|
+
return itemError(location, contentType, "write_failed", `The file could not be written: ${err.message}.`);
|
|
751
952
|
}
|
|
752
|
-
return itemError(
|
|
953
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
753
954
|
}
|
|
754
955
|
try {
|
|
755
956
|
await target.handle.close();
|
|
756
957
|
}
|
|
757
958
|
catch (err) {
|
|
758
959
|
await removePartial();
|
|
759
|
-
return itemError(
|
|
960
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
760
961
|
}
|
|
761
962
|
// A body shorter than it declared gives the unused reservation back.
|
|
762
963
|
committedBytes -= share() - written;
|
|
763
964
|
savedBytes += written;
|
|
764
|
-
return {
|
|
965
|
+
return {
|
|
966
|
+
uri,
|
|
967
|
+
found_at: location.found_at,
|
|
968
|
+
path: target.path,
|
|
969
|
+
content_type: contentType,
|
|
970
|
+
size: written,
|
|
971
|
+
error: null,
|
|
972
|
+
};
|
|
765
973
|
};
|
|
766
974
|
const worker = async () => {
|
|
767
975
|
for (;;) {
|
|
@@ -769,40 +977,40 @@ export async function downloadArtifacts(client, request) {
|
|
|
769
977
|
next += 1;
|
|
770
978
|
if (index >= uris.length)
|
|
771
979
|
return;
|
|
772
|
-
const
|
|
980
|
+
const location = locations[index];
|
|
773
981
|
let entry = resolved[index];
|
|
774
982
|
if (credentialFailure !== undefined || aborted()) {
|
|
775
|
-
outcomes[index] = itemError(
|
|
983
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", credentialFailure ? SKIPPED_CREDENTIAL : SKIPPED_ABORTED);
|
|
776
984
|
continue;
|
|
777
985
|
}
|
|
778
986
|
if (limitReached) {
|
|
779
|
-
outcomes[index] = itemError(
|
|
987
|
+
outcomes[index] = itemError(location, entry.content_type, "total_limit_exceeded", SKIPPED_TOTAL);
|
|
780
988
|
continue;
|
|
781
989
|
}
|
|
782
990
|
if (entry.error === null && isExpired(entry.expires_at)) {
|
|
783
991
|
try {
|
|
784
|
-
const again = await resolveArtifacts(client, [uri], { signal });
|
|
992
|
+
const again = await resolveArtifacts(client, [location.uri], { signal });
|
|
785
993
|
entry = again[0];
|
|
786
994
|
}
|
|
787
995
|
catch (err) {
|
|
788
996
|
if (isCredentialRefusal(err)) {
|
|
789
997
|
credentialFailure = err;
|
|
790
|
-
outcomes[index] = itemError(
|
|
998
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_CREDENTIAL);
|
|
791
999
|
continue;
|
|
792
1000
|
}
|
|
793
1001
|
if (aborted()) {
|
|
794
|
-
outcomes[index] = itemError(
|
|
1002
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_ABORTED);
|
|
795
1003
|
continue;
|
|
796
1004
|
}
|
|
797
|
-
outcomes[index] = itemError(
|
|
1005
|
+
outcomes[index] = itemError(location, entry.content_type, "resolve_failed", `The expired link could not be re-resolved: ${err instanceof Error ? err.message : String(err)}.`);
|
|
798
1006
|
continue;
|
|
799
1007
|
}
|
|
800
1008
|
}
|
|
801
1009
|
if (entry.error !== null) {
|
|
802
|
-
outcomes[index] = itemError(
|
|
1010
|
+
outcomes[index] = itemError(location, null, entry.error.code, entry.error.detail);
|
|
803
1011
|
continue;
|
|
804
1012
|
}
|
|
805
|
-
outcomes[index] = await saveOne(index,
|
|
1013
|
+
outcomes[index] = await saveOne(index, entry.url, entry.content_type);
|
|
806
1014
|
}
|
|
807
1015
|
};
|
|
808
1016
|
try {
|
|
@@ -815,7 +1023,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
815
1023
|
throw new ArtifactAuthenticationError(`The resolve route refused the credential (${credentialFailure.status}) part-way through ` +
|
|
816
1024
|
"the download; the verdict so far is on this error.", credentialFailure.status, verdictSoFar(), { cause: credentialFailure });
|
|
817
1025
|
}
|
|
818
|
-
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(
|
|
1026
|
+
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_ABORTED)), request.signal?.aborted === true);
|
|
819
1027
|
}
|
|
820
1028
|
function assembleVerdict(scope, artifacts, aborted) {
|
|
821
1029
|
const saved_paths = artifacts.flatMap((artifact) => artifact.error === null ? [artifact.path] : []);
|