@pipelex/sdk 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/artifacts.d.ts +88 -26
- package/dist/artifacts.js +302 -99
- package/dist/artifacts.js.map +1 -1
- package/dist/client.d.ts +3 -2
- package/dist/client.js +3 -2
- package/dist/client.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +3 -3
- package/docs/artifact-download.md +49 -14
- package/docs/run-results.md +1 -1
- package/package.json +1 -1
package/dist/artifacts.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* The artifact stack — the download twin of `prepareInputs`, in layers so each
|
|
3
3
|
* operation is usable without the next:
|
|
4
4
|
*
|
|
5
|
-
* - {@link
|
|
6
|
-
* ARE `pipelex-storage://` references
|
|
5
|
+
* - {@link locateArtifacts} — a pure walk of any JSON value for the strings that
|
|
6
|
+
* ARE `pipelex-storage://` references, each with every path it sits at.
|
|
7
|
+
* {@link collectArtifacts} is the same walk's bare references. No network, no key.
|
|
7
8
|
* - {@link resolveArtifacts} — the platform's bulk resolve route over a whole
|
|
8
9
|
* list, chunked at the route's bound, one verdict per reference.
|
|
9
10
|
* - {@link fetchArtifact} — a bounded `Response` for one reference: resolved
|
|
@@ -20,7 +21,7 @@
|
|
|
20
21
|
*
|
|
21
22
|
* `node:fs/promises`, `node:path` and `undici` are imported dynamically, behind
|
|
22
23
|
* `isNodeRuntime()`, so the module's static import graph stays free of Node
|
|
23
|
-
* builtins:
|
|
24
|
+
* builtins: the walk and `resolveArtifacts` run anywhere, `fetchArtifact`
|
|
24
25
|
* runs anywhere the global `fetch` does (the dispatcher is a Node refinement), and
|
|
25
26
|
* only `downloadArtifacts` touches a filesystem.
|
|
26
27
|
*/
|
|
@@ -56,12 +57,24 @@ export const DEFAULT_DOWNLOAD_CONCURRENCY = 4;
|
|
|
56
57
|
const EXPIRY_MARGIN_MS = 10_000;
|
|
57
58
|
/** Longest filename `downloadArtifacts` writes, extension included. */
|
|
58
59
|
const MAX_FILENAME_LENGTH = 128;
|
|
60
|
+
/**
|
|
61
|
+
* Longest extension taken from a storage key, dot excluded. Anything longer after
|
|
62
|
+
* the key's last dot is read as part of a name rather than as an extension, and
|
|
63
|
+
* the content type's extension is used instead.
|
|
64
|
+
*/
|
|
65
|
+
const MAX_EXTENSION_LENGTH = 10;
|
|
66
|
+
/**
|
|
67
|
+
* Stems Windows reserves for a device, in any case and whatever the extension:
|
|
68
|
+
* `aux.png` there names the auxiliary device, not a file. A stem is a field name
|
|
69
|
+
* the method author chose, so `$.aux.url` would otherwise reach one.
|
|
70
|
+
*/
|
|
71
|
+
const WINDOWS_DEVICE_STEM = /^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
|
|
59
72
|
/** Ceiling on collision suffixes before the never-overwrite rule gives up. */
|
|
60
73
|
const MAX_UNIQUE_ATTEMPTS = 10_000;
|
|
61
74
|
/**
|
|
62
|
-
* The extension
|
|
63
|
-
* type is one of the artifact types a run produces. Deliberately
|
|
64
|
-
* unknown type simply gets no extension, never a guessed one.
|
|
75
|
+
* The extension a saved file takes when its storage key carries none and the
|
|
76
|
+
* resolved content type is one of the artifact types a run produces. Deliberately
|
|
77
|
+
* short: an unknown type simply gets no extension, never a guessed one.
|
|
65
78
|
*/
|
|
66
79
|
const EXTENSION_BY_CONTENT_TYPE = {
|
|
67
80
|
"image/png": ".png",
|
|
@@ -76,37 +89,138 @@ const EXTENSION_BY_CONTENT_TYPE = {
|
|
|
76
89
|
"text/csv": ".csv",
|
|
77
90
|
"application/json": ".json",
|
|
78
91
|
};
|
|
79
|
-
|
|
92
|
+
/** An object key rendered as `.key` in a path; any other key is rendered as `["…"]`. */
|
|
93
|
+
const IDENTIFIER_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
94
|
+
/**
|
|
95
|
+
* Every `pipelex-storage://` reference inside a JSON-shaped value, each with
|
|
96
|
+
* every path at which it occurs. The references are deduplicated and kept in
|
|
97
|
+
* discovery order (the order of their first sighting), and each one's `found_at`
|
|
98
|
+
* lists its paths in walk order, so `found_at[0]` is where it was first seen.
|
|
99
|
+
*
|
|
100
|
+
* The string test is {@link collectArtifacts}'s: a string counts only when it IS
|
|
101
|
+
* a reference. A path is rooted at `$`, the walked value itself; an object key
|
|
102
|
+
* matching `^[A-Za-z_][A-Za-z0-9_]*$` is written `.key`, any other key `["…"]` in
|
|
103
|
+
* JSON string escaping, and an array index `[n]`. The runtime serializes a
|
|
104
|
+
* produced image or document as content carrying its reference in `url`, so a
|
|
105
|
+
* typical path ends there: `$.rooms[3].staged_photo.url`, `$.items[0].url`, or
|
|
106
|
+
* `$.url` for an output that is one image. Pure — no network, no key.
|
|
107
|
+
*/
|
|
108
|
+
export function locateArtifacts(value) {
|
|
109
|
+
return walkReferences(value).map(renderLocation);
|
|
110
|
+
}
|
|
80
111
|
/**
|
|
81
112
|
* Every `pipelex-storage://` reference inside a JSON-shaped value, deduplicated,
|
|
82
|
-
* in discovery order
|
|
83
|
-
* string
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
113
|
+
* in discovery order — the references of {@link locateArtifacts}, without their
|
|
114
|
+
* paths. A string counts only when it IS a reference — the whole string, scheme
|
|
115
|
+
* first, with something after the scheme; text that merely contains one does
|
|
116
|
+
* not. The scheme is unambiguous, so this walk is a contract rather than a
|
|
117
|
+
* heuristic: the runtime serializes a produced image or document as content
|
|
118
|
+
* carrying its reference in `url`, beside an expiring `public_url` this walk
|
|
119
|
+
* ignores. Pure — no network, no key — so a consumer can count or list a run's
|
|
120
|
+
* produced files without resolving any of them.
|
|
89
121
|
*/
|
|
90
122
|
export function collectArtifacts(value) {
|
|
91
|
-
|
|
92
|
-
walkForReferences(value, found);
|
|
93
|
-
return [...found];
|
|
123
|
+
return walkReferences(value, false).map((located) => located.uri);
|
|
94
124
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
125
|
+
/**
|
|
126
|
+
* The walk both public functions share: depth first, keys in `Object.entries`
|
|
127
|
+
* order. With `withPaths` off it records each reference once and no path at
|
|
128
|
+
* all, which is what `collectArtifacts` needs: copying the trail for every
|
|
129
|
+
* occurrence costs memory in proportion to occurrences times depth, where the
|
|
130
|
+
* deduplicated list needs only one entry per unique reference.
|
|
131
|
+
*/
|
|
132
|
+
function walkReferences(value, withPaths = true) {
|
|
133
|
+
// A Map keeps insertion order, which is the order of first sighting.
|
|
134
|
+
const byUri = new Map();
|
|
135
|
+
const trail = [];
|
|
136
|
+
const visit = (node) => {
|
|
137
|
+
if (typeof node === "string") {
|
|
138
|
+
if (!isStorageReference(node))
|
|
139
|
+
return;
|
|
140
|
+
const known = byUri.get(node);
|
|
141
|
+
if (!withPaths) {
|
|
142
|
+
if (known === undefined)
|
|
143
|
+
byUri.set(node, { uri: node, paths: [] });
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const path = [...trail];
|
|
147
|
+
if (known === undefined)
|
|
148
|
+
byUri.set(node, { uri: node, paths: [path] });
|
|
149
|
+
else
|
|
150
|
+
known.paths.push(path);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (Array.isArray(node)) {
|
|
154
|
+
for (let index = 0; index < node.length; index += 1) {
|
|
155
|
+
trail.push(index);
|
|
156
|
+
visit(node[index]);
|
|
157
|
+
trail.pop();
|
|
158
|
+
}
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (typeof node === "object" && node !== null) {
|
|
162
|
+
for (const [key, entry] of Object.entries(node)) {
|
|
163
|
+
trail.push(key);
|
|
164
|
+
visit(entry);
|
|
165
|
+
trail.pop();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
visit(value);
|
|
170
|
+
return [...byUri.values()];
|
|
171
|
+
}
|
|
172
|
+
function renderLocation(located) {
|
|
173
|
+
return { uri: located.uri, found_at: located.paths.map(renderPath) };
|
|
174
|
+
}
|
|
175
|
+
/** Segments to the `$`-rooted notation `found_at` carries. */
|
|
176
|
+
function renderPath(segments) {
|
|
177
|
+
let path = "$";
|
|
178
|
+
for (const segment of segments) {
|
|
179
|
+
if (typeof segment === "number")
|
|
180
|
+
path += `[${segment}]`;
|
|
181
|
+
else if (IDENTIFIER_KEY.test(segment))
|
|
182
|
+
path += `.${segment}`;
|
|
183
|
+
else
|
|
184
|
+
path += `[${JSON.stringify(segment)}]`;
|
|
105
185
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
186
|
+
return path;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* The `$`-rooted notation back to segments, for a location that reaches
|
|
190
|
+
* {@link artifactFilename} from outside the walk. The rendering is lossless, so
|
|
191
|
+
* this is exact for every path the walk produced; anything else is `undefined`.
|
|
192
|
+
*/
|
|
193
|
+
function parsePath(path) {
|
|
194
|
+
if (!path.startsWith("$"))
|
|
195
|
+
return undefined;
|
|
196
|
+
const segments = [];
|
|
197
|
+
let at = 1;
|
|
198
|
+
while (at < path.length) {
|
|
199
|
+
const rest = path.slice(at);
|
|
200
|
+
const key = /^\.([A-Za-z_][A-Za-z0-9_]*)/.exec(rest);
|
|
201
|
+
if (key !== null) {
|
|
202
|
+
segments.push(key[1]);
|
|
203
|
+
at += key[0].length;
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const index = /^\[(\d+)\]/.exec(rest);
|
|
207
|
+
if (index !== null) {
|
|
208
|
+
segments.push(Number(index[1]));
|
|
209
|
+
at += index[0].length;
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
const quoted = /^\[("(?:[^"\\]|\\.)*")\]/s.exec(rest);
|
|
213
|
+
if (quoted === null)
|
|
214
|
+
return undefined;
|
|
215
|
+
try {
|
|
216
|
+
segments.push(JSON.parse(quoted[1]));
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
at += quoted[0].length;
|
|
109
222
|
}
|
|
223
|
+
return segments;
|
|
110
224
|
}
|
|
111
225
|
/** A string that is a storage reference: the scheme, then at least one character. */
|
|
112
226
|
export function isStorageReference(value) {
|
|
@@ -114,18 +228,95 @@ export function isStorageReference(value) {
|
|
|
114
228
|
}
|
|
115
229
|
// ── artifactFilename ─────────────────────────────────────────────────
|
|
116
230
|
/**
|
|
117
|
-
* The bare filename a
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
231
|
+
* The bare filename a reference is saved under, named after the field it fills:
|
|
232
|
+
* the path in `location.found_at[0]`, where the reference was first seen.
|
|
233
|
+
*
|
|
234
|
+
* 1. A final object key `url` is dropped, since the runtime's image and
|
|
235
|
+
* document contents carry their reference there; a reference under any other
|
|
236
|
+
* key keeps that key, and a `url` key that is not final is kept.
|
|
237
|
+
* 2. Each key is reduced to `[A-Za-z0-9_]`, every other character (`-` and `.`
|
|
238
|
+
* included) becoming `_`; an index stays its decimal digits.
|
|
239
|
+
* 3. The segments are joined with `-`. An empty result — the reference is the
|
|
240
|
+
* walked value itself, or its `url` — becomes the scope's name.
|
|
241
|
+
* 4. Over the filename length cap (`MAX_FILENAME_LENGTH`, extension included),
|
|
242
|
+
* the tail is kept: whole leading segments are dropped first, since the last
|
|
243
|
+
* ones are the specific ones, and a single segment still too long is cut to
|
|
244
|
+
* fit.
|
|
245
|
+
* 5. A stem Windows reserves for a device (`con`, `prn`, `aux`, `nul`,
|
|
246
|
+
* `com0`–`com9`, `lpt0`–`lpt9`, in any case) gets a trailing `_`, so
|
|
247
|
+
* `$.aux.url` is saved as `aux_.png`.
|
|
248
|
+
* 6. The extension is the storage key's own, reduced to `[A-Za-z0-9]`, when it
|
|
249
|
+
* has a short one; otherwise the content type's, for the types a run
|
|
250
|
+
* produces; otherwise there is none.
|
|
251
|
+
*
|
|
252
|
+
* So `$.rooms[3].staged_photo.url` is saved as `rooms-3-staged_photo.png`, and
|
|
253
|
+
* `$.url` in `main_stuff` as `main_stuff.png`. The stem is ASCII letters, digits,
|
|
254
|
+
* `_` and the `-` joins, never empty and never a device name, so the name can
|
|
255
|
+
* only ever be a regular file directly inside the target directory. A collision on disk is not this
|
|
125
256
|
* function's concern: `downloadArtifacts` suffixes the stem (`name-1.ext`) on
|
|
126
257
|
* exclusive creation, so a file is never overwritten.
|
|
258
|
+
*
|
|
259
|
+
* Throws `ArtifactOperationError` for a location whose `found_at[0]` is not a path
|
|
260
|
+
* in the notation {@link locateArtifacts} writes, or for an unknown scope.
|
|
127
261
|
*/
|
|
128
|
-
export function artifactFilename(
|
|
262
|
+
export function artifactFilename(location, contentType, scope) {
|
|
263
|
+
requireScope(scope);
|
|
264
|
+
// A JavaScript caller can hand anything here — the old signature's bare uri
|
|
265
|
+
// string among them — and every shape must reach the documented refusal
|
|
266
|
+
// rather than a TypeError, or a string's first character.
|
|
267
|
+
const loose = location;
|
|
268
|
+
const uri = loose?.uri;
|
|
269
|
+
if (typeof uri !== "string") {
|
|
270
|
+
throw new ArtifactOperationError(`artifactFilename needs a location carrying its reference as a string "uri"; got ${String(uri)}.`);
|
|
271
|
+
}
|
|
272
|
+
const foundAt = loose?.found_at;
|
|
273
|
+
const first = Array.isArray(foundAt) ? foundAt[0] : undefined;
|
|
274
|
+
const segments = typeof first === "string" ? parsePath(first) : undefined;
|
|
275
|
+
if (segments === undefined) {
|
|
276
|
+
throw new ArtifactOperationError(`artifactFilename needs a location whose first "found_at" entry is a path such as ` +
|
|
277
|
+
`"$.items[0].url"; got ${typeof first === "string" ? JSON.stringify(first) : String(first)}.`);
|
|
278
|
+
}
|
|
279
|
+
return filenameFor(segments, uri, contentType, scope);
|
|
280
|
+
}
|
|
281
|
+
/** The naming rule of {@link artifactFilename}, over the walk's own segments. */
|
|
282
|
+
function filenameFor(segments, uri, contentType, scope) {
|
|
283
|
+
const named = segments.at(-1) === "url" ? segments.slice(0, -1) : segments;
|
|
284
|
+
const words = named
|
|
285
|
+
.map((segment) => typeof segment === "number" ? String(segment) : segment.replace(/[^A-Za-z0-9_]/gu, "_"))
|
|
286
|
+
// Only the empty key reduces to nothing, and it says nothing about the field.
|
|
287
|
+
.filter((word) => word !== "");
|
|
288
|
+
const extension = extensionFor(uri, contentType);
|
|
289
|
+
const stem = fitStem(words.length > 0 ? words : [scope], MAX_FILENAME_LENGTH - extension.length);
|
|
290
|
+
// A device stem is at most four characters, so the `_` cannot overrun the cap.
|
|
291
|
+
return (WINDOWS_DEVICE_STEM.test(stem) ? `${stem}_` : stem) + extension;
|
|
292
|
+
}
|
|
293
|
+
/** The words joined with `-` within `budget` characters, keeping the tail. */
|
|
294
|
+
function fitStem(words, budget) {
|
|
295
|
+
let start = 0;
|
|
296
|
+
let length = words.reduce((sum, word) => sum + word.length, 0) + words.length - 1;
|
|
297
|
+
while (length > budget && start < words.length - 1) {
|
|
298
|
+
length -= words[start].length + 1;
|
|
299
|
+
start += 1;
|
|
300
|
+
}
|
|
301
|
+
const stem = words.slice(start).join("-");
|
|
302
|
+
return stem.length > budget ? stem.slice(0, budget) : stem;
|
|
303
|
+
}
|
|
304
|
+
/** `.ext` for the saved file — the storage key's own, else the content type's — or `""`. */
|
|
305
|
+
function extensionFor(uri, contentType) {
|
|
306
|
+
const fromKey = storageKeyExtension(uri);
|
|
307
|
+
if (fromKey !== "")
|
|
308
|
+
return `.${fromKey}`;
|
|
309
|
+
if (contentType == null)
|
|
310
|
+
return "";
|
|
311
|
+
return EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()] ?? "";
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* The extension the storage key's last segment carries, without its dot, reduced
|
|
315
|
+
* to `[A-Za-z0-9]` — or `""` when it has none, or none that short. The segment is
|
|
316
|
+
* what follows the last `/` or `\` once the scheme, query and fragment are gone,
|
|
317
|
+
* percent-decoded when it decodes; a leading dot is not an extension.
|
|
318
|
+
*/
|
|
319
|
+
function storageKeyExtension(uri) {
|
|
129
320
|
const key = uri.startsWith(PIPELEX_STORAGE_SCHEME)
|
|
130
321
|
? uri.slice(PIPELEX_STORAGE_SCHEME.length)
|
|
131
322
|
: uri;
|
|
@@ -138,41 +329,24 @@ export function artifactFilename(uri, contentType, index) {
|
|
|
138
329
|
decoded = decodeURIComponent(segment);
|
|
139
330
|
}
|
|
140
331
|
catch {
|
|
141
|
-
// A malformed escape sequence is kept as typed;
|
|
142
|
-
}
|
|
143
|
-
let name = decoded
|
|
144
|
-
.replace(/[^A-Za-z0-9._-]/g, "_")
|
|
145
|
-
.replace(/^[._-]+/, "")
|
|
146
|
-
.replace(/[._-]+$/, "");
|
|
147
|
-
if (name === "") {
|
|
148
|
-
name = `artifact-${index + 1}`;
|
|
149
|
-
}
|
|
150
|
-
if (name.length > MAX_FILENAME_LENGTH) {
|
|
151
|
-
const ext = extensionOf(name);
|
|
152
|
-
// The extension is kept only if there is room left for a stem. An extension
|
|
153
|
-
// at least as long as the cap would give `slice` a negative start, which
|
|
154
|
-
// counts from the END and yields a name LONGER than the cap — so a
|
|
155
|
-
// pathological extension is dropped rather than preserved.
|
|
156
|
-
name =
|
|
157
|
-
ext.length < MAX_FILENAME_LENGTH
|
|
158
|
-
? name.slice(0, MAX_FILENAME_LENGTH - ext.length) + ext
|
|
159
|
-
: name.slice(0, MAX_FILENAME_LENGTH);
|
|
160
|
-
}
|
|
161
|
-
if (extensionOf(name) === "") {
|
|
162
|
-
const ext = contentType == null
|
|
163
|
-
? undefined
|
|
164
|
-
: EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()];
|
|
165
|
-
if (ext !== undefined) {
|
|
166
|
-
name += ext;
|
|
167
|
-
}
|
|
332
|
+
// A malformed escape sequence is kept as typed; the reduction handles it.
|
|
168
333
|
}
|
|
169
|
-
|
|
334
|
+
const dot = decoded.lastIndexOf(".");
|
|
335
|
+
if (dot <= 0)
|
|
336
|
+
return "";
|
|
337
|
+
const extension = decoded.slice(dot + 1).replace(/[^A-Za-z0-9]/g, "");
|
|
338
|
+
return extension.length <= MAX_EXTENSION_LENGTH ? extension : "";
|
|
170
339
|
}
|
|
171
340
|
/** `path.extname` for a bare filename, without the `node:path` import: `.ext`, or `""` (a leading dot is not an extension). */
|
|
172
341
|
function extensionOf(name) {
|
|
173
342
|
const dot = name.lastIndexOf(".");
|
|
174
343
|
return dot > 0 ? name.slice(dot) : "";
|
|
175
344
|
}
|
|
345
|
+
function requireScope(scope) {
|
|
346
|
+
if (scope !== "main_stuff" && scope !== "working_memory") {
|
|
347
|
+
throw new ArtifactOperationError(`"scope" must be "main_stuff" or "working_memory", got ${String(scope)}.`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
176
350
|
// ── resolveArtifacts ─────────────────────────────────────────────────
|
|
177
351
|
/**
|
|
178
352
|
* Resolve a list of references through the bulk route, chunked at
|
|
@@ -545,8 +719,15 @@ class ArtifactWriteError extends Error {
|
|
|
545
719
|
const SKIPPED_ABORTED = "The download was aborted before this artifact was fetched.";
|
|
546
720
|
const SKIPPED_CREDENTIAL = "The download stopped on a credential failure before this artifact was fetched.";
|
|
547
721
|
const SKIPPED_TOTAL = "Skipped: the download's total byte limit was reached by an earlier artifact.";
|
|
548
|
-
function itemError(
|
|
549
|
-
return {
|
|
722
|
+
function itemError(location, contentType, code, detail) {
|
|
723
|
+
return {
|
|
724
|
+
uri: location.uri,
|
|
725
|
+
found_at: location.found_at,
|
|
726
|
+
path: null,
|
|
727
|
+
content_type: contentType,
|
|
728
|
+
size: null,
|
|
729
|
+
error: { code, detail },
|
|
730
|
+
};
|
|
550
731
|
}
|
|
551
732
|
function isExpired(expiresAt) {
|
|
552
733
|
const at = Date.parse(expiresAt);
|
|
@@ -571,19 +752,20 @@ async function readCompletedResults(client, runId, signal) {
|
|
|
571
752
|
* Save a run's produced files under `dir` — Node-only, like the path-string arm
|
|
572
753
|
* of `uploadFile`. Takes a `run_id` (the results are re-read, so a run is
|
|
573
754
|
* downloadable days later) or a `RunResults` in hand; walks the requested scope
|
|
574
|
-
* with {@link
|
|
755
|
+
* with {@link locateArtifacts}; resolves the whole set through the bulk route
|
|
575
756
|
* ahead of the workers; then a bounded pool of workers each fetch → open with
|
|
576
757
|
* `wx` → write, re-resolving any link that has expired by the time a worker
|
|
577
|
-
* reaches it. The embedded `public_url` is never used.
|
|
578
|
-
* {@link artifactFilename} and never overwritten;
|
|
579
|
-
* unlinks its partial file.
|
|
758
|
+
* reaches it. The embedded `public_url` is never used. Each file is named after
|
|
759
|
+
* the field it fills, by {@link artifactFilename}'s rule, and never overwritten;
|
|
760
|
+
* a failed or aborted download unlinks its partial file.
|
|
580
761
|
*
|
|
581
762
|
* Returns a produced verdict: one entry per reference, errors as values. It
|
|
582
763
|
* throws only when no verdict can be produced — `RunStillRunningError` or
|
|
583
764
|
* `RunFailedError` for a run that has not completed, `ScopeUnavailableError`
|
|
584
765
|
* when the scope's artifact is null or missing, `ArtifactAuthenticationError`
|
|
585
766
|
* (carrying the verdict so far) when the resolve route refuses the credential,
|
|
586
|
-
* `ArtifactOperationError` outside Node
|
|
767
|
+
* `ArtifactOperationError` outside Node, for an unusable `dir` or for an
|
|
768
|
+
* unknown `scope`, and the
|
|
587
769
|
* transport and lifecycle errors of the reads it makes (`ApiResponseError` for
|
|
588
770
|
* a deployment without the bulk route, `RunLifecycleUnavailableError` for a
|
|
589
771
|
* bare runner asked by id, `ApiUnreachableError`).
|
|
@@ -594,6 +776,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
594
776
|
"with resolveArtifacts or stream with fetchArtifact instead.");
|
|
595
777
|
}
|
|
596
778
|
const scope = request.scope ?? "main_stuff";
|
|
779
|
+
requireScope(scope);
|
|
597
780
|
const concurrency = request.concurrency ?? DEFAULT_DOWNLOAD_CONCURRENCY;
|
|
598
781
|
const maxTotalBytes = request.maxTotalBytes ?? DEFAULT_DOWNLOAD_MAX_TOTAL_BYTES;
|
|
599
782
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
@@ -619,10 +802,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
619
802
|
if (walked == null) {
|
|
620
803
|
throw new ScopeUnavailableError(scope, runId);
|
|
621
804
|
}
|
|
622
|
-
|
|
623
|
-
|
|
805
|
+
// The walk's own record names the files; `locations` is what the verdict reports.
|
|
806
|
+
const located = walkReferences(walked);
|
|
807
|
+
if (located.length === 0) {
|
|
624
808
|
return { scope, artifacts: [], saved_paths: [], all_saved: true };
|
|
625
809
|
}
|
|
810
|
+
const locations = located.map(renderLocation);
|
|
811
|
+
const uris = located.map((reference) => reference.uri);
|
|
626
812
|
const fs = (await import("node:fs/promises"));
|
|
627
813
|
const path = (await import("node:path"));
|
|
628
814
|
const dir = path.resolve(request.dir);
|
|
@@ -636,7 +822,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
636
822
|
// must name every reference, reached or not.
|
|
637
823
|
const outcomes = Array.from({ length: uris.length }, () => undefined);
|
|
638
824
|
const verdictSoFar = () => {
|
|
639
|
-
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(
|
|
825
|
+
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_CREDENTIAL));
|
|
640
826
|
return assembleVerdict(scope, artifacts, request.signal?.aborted === true);
|
|
641
827
|
};
|
|
642
828
|
let resolved;
|
|
@@ -673,13 +859,15 @@ export async function downloadArtifacts(client, request) {
|
|
|
673
859
|
let savedBytes = 0;
|
|
674
860
|
let limitReached = false;
|
|
675
861
|
let next = 0;
|
|
676
|
-
const saveOne = async (index,
|
|
862
|
+
const saveOne = async (index, url, contentType) => {
|
|
863
|
+
const location = locations[index];
|
|
864
|
+
const uri = location.uri;
|
|
677
865
|
let response;
|
|
678
866
|
try {
|
|
679
867
|
response = await fetchResolvedUrl(uri, url, workerBounds, dispatcher, () => undefined);
|
|
680
868
|
}
|
|
681
869
|
catch (err) {
|
|
682
|
-
return itemError(
|
|
870
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
683
871
|
}
|
|
684
872
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
685
873
|
const reserved = Number.isFinite(declaredLength) ? declaredLength : 0;
|
|
@@ -687,7 +875,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
687
875
|
if (savedBytes + reserved > maxTotalBytes)
|
|
688
876
|
limitReached = true;
|
|
689
877
|
await discard(response);
|
|
690
|
-
return itemError(
|
|
878
|
+
return itemError(location, contentType, "total_limit_exceeded", `Saving this ${formatMiB(reserved)} artifact would take the download past its ` +
|
|
691
879
|
`${formatMiB(maxTotalBytes)} total limit.`);
|
|
692
880
|
}
|
|
693
881
|
committedBytes += reserved;
|
|
@@ -695,12 +883,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
695
883
|
const share = () => Math.max(written, reserved);
|
|
696
884
|
let target;
|
|
697
885
|
try {
|
|
698
|
-
|
|
886
|
+
const filename = filenameFor(located[index].paths[0], uri, contentType, scope);
|
|
887
|
+
target = await openUniqueFile(fs, path, dir, filename);
|
|
699
888
|
}
|
|
700
889
|
catch (err) {
|
|
701
890
|
committedBytes -= share();
|
|
702
891
|
await discard(response);
|
|
703
|
-
return itemError(
|
|
892
|
+
return itemError(location, contentType, "write_failed", `The file could not be created: ${err instanceof Error ? err.message : String(err)}.`);
|
|
704
893
|
}
|
|
705
894
|
const removePartial = async () => {
|
|
706
895
|
committedBytes -= share();
|
|
@@ -713,10 +902,17 @@ export async function downloadArtifacts(client, request) {
|
|
|
713
902
|
}
|
|
714
903
|
catch (err) {
|
|
715
904
|
await removePartial();
|
|
716
|
-
return itemError(
|
|
905
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
717
906
|
}
|
|
718
907
|
committedBytes -= share();
|
|
719
|
-
return {
|
|
908
|
+
return {
|
|
909
|
+
uri,
|
|
910
|
+
found_at: location.found_at,
|
|
911
|
+
path: target.path,
|
|
912
|
+
content_type: contentType,
|
|
913
|
+
size: 0,
|
|
914
|
+
error: null,
|
|
915
|
+
};
|
|
720
916
|
}
|
|
721
917
|
const reader = response.body.getReader();
|
|
722
918
|
try {
|
|
@@ -731,7 +927,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
731
927
|
limitReached = true;
|
|
732
928
|
await reader.cancel().catch(() => undefined);
|
|
733
929
|
await removePartial();
|
|
734
|
-
return itemError(
|
|
930
|
+
return itemError(location, contentType, "total_limit_exceeded", `This artifact took the download past its ${formatMiB(maxTotalBytes)} total limit.`);
|
|
735
931
|
}
|
|
736
932
|
committedBytes += growth;
|
|
737
933
|
written += value.byteLength;
|
|
@@ -747,21 +943,28 @@ export async function downloadArtifacts(client, request) {
|
|
|
747
943
|
await reader.cancel().catch(() => undefined);
|
|
748
944
|
await removePartial();
|
|
749
945
|
if (err instanceof ArtifactWriteError) {
|
|
750
|
-
return itemError(
|
|
946
|
+
return itemError(location, contentType, "write_failed", `The file could not be written: ${err.message}.`);
|
|
751
947
|
}
|
|
752
|
-
return itemError(
|
|
948
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
753
949
|
}
|
|
754
950
|
try {
|
|
755
951
|
await target.handle.close();
|
|
756
952
|
}
|
|
757
953
|
catch (err) {
|
|
758
954
|
await removePartial();
|
|
759
|
-
return itemError(
|
|
955
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
760
956
|
}
|
|
761
957
|
// A body shorter than it declared gives the unused reservation back.
|
|
762
958
|
committedBytes -= share() - written;
|
|
763
959
|
savedBytes += written;
|
|
764
|
-
return {
|
|
960
|
+
return {
|
|
961
|
+
uri,
|
|
962
|
+
found_at: location.found_at,
|
|
963
|
+
path: target.path,
|
|
964
|
+
content_type: contentType,
|
|
965
|
+
size: written,
|
|
966
|
+
error: null,
|
|
967
|
+
};
|
|
765
968
|
};
|
|
766
969
|
const worker = async () => {
|
|
767
970
|
for (;;) {
|
|
@@ -769,40 +972,40 @@ export async function downloadArtifacts(client, request) {
|
|
|
769
972
|
next += 1;
|
|
770
973
|
if (index >= uris.length)
|
|
771
974
|
return;
|
|
772
|
-
const
|
|
975
|
+
const location = locations[index];
|
|
773
976
|
let entry = resolved[index];
|
|
774
977
|
if (credentialFailure !== undefined || aborted()) {
|
|
775
|
-
outcomes[index] = itemError(
|
|
978
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", credentialFailure ? SKIPPED_CREDENTIAL : SKIPPED_ABORTED);
|
|
776
979
|
continue;
|
|
777
980
|
}
|
|
778
981
|
if (limitReached) {
|
|
779
|
-
outcomes[index] = itemError(
|
|
982
|
+
outcomes[index] = itemError(location, entry.content_type, "total_limit_exceeded", SKIPPED_TOTAL);
|
|
780
983
|
continue;
|
|
781
984
|
}
|
|
782
985
|
if (entry.error === null && isExpired(entry.expires_at)) {
|
|
783
986
|
try {
|
|
784
|
-
const again = await resolveArtifacts(client, [uri], { signal });
|
|
987
|
+
const again = await resolveArtifacts(client, [location.uri], { signal });
|
|
785
988
|
entry = again[0];
|
|
786
989
|
}
|
|
787
990
|
catch (err) {
|
|
788
991
|
if (isCredentialRefusal(err)) {
|
|
789
992
|
credentialFailure = err;
|
|
790
|
-
outcomes[index] = itemError(
|
|
993
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_CREDENTIAL);
|
|
791
994
|
continue;
|
|
792
995
|
}
|
|
793
996
|
if (aborted()) {
|
|
794
|
-
outcomes[index] = itemError(
|
|
997
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_ABORTED);
|
|
795
998
|
continue;
|
|
796
999
|
}
|
|
797
|
-
outcomes[index] = itemError(
|
|
1000
|
+
outcomes[index] = itemError(location, entry.content_type, "resolve_failed", `The expired link could not be re-resolved: ${err instanceof Error ? err.message : String(err)}.`);
|
|
798
1001
|
continue;
|
|
799
1002
|
}
|
|
800
1003
|
}
|
|
801
1004
|
if (entry.error !== null) {
|
|
802
|
-
outcomes[index] = itemError(
|
|
1005
|
+
outcomes[index] = itemError(location, null, entry.error.code, entry.error.detail);
|
|
803
1006
|
continue;
|
|
804
1007
|
}
|
|
805
|
-
outcomes[index] = await saveOne(index,
|
|
1008
|
+
outcomes[index] = await saveOne(index, entry.url, entry.content_type);
|
|
806
1009
|
}
|
|
807
1010
|
};
|
|
808
1011
|
try {
|
|
@@ -815,7 +1018,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
815
1018
|
throw new ArtifactAuthenticationError(`The resolve route refused the credential (${credentialFailure.status}) part-way through ` +
|
|
816
1019
|
"the download; the verdict so far is on this error.", credentialFailure.status, verdictSoFar(), { cause: credentialFailure });
|
|
817
1020
|
}
|
|
818
|
-
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(
|
|
1021
|
+
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_ABORTED)), request.signal?.aborted === true);
|
|
819
1022
|
}
|
|
820
1023
|
function assembleVerdict(scope, artifacts, aborted) {
|
|
821
1024
|
const saved_paths = artifacts.flatMap((artifact) => artifact.error === null ? [artifact.path] : []);
|