@pipelex/sdk 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/dist/artifacts.d.ts +88 -26
- package/dist/artifacts.js +305 -100
- package/dist/artifacts.js.map +1 -1
- package/dist/client.d.ts +23 -3
- package/dist/client.js +25 -4
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +25 -7
- package/dist/errors.js +15 -7
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +7 -4
- package/dist/index.js.map +1 -1
- package/dist/product-models.d.ts +42 -1
- package/dist/product-models.js +1 -1
- package/dist/upload-grant.d.ts +60 -0
- package/dist/upload-grant.js +227 -0
- package/dist/upload-grant.js.map +1 -0
- package/dist/upload.d.ts +4 -3
- package/dist/upload.js +6 -5
- package/dist/upload.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +7 -4
- package/docs/artifact-download.md +49 -14
- package/docs/input-preparation.md +54 -5
- package/docs/run-results.md +1 -1
- package/package.json +6 -1
package/dist/artifacts.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* The artifact stack — the download twin of `prepareInputs`, in layers so each
|
|
3
3
|
* operation is usable without the next:
|
|
4
4
|
*
|
|
5
|
-
* - {@link
|
|
6
|
-
* ARE `pipelex-storage://` references
|
|
5
|
+
* - {@link locateArtifacts} — a pure walk of any JSON value for the strings that
|
|
6
|
+
* ARE `pipelex-storage://` references, each with every path it sits at.
|
|
7
|
+
* {@link collectArtifacts} is the same walk's bare references. No network, no key.
|
|
7
8
|
* - {@link resolveArtifacts} — the platform's bulk resolve route over a whole
|
|
8
9
|
* list, chunked at the route's bound, one verdict per reference.
|
|
9
10
|
* - {@link fetchArtifact} — a bounded `Response` for one reference: resolved
|
|
@@ -20,7 +21,7 @@
|
|
|
20
21
|
*
|
|
21
22
|
* `node:fs/promises`, `node:path` and `undici` are imported dynamically, behind
|
|
22
23
|
* `isNodeRuntime()`, so the module's static import graph stays free of Node
|
|
23
|
-
* builtins:
|
|
24
|
+
* builtins: the walk and `resolveArtifacts` run anywhere, `fetchArtifact`
|
|
24
25
|
* runs anywhere the global `fetch` does (the dispatcher is a Node refinement), and
|
|
25
26
|
* only `downloadArtifacts` touches a filesystem.
|
|
26
27
|
*/
|
|
@@ -56,12 +57,24 @@ export const DEFAULT_DOWNLOAD_CONCURRENCY = 4;
|
|
|
56
57
|
const EXPIRY_MARGIN_MS = 10_000;
|
|
57
58
|
/** Longest filename `downloadArtifacts` writes, extension included. */
|
|
58
59
|
const MAX_FILENAME_LENGTH = 128;
|
|
60
|
+
/**
|
|
61
|
+
* Longest extension taken from a storage key, dot excluded. Anything longer after
|
|
62
|
+
* the key's last dot is read as part of a name rather than as an extension, and
|
|
63
|
+
* the content type's extension is used instead.
|
|
64
|
+
*/
|
|
65
|
+
const MAX_EXTENSION_LENGTH = 10;
|
|
66
|
+
/**
|
|
67
|
+
* Stems Windows reserves for a device, in any case and whatever the extension:
|
|
68
|
+
* `aux.png` there names the auxiliary device, not a file. A stem is a field name
|
|
69
|
+
* the method author chose, so `$.aux.url` would otherwise reach one.
|
|
70
|
+
*/
|
|
71
|
+
const WINDOWS_DEVICE_STEM = /^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
|
|
59
72
|
/** Ceiling on collision suffixes before the never-overwrite rule gives up. */
|
|
60
73
|
const MAX_UNIQUE_ATTEMPTS = 10_000;
|
|
61
74
|
/**
|
|
62
|
-
* The extension
|
|
63
|
-
* type is one of the artifact types a run produces. Deliberately
|
|
64
|
-
* unknown type simply gets no extension, never a guessed one.
|
|
75
|
+
* The extension a saved file takes when its storage key carries none and the
|
|
76
|
+
* resolved content type is one of the artifact types a run produces. Deliberately
|
|
77
|
+
* short: an unknown type simply gets no extension, never a guessed one.
|
|
65
78
|
*/
|
|
66
79
|
const EXTENSION_BY_CONTENT_TYPE = {
|
|
67
80
|
"image/png": ".png",
|
|
@@ -76,37 +89,138 @@ const EXTENSION_BY_CONTENT_TYPE = {
|
|
|
76
89
|
"text/csv": ".csv",
|
|
77
90
|
"application/json": ".json",
|
|
78
91
|
};
|
|
79
|
-
|
|
92
|
+
/** An object key rendered as `.key` in a path; any other key is rendered as `["…"]`. */
|
|
93
|
+
const IDENTIFIER_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
94
|
+
/**
|
|
95
|
+
* Every `pipelex-storage://` reference inside a JSON-shaped value, each with
|
|
96
|
+
* every path at which it occurs. The references are deduplicated and kept in
|
|
97
|
+
* discovery order (the order of their first sighting), and each one's `found_at`
|
|
98
|
+
* lists its paths in walk order, so `found_at[0]` is where it was first seen.
|
|
99
|
+
*
|
|
100
|
+
* The string test is {@link collectArtifacts}'s: a string counts only when it IS
|
|
101
|
+
* a reference. A path is rooted at `$`, the walked value itself; an object key
|
|
102
|
+
* matching `^[A-Za-z_][A-Za-z0-9_]*$` is written `.key`, any other key `["…"]` in
|
|
103
|
+
* JSON string escaping, and an array index `[n]`. The runtime serializes a
|
|
104
|
+
* produced image or document as content carrying its reference in `url`, so a
|
|
105
|
+
* typical path ends there: `$.rooms[3].staged_photo.url`, `$.items[0].url`, or
|
|
106
|
+
* `$.url` for an output that is one image. Pure — no network, no key.
|
|
107
|
+
*/
|
|
108
|
+
export function locateArtifacts(value) {
|
|
109
|
+
return walkReferences(value).map(renderLocation);
|
|
110
|
+
}
|
|
80
111
|
/**
|
|
81
112
|
* Every `pipelex-storage://` reference inside a JSON-shaped value, deduplicated,
|
|
82
|
-
* in discovery order
|
|
83
|
-
* string
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
113
|
+
* in discovery order — the references of {@link locateArtifacts}, without their
|
|
114
|
+
* paths. A string counts only when it IS a reference — the whole string, scheme
|
|
115
|
+
* first, with something after the scheme; text that merely contains one does
|
|
116
|
+
* not. The scheme is unambiguous, so this walk is a contract rather than a
|
|
117
|
+
* heuristic: the runtime serializes a produced image or document as content
|
|
118
|
+
* carrying its reference in `url`, beside an expiring `public_url` this walk
|
|
119
|
+
* ignores. Pure — no network, no key — so a consumer can count or list a run's
|
|
120
|
+
* produced files without resolving any of them.
|
|
89
121
|
*/
|
|
90
122
|
export function collectArtifacts(value) {
|
|
91
|
-
|
|
92
|
-
walkForReferences(value, found);
|
|
93
|
-
return [...found];
|
|
123
|
+
return walkReferences(value, false).map((located) => located.uri);
|
|
94
124
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
125
|
+
/**
|
|
126
|
+
* The walk both public functions share: depth first, keys in `Object.entries`
|
|
127
|
+
* order. With `withPaths` off it records each reference once and no path at
|
|
128
|
+
* all, which is what `collectArtifacts` needs: copying the trail for every
|
|
129
|
+
* occurrence costs memory in proportion to occurrences times depth, where the
|
|
130
|
+
* deduplicated list needs only one entry per unique reference.
|
|
131
|
+
*/
|
|
132
|
+
function walkReferences(value, withPaths = true) {
|
|
133
|
+
// A Map keeps insertion order, which is the order of first sighting.
|
|
134
|
+
const byUri = new Map();
|
|
135
|
+
const trail = [];
|
|
136
|
+
const visit = (node) => {
|
|
137
|
+
if (typeof node === "string") {
|
|
138
|
+
if (!isStorageReference(node))
|
|
139
|
+
return;
|
|
140
|
+
const known = byUri.get(node);
|
|
141
|
+
if (!withPaths) {
|
|
142
|
+
if (known === undefined)
|
|
143
|
+
byUri.set(node, { uri: node, paths: [] });
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const path = [...trail];
|
|
147
|
+
if (known === undefined)
|
|
148
|
+
byUri.set(node, { uri: node, paths: [path] });
|
|
149
|
+
else
|
|
150
|
+
known.paths.push(path);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (Array.isArray(node)) {
|
|
154
|
+
for (let index = 0; index < node.length; index += 1) {
|
|
155
|
+
trail.push(index);
|
|
156
|
+
visit(node[index]);
|
|
157
|
+
trail.pop();
|
|
158
|
+
}
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if (typeof node === "object" && node !== null) {
|
|
162
|
+
for (const [key, entry] of Object.entries(node)) {
|
|
163
|
+
trail.push(key);
|
|
164
|
+
visit(entry);
|
|
165
|
+
trail.pop();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
visit(value);
|
|
170
|
+
return [...byUri.values()];
|
|
171
|
+
}
|
|
172
|
+
function renderLocation(located) {
|
|
173
|
+
return { uri: located.uri, found_at: located.paths.map(renderPath) };
|
|
174
|
+
}
|
|
175
|
+
/** Segments to the `$`-rooted notation `found_at` carries. */
|
|
176
|
+
function renderPath(segments) {
|
|
177
|
+
let path = "$";
|
|
178
|
+
for (const segment of segments) {
|
|
179
|
+
if (typeof segment === "number")
|
|
180
|
+
path += `[${segment}]`;
|
|
181
|
+
else if (IDENTIFIER_KEY.test(segment))
|
|
182
|
+
path += `.${segment}`;
|
|
183
|
+
else
|
|
184
|
+
path += `[${JSON.stringify(segment)}]`;
|
|
105
185
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
186
|
+
return path;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* The `$`-rooted notation back to segments, for a location that reaches
|
|
190
|
+
* {@link artifactFilename} from outside the walk. The rendering is lossless, so
|
|
191
|
+
* this is exact for every path the walk produced; anything else is `undefined`.
|
|
192
|
+
*/
|
|
193
|
+
function parsePath(path) {
|
|
194
|
+
if (!path.startsWith("$"))
|
|
195
|
+
return undefined;
|
|
196
|
+
const segments = [];
|
|
197
|
+
let at = 1;
|
|
198
|
+
while (at < path.length) {
|
|
199
|
+
const rest = path.slice(at);
|
|
200
|
+
const key = /^\.([A-Za-z_][A-Za-z0-9_]*)/.exec(rest);
|
|
201
|
+
if (key !== null) {
|
|
202
|
+
segments.push(key[1]);
|
|
203
|
+
at += key[0].length;
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const index = /^\[(\d+)\]/.exec(rest);
|
|
207
|
+
if (index !== null) {
|
|
208
|
+
segments.push(Number(index[1]));
|
|
209
|
+
at += index[0].length;
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
const quoted = /^\[("(?:[^"\\]|\\.)*")\]/s.exec(rest);
|
|
213
|
+
if (quoted === null)
|
|
214
|
+
return undefined;
|
|
215
|
+
try {
|
|
216
|
+
segments.push(JSON.parse(quoted[1]));
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
at += quoted[0].length;
|
|
109
222
|
}
|
|
223
|
+
return segments;
|
|
110
224
|
}
|
|
111
225
|
/** A string that is a storage reference: the scheme, then at least one character. */
|
|
112
226
|
export function isStorageReference(value) {
|
|
@@ -114,18 +228,95 @@ export function isStorageReference(value) {
|
|
|
114
228
|
}
|
|
115
229
|
// ── artifactFilename ─────────────────────────────────────────────────
|
|
116
230
|
/**
|
|
117
|
-
* The bare filename a
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
231
|
+
* The bare filename a reference is saved under, named after the field it fills:
|
|
232
|
+
* the path in `location.found_at[0]`, where the reference was first seen.
|
|
233
|
+
*
|
|
234
|
+
* 1. A final object key `url` is dropped, since the runtime's image and
|
|
235
|
+
* document contents carry their reference there; a reference under any other
|
|
236
|
+
* key keeps that key, and a `url` key that is not final is kept.
|
|
237
|
+
* 2. Each key is reduced to `[A-Za-z0-9_]`, every other character (`-` and `.`
|
|
238
|
+
* included) becoming `_`; an index stays its decimal digits.
|
|
239
|
+
* 3. The segments are joined with `-`. An empty result — the reference is the
|
|
240
|
+
* walked value itself, or its `url` — becomes the scope's name.
|
|
241
|
+
* 4. Over the filename length cap (`MAX_FILENAME_LENGTH`, extension included),
|
|
242
|
+
* the tail is kept: whole leading segments are dropped first, since the last
|
|
243
|
+
* ones are the specific ones, and a single segment still too long is cut to
|
|
244
|
+
* fit.
|
|
245
|
+
* 5. A stem Windows reserves for a device (`con`, `prn`, `aux`, `nul`,
|
|
246
|
+
* `com0`–`com9`, `lpt0`–`lpt9`, in any case) gets a trailing `_`, so
|
|
247
|
+
* `$.aux.url` is saved as `aux_.png`.
|
|
248
|
+
* 6. The extension is the storage key's own, reduced to `[A-Za-z0-9]`, when it
|
|
249
|
+
* has a short one; otherwise the content type's, for the types a run
|
|
250
|
+
* produces; otherwise there is none.
|
|
251
|
+
*
|
|
252
|
+
* So `$.rooms[3].staged_photo.url` is saved as `rooms-3-staged_photo.png`, and
|
|
253
|
+
* `$.url` in `main_stuff` as `main_stuff.png`. The stem is ASCII letters, digits,
|
|
254
|
+
* `_` and the `-` joins, never empty and never a device name, so the name can
|
|
255
|
+
* only ever be a regular file directly inside the target directory. A collision on disk is not this
|
|
125
256
|
* function's concern: `downloadArtifacts` suffixes the stem (`name-1.ext`) on
|
|
126
257
|
* exclusive creation, so a file is never overwritten.
|
|
258
|
+
*
|
|
259
|
+
* Throws `ArtifactOperationError` for a location whose `found_at[0]` is not a path
|
|
260
|
+
* in the notation {@link locateArtifacts} writes, or for an unknown scope.
|
|
127
261
|
*/
|
|
128
|
-
export function artifactFilename(
|
|
262
|
+
export function artifactFilename(location, contentType, scope) {
|
|
263
|
+
requireScope(scope);
|
|
264
|
+
// A JavaScript caller can hand anything here — the old signature's bare uri
|
|
265
|
+
// string among them — and every shape must reach the documented refusal
|
|
266
|
+
// rather than a TypeError, or a string's first character.
|
|
267
|
+
const loose = location;
|
|
268
|
+
const uri = loose?.uri;
|
|
269
|
+
if (typeof uri !== "string") {
|
|
270
|
+
throw new ArtifactOperationError(`artifactFilename needs a location carrying its reference as a string "uri"; got ${String(uri)}.`);
|
|
271
|
+
}
|
|
272
|
+
const foundAt = loose?.found_at;
|
|
273
|
+
const first = Array.isArray(foundAt) ? foundAt[0] : undefined;
|
|
274
|
+
const segments = typeof first === "string" ? parsePath(first) : undefined;
|
|
275
|
+
if (segments === undefined) {
|
|
276
|
+
throw new ArtifactOperationError(`artifactFilename needs a location whose first "found_at" entry is a path such as ` +
|
|
277
|
+
`"$.items[0].url"; got ${typeof first === "string" ? JSON.stringify(first) : String(first)}.`);
|
|
278
|
+
}
|
|
279
|
+
return filenameFor(segments, uri, contentType, scope);
|
|
280
|
+
}
|
|
281
|
+
/** The naming rule of {@link artifactFilename}, over the walk's own segments. */
|
|
282
|
+
function filenameFor(segments, uri, contentType, scope) {
|
|
283
|
+
const named = segments.at(-1) === "url" ? segments.slice(0, -1) : segments;
|
|
284
|
+
const words = named
|
|
285
|
+
.map((segment) => typeof segment === "number" ? String(segment) : segment.replace(/[^A-Za-z0-9_]/gu, "_"))
|
|
286
|
+
// Only the empty key reduces to nothing, and it says nothing about the field.
|
|
287
|
+
.filter((word) => word !== "");
|
|
288
|
+
const extension = extensionFor(uri, contentType);
|
|
289
|
+
const stem = fitStem(words.length > 0 ? words : [scope], MAX_FILENAME_LENGTH - extension.length);
|
|
290
|
+
// A device stem is at most four characters, so the `_` cannot overrun the cap.
|
|
291
|
+
return (WINDOWS_DEVICE_STEM.test(stem) ? `${stem}_` : stem) + extension;
|
|
292
|
+
}
|
|
293
|
+
/** The words joined with `-` within `budget` characters, keeping the tail. */
|
|
294
|
+
function fitStem(words, budget) {
|
|
295
|
+
let start = 0;
|
|
296
|
+
let length = words.reduce((sum, word) => sum + word.length, 0) + words.length - 1;
|
|
297
|
+
while (length > budget && start < words.length - 1) {
|
|
298
|
+
length -= words[start].length + 1;
|
|
299
|
+
start += 1;
|
|
300
|
+
}
|
|
301
|
+
const stem = words.slice(start).join("-");
|
|
302
|
+
return stem.length > budget ? stem.slice(0, budget) : stem;
|
|
303
|
+
}
|
|
304
|
+
/** `.ext` for the saved file — the storage key's own, else the content type's — or `""`. */
|
|
305
|
+
function extensionFor(uri, contentType) {
|
|
306
|
+
const fromKey = storageKeyExtension(uri);
|
|
307
|
+
if (fromKey !== "")
|
|
308
|
+
return `.${fromKey}`;
|
|
309
|
+
if (contentType == null)
|
|
310
|
+
return "";
|
|
311
|
+
return EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()] ?? "";
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* The extension the storage key's last segment carries, without its dot, reduced
|
|
315
|
+
* to `[A-Za-z0-9]` — or `""` when it has none, or none that short. The segment is
|
|
316
|
+
* what follows the last `/` or `\` once the scheme, query and fragment are gone,
|
|
317
|
+
* percent-decoded when it decodes; a leading dot is not an extension.
|
|
318
|
+
*/
|
|
319
|
+
function storageKeyExtension(uri) {
|
|
129
320
|
const key = uri.startsWith(PIPELEX_STORAGE_SCHEME)
|
|
130
321
|
? uri.slice(PIPELEX_STORAGE_SCHEME.length)
|
|
131
322
|
: uri;
|
|
@@ -138,41 +329,24 @@ export function artifactFilename(uri, contentType, index) {
|
|
|
138
329
|
decoded = decodeURIComponent(segment);
|
|
139
330
|
}
|
|
140
331
|
catch {
|
|
141
|
-
// A malformed escape sequence is kept as typed;
|
|
142
|
-
}
|
|
143
|
-
let name = decoded
|
|
144
|
-
.replace(/[^A-Za-z0-9._-]/g, "_")
|
|
145
|
-
.replace(/^[._-]+/, "")
|
|
146
|
-
.replace(/[._-]+$/, "");
|
|
147
|
-
if (name === "") {
|
|
148
|
-
name = `artifact-${index + 1}`;
|
|
149
|
-
}
|
|
150
|
-
if (name.length > MAX_FILENAME_LENGTH) {
|
|
151
|
-
const ext = extensionOf(name);
|
|
152
|
-
// The extension is kept only if there is room left for a stem. An extension
|
|
153
|
-
// at least as long as the cap would give `slice` a negative start, which
|
|
154
|
-
// counts from the END and yields a name LONGER than the cap — so a
|
|
155
|
-
// pathological extension is dropped rather than preserved.
|
|
156
|
-
name =
|
|
157
|
-
ext.length < MAX_FILENAME_LENGTH
|
|
158
|
-
? name.slice(0, MAX_FILENAME_LENGTH - ext.length) + ext
|
|
159
|
-
: name.slice(0, MAX_FILENAME_LENGTH);
|
|
160
|
-
}
|
|
161
|
-
if (extensionOf(name) === "") {
|
|
162
|
-
const ext = contentType == null
|
|
163
|
-
? undefined
|
|
164
|
-
: EXTENSION_BY_CONTENT_TYPE[contentType.split(";")[0].trim().toLowerCase()];
|
|
165
|
-
if (ext !== undefined) {
|
|
166
|
-
name += ext;
|
|
167
|
-
}
|
|
332
|
+
// A malformed escape sequence is kept as typed; the reduction handles it.
|
|
168
333
|
}
|
|
169
|
-
|
|
334
|
+
const dot = decoded.lastIndexOf(".");
|
|
335
|
+
if (dot <= 0)
|
|
336
|
+
return "";
|
|
337
|
+
const extension = decoded.slice(dot + 1).replace(/[^A-Za-z0-9]/g, "");
|
|
338
|
+
return extension.length <= MAX_EXTENSION_LENGTH ? extension : "";
|
|
170
339
|
}
|
|
171
340
|
/** `path.extname` for a bare filename, without the `node:path` import: `.ext`, or `""` (a leading dot is not an extension). */
|
|
172
341
|
function extensionOf(name) {
|
|
173
342
|
const dot = name.lastIndexOf(".");
|
|
174
343
|
return dot > 0 ? name.slice(dot) : "";
|
|
175
344
|
}
|
|
345
|
+
function requireScope(scope) {
|
|
346
|
+
if (scope !== "main_stuff" && scope !== "working_memory") {
|
|
347
|
+
throw new ArtifactOperationError(`"scope" must be "main_stuff" or "working_memory", got ${String(scope)}.`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
176
350
|
// ── resolveArtifacts ─────────────────────────────────────────────────
|
|
177
351
|
/**
|
|
178
352
|
* Resolve a list of references through the bulk route, chunked at
|
|
@@ -336,8 +510,10 @@ async function fetchResolvedUrl(uri, downloadUrl, bounds, dispatcher, release) {
|
|
|
336
510
|
if (timedOut) {
|
|
337
511
|
return new ArtifactFetchError(`Fetching the artifact timed out after ${bounds.timeoutMs}ms.`, uri, "timeout", undefined, { cause: err });
|
|
338
512
|
}
|
|
513
|
+
// The caller's reason, not `err`: a browser rejects the fetch, and errors a body
|
|
514
|
+
// cut short, with a generic AbortError rather than the signal's reason.
|
|
339
515
|
if (userSignal?.aborted)
|
|
340
|
-
return
|
|
516
|
+
return userSignal.reason;
|
|
341
517
|
if (err instanceof ArtifactFetchError)
|
|
342
518
|
return err;
|
|
343
519
|
return new ArtifactFetchError(`The artifact could not be fetched: ${err instanceof Error ? err.message : String(err)}.`, uri, "network", undefined, { cause: err });
|
|
@@ -543,8 +719,15 @@ class ArtifactWriteError extends Error {
|
|
|
543
719
|
const SKIPPED_ABORTED = "The download was aborted before this artifact was fetched.";
|
|
544
720
|
const SKIPPED_CREDENTIAL = "The download stopped on a credential failure before this artifact was fetched.";
|
|
545
721
|
const SKIPPED_TOTAL = "Skipped: the download's total byte limit was reached by an earlier artifact.";
|
|
546
|
-
function itemError(
|
|
547
|
-
return {
|
|
722
|
+
function itemError(location, contentType, code, detail) {
|
|
723
|
+
return {
|
|
724
|
+
uri: location.uri,
|
|
725
|
+
found_at: location.found_at,
|
|
726
|
+
path: null,
|
|
727
|
+
content_type: contentType,
|
|
728
|
+
size: null,
|
|
729
|
+
error: { code, detail },
|
|
730
|
+
};
|
|
548
731
|
}
|
|
549
732
|
function isExpired(expiresAt) {
|
|
550
733
|
const at = Date.parse(expiresAt);
|
|
@@ -569,19 +752,20 @@ async function readCompletedResults(client, runId, signal) {
|
|
|
569
752
|
* Save a run's produced files under `dir` — Node-only, like the path-string arm
|
|
570
753
|
* of `uploadFile`. Takes a `run_id` (the results are re-read, so a run is
|
|
571
754
|
* downloadable days later) or a `RunResults` in hand; walks the requested scope
|
|
572
|
-
* with {@link
|
|
755
|
+
* with {@link locateArtifacts}; resolves the whole set through the bulk route
|
|
573
756
|
* ahead of the workers; then a bounded pool of workers each fetch → open with
|
|
574
757
|
* `wx` → write, re-resolving any link that has expired by the time a worker
|
|
575
|
-
* reaches it. The embedded `public_url` is never used.
|
|
576
|
-
* {@link artifactFilename} and never overwritten;
|
|
577
|
-
* unlinks its partial file.
|
|
758
|
+
* reaches it. The embedded `public_url` is never used. Each file is named after
|
|
759
|
+
* the field it fills, by {@link artifactFilename}'s rule, and never overwritten;
|
|
760
|
+
* a failed or aborted download unlinks its partial file.
|
|
578
761
|
*
|
|
579
762
|
* Returns a produced verdict: one entry per reference, errors as values. It
|
|
580
763
|
* throws only when no verdict can be produced — `RunStillRunningError` or
|
|
581
764
|
* `RunFailedError` for a run that has not completed, `ScopeUnavailableError`
|
|
582
765
|
* when the scope's artifact is null or missing, `ArtifactAuthenticationError`
|
|
583
766
|
* (carrying the verdict so far) when the resolve route refuses the credential,
|
|
584
|
-
* `ArtifactOperationError` outside Node
|
|
767
|
+
* `ArtifactOperationError` outside Node, for an unusable `dir` or for an
|
|
768
|
+
* unknown `scope`, and the
|
|
585
769
|
* transport and lifecycle errors of the reads it makes (`ApiResponseError` for
|
|
586
770
|
* a deployment without the bulk route, `RunLifecycleUnavailableError` for a
|
|
587
771
|
* bare runner asked by id, `ApiUnreachableError`).
|
|
@@ -592,6 +776,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
592
776
|
"with resolveArtifacts or stream with fetchArtifact instead.");
|
|
593
777
|
}
|
|
594
778
|
const scope = request.scope ?? "main_stuff";
|
|
779
|
+
requireScope(scope);
|
|
595
780
|
const concurrency = request.concurrency ?? DEFAULT_DOWNLOAD_CONCURRENCY;
|
|
596
781
|
const maxTotalBytes = request.maxTotalBytes ?? DEFAULT_DOWNLOAD_MAX_TOTAL_BYTES;
|
|
597
782
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
@@ -617,10 +802,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
617
802
|
if (walked == null) {
|
|
618
803
|
throw new ScopeUnavailableError(scope, runId);
|
|
619
804
|
}
|
|
620
|
-
|
|
621
|
-
|
|
805
|
+
// The walk's own record names the files; `locations` is what the verdict reports.
|
|
806
|
+
const located = walkReferences(walked);
|
|
807
|
+
if (located.length === 0) {
|
|
622
808
|
return { scope, artifacts: [], saved_paths: [], all_saved: true };
|
|
623
809
|
}
|
|
810
|
+
const locations = located.map(renderLocation);
|
|
811
|
+
const uris = located.map((reference) => reference.uri);
|
|
624
812
|
const fs = (await import("node:fs/promises"));
|
|
625
813
|
const path = (await import("node:path"));
|
|
626
814
|
const dir = path.resolve(request.dir);
|
|
@@ -634,7 +822,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
634
822
|
// must name every reference, reached or not.
|
|
635
823
|
const outcomes = Array.from({ length: uris.length }, () => undefined);
|
|
636
824
|
const verdictSoFar = () => {
|
|
637
|
-
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(
|
|
825
|
+
const artifacts = outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_CREDENTIAL));
|
|
638
826
|
return assembleVerdict(scope, artifacts, request.signal?.aborted === true);
|
|
639
827
|
};
|
|
640
828
|
let resolved;
|
|
@@ -671,13 +859,15 @@ export async function downloadArtifacts(client, request) {
|
|
|
671
859
|
let savedBytes = 0;
|
|
672
860
|
let limitReached = false;
|
|
673
861
|
let next = 0;
|
|
674
|
-
const saveOne = async (index,
|
|
862
|
+
const saveOne = async (index, url, contentType) => {
|
|
863
|
+
const location = locations[index];
|
|
864
|
+
const uri = location.uri;
|
|
675
865
|
let response;
|
|
676
866
|
try {
|
|
677
867
|
response = await fetchResolvedUrl(uri, url, workerBounds, dispatcher, () => undefined);
|
|
678
868
|
}
|
|
679
869
|
catch (err) {
|
|
680
|
-
return itemError(
|
|
870
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
681
871
|
}
|
|
682
872
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
683
873
|
const reserved = Number.isFinite(declaredLength) ? declaredLength : 0;
|
|
@@ -685,7 +875,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
685
875
|
if (savedBytes + reserved > maxTotalBytes)
|
|
686
876
|
limitReached = true;
|
|
687
877
|
await discard(response);
|
|
688
|
-
return itemError(
|
|
878
|
+
return itemError(location, contentType, "total_limit_exceeded", `Saving this ${formatMiB(reserved)} artifact would take the download past its ` +
|
|
689
879
|
`${formatMiB(maxTotalBytes)} total limit.`);
|
|
690
880
|
}
|
|
691
881
|
committedBytes += reserved;
|
|
@@ -693,12 +883,13 @@ export async function downloadArtifacts(client, request) {
|
|
|
693
883
|
const share = () => Math.max(written, reserved);
|
|
694
884
|
let target;
|
|
695
885
|
try {
|
|
696
|
-
|
|
886
|
+
const filename = filenameFor(located[index].paths[0], uri, contentType, scope);
|
|
887
|
+
target = await openUniqueFile(fs, path, dir, filename);
|
|
697
888
|
}
|
|
698
889
|
catch (err) {
|
|
699
890
|
committedBytes -= share();
|
|
700
891
|
await discard(response);
|
|
701
|
-
return itemError(
|
|
892
|
+
return itemError(location, contentType, "write_failed", `The file could not be created: ${err instanceof Error ? err.message : String(err)}.`);
|
|
702
893
|
}
|
|
703
894
|
const removePartial = async () => {
|
|
704
895
|
committedBytes -= share();
|
|
@@ -711,10 +902,17 @@ export async function downloadArtifacts(client, request) {
|
|
|
711
902
|
}
|
|
712
903
|
catch (err) {
|
|
713
904
|
await removePartial();
|
|
714
|
-
return itemError(
|
|
905
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
715
906
|
}
|
|
716
907
|
committedBytes -= share();
|
|
717
|
-
return {
|
|
908
|
+
return {
|
|
909
|
+
uri,
|
|
910
|
+
found_at: location.found_at,
|
|
911
|
+
path: target.path,
|
|
912
|
+
content_type: contentType,
|
|
913
|
+
size: 0,
|
|
914
|
+
error: null,
|
|
915
|
+
};
|
|
718
916
|
}
|
|
719
917
|
const reader = response.body.getReader();
|
|
720
918
|
try {
|
|
@@ -729,7 +927,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
729
927
|
limitReached = true;
|
|
730
928
|
await reader.cancel().catch(() => undefined);
|
|
731
929
|
await removePartial();
|
|
732
|
-
return itemError(
|
|
930
|
+
return itemError(location, contentType, "total_limit_exceeded", `This artifact took the download past its ${formatMiB(maxTotalBytes)} total limit.`);
|
|
733
931
|
}
|
|
734
932
|
committedBytes += growth;
|
|
735
933
|
written += value.byteLength;
|
|
@@ -745,21 +943,28 @@ export async function downloadArtifacts(client, request) {
|
|
|
745
943
|
await reader.cancel().catch(() => undefined);
|
|
746
944
|
await removePartial();
|
|
747
945
|
if (err instanceof ArtifactWriteError) {
|
|
748
|
-
return itemError(
|
|
946
|
+
return itemError(location, contentType, "write_failed", `The file could not be written: ${err.message}.`);
|
|
749
947
|
}
|
|
750
|
-
return itemError(
|
|
948
|
+
return itemError(location, contentType, ...classifyFailure(err, signal));
|
|
751
949
|
}
|
|
752
950
|
try {
|
|
753
951
|
await target.handle.close();
|
|
754
952
|
}
|
|
755
953
|
catch (err) {
|
|
756
954
|
await removePartial();
|
|
757
|
-
return itemError(
|
|
955
|
+
return itemError(location, contentType, "write_failed", `The file could not be closed: ${err instanceof Error ? err.message : String(err)}.`);
|
|
758
956
|
}
|
|
759
957
|
// A body shorter than it declared gives the unused reservation back.
|
|
760
958
|
committedBytes -= share() - written;
|
|
761
959
|
savedBytes += written;
|
|
762
|
-
return {
|
|
960
|
+
return {
|
|
961
|
+
uri,
|
|
962
|
+
found_at: location.found_at,
|
|
963
|
+
path: target.path,
|
|
964
|
+
content_type: contentType,
|
|
965
|
+
size: written,
|
|
966
|
+
error: null,
|
|
967
|
+
};
|
|
763
968
|
};
|
|
764
969
|
const worker = async () => {
|
|
765
970
|
for (;;) {
|
|
@@ -767,40 +972,40 @@ export async function downloadArtifacts(client, request) {
|
|
|
767
972
|
next += 1;
|
|
768
973
|
if (index >= uris.length)
|
|
769
974
|
return;
|
|
770
|
-
const
|
|
975
|
+
const location = locations[index];
|
|
771
976
|
let entry = resolved[index];
|
|
772
977
|
if (credentialFailure !== undefined || aborted()) {
|
|
773
|
-
outcomes[index] = itemError(
|
|
978
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", credentialFailure ? SKIPPED_CREDENTIAL : SKIPPED_ABORTED);
|
|
774
979
|
continue;
|
|
775
980
|
}
|
|
776
981
|
if (limitReached) {
|
|
777
|
-
outcomes[index] = itemError(
|
|
982
|
+
outcomes[index] = itemError(location, entry.content_type, "total_limit_exceeded", SKIPPED_TOTAL);
|
|
778
983
|
continue;
|
|
779
984
|
}
|
|
780
985
|
if (entry.error === null && isExpired(entry.expires_at)) {
|
|
781
986
|
try {
|
|
782
|
-
const again = await resolveArtifacts(client, [uri], { signal });
|
|
987
|
+
const again = await resolveArtifacts(client, [location.uri], { signal });
|
|
783
988
|
entry = again[0];
|
|
784
989
|
}
|
|
785
990
|
catch (err) {
|
|
786
991
|
if (isCredentialRefusal(err)) {
|
|
787
992
|
credentialFailure = err;
|
|
788
|
-
outcomes[index] = itemError(
|
|
993
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_CREDENTIAL);
|
|
789
994
|
continue;
|
|
790
995
|
}
|
|
791
996
|
if (aborted()) {
|
|
792
|
-
outcomes[index] = itemError(
|
|
997
|
+
outcomes[index] = itemError(location, entry.content_type, "aborted", SKIPPED_ABORTED);
|
|
793
998
|
continue;
|
|
794
999
|
}
|
|
795
|
-
outcomes[index] = itemError(
|
|
1000
|
+
outcomes[index] = itemError(location, entry.content_type, "resolve_failed", `The expired link could not be re-resolved: ${err instanceof Error ? err.message : String(err)}.`);
|
|
796
1001
|
continue;
|
|
797
1002
|
}
|
|
798
1003
|
}
|
|
799
1004
|
if (entry.error !== null) {
|
|
800
|
-
outcomes[index] = itemError(
|
|
1005
|
+
outcomes[index] = itemError(location, null, entry.error.code, entry.error.detail);
|
|
801
1006
|
continue;
|
|
802
1007
|
}
|
|
803
|
-
outcomes[index] = await saveOne(index,
|
|
1008
|
+
outcomes[index] = await saveOne(index, entry.url, entry.content_type);
|
|
804
1009
|
}
|
|
805
1010
|
};
|
|
806
1011
|
try {
|
|
@@ -813,7 +1018,7 @@ export async function downloadArtifacts(client, request) {
|
|
|
813
1018
|
throw new ArtifactAuthenticationError(`The resolve route refused the credential (${credentialFailure.status}) part-way through ` +
|
|
814
1019
|
"the download; the verdict so far is on this error.", credentialFailure.status, verdictSoFar(), { cause: credentialFailure });
|
|
815
1020
|
}
|
|
816
|
-
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(
|
|
1021
|
+
return assembleVerdict(scope, outcomes.map((outcome, index) => outcome ?? itemError(locations[index], null, "aborted", SKIPPED_ABORTED)), request.signal?.aborted === true);
|
|
817
1022
|
}
|
|
818
1023
|
function assembleVerdict(scope, artifacts, aborted) {
|
|
819
1024
|
const saved_paths = artifacts.flatMap((artifact) => artifact.error === null ? [artifact.path] : []);
|