@the-open-engine/zeroshot 6.25.0 → 6.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +102 -74
- package/package.json +4 -2
- package/src/agent/provider-session.js +18 -4
- package/src/hosted-target/bounds.ts +6 -0
- package/src/hosted-target/errors.ts +90 -0
- package/src/hosted-target/index.ts +44 -0
- package/src/hosted-target/response-validation.ts +86 -0
- package/src/hosted-target/retry.ts +46 -0
- package/src/hosted-target/target-adapter.ts +10 -0
- package/src/hosted-target/types.ts +58 -0
- package/src/hosted-target/zero-cloud-v1-adapter.ts +386 -0
- package/src/omp-session-limits.js +25 -1
- package/src/omp-session-partition.js +159 -32
- package/src/omp-session-verifier.js +251 -87
- package/task-lib/commands/clean.js +80 -46
- package/task-lib/commands/kill.js +21 -0
- package/task-lib/commands/resume.js +20 -0
- package/task-lib/commands/run.js +17 -2
- package/task-lib/omp-session-cleanup.js +46 -9
- package/task-lib/omp-session-ownership-schema.js +21 -15
- package/task-lib/omp-session-ownership.js +66 -31
- package/task-lib/rpc-watcher.js +51 -15
- package/task-lib/runner.js +51 -4
- package/task-lib/store.js +67 -72
|
@@ -20,12 +20,28 @@
|
|
|
20
20
|
// byte read comes from `fstat`/`read` on that same descriptor. There is no lstat -> open -> stat
|
|
21
21
|
// pathname sequence and no re-open of a mutable name after validation, so the substituted-file
|
|
22
22
|
// race (CodeQL js/file-system-race) cannot apply: the object we checked is the object we read.
|
|
23
|
-
// Directory listings additionally re-pin and compare identity around
|
|
24
|
-
// substitution as a verification failure rather than silently traversing the replacement.
|
|
23
|
+
// Directory listings additionally re-pin and compare identity around the enumeration, which reports
|
|
24
|
+
// a substitution as a verification failure rather than silently traversing the replacement.
|
|
25
|
+
//
|
|
26
|
+
// Names are handled as RAW BYTES wherever the platform has raw bytes to give. A POSIX filename is
|
|
27
|
+
// an opaque byte string, not text: two distinct files can carry names that are both invalid UTF-8
|
|
28
|
+
// and that Node's string decoding collapses to the same run of U+FFFD replacement characters.
|
|
29
|
+
// Hashing the re-encoded string would then give two different trees the same manifest digest — a
|
|
30
|
+
// collision an attacker picks the filenames for. So directory entries are enumerated with
|
|
31
|
+
// `encoding: 'buffer'`, every relative path is assembled, bounded, separator-checked, sorted, and
|
|
32
|
+
// length-prefix-hashed as bytes, and child paths are opened from those same bytes rather than from
|
|
33
|
+
// a string round trip. Windows has no such thing: the OS gives UTF-16 names, Node's fs rejects
|
|
34
|
+
// Buffer paths there, and the existing string behavior is already lossless — so that platform keeps
|
|
35
|
+
// it, with the same manifest layout.
|
|
36
|
+
//
|
|
37
|
+
// Memory is bounded by the pinned OMP_SESSION_LIMITS rather than by the input: file bytes are
|
|
38
|
+
// streamed and hashed in fixed chunks, one JSONL record is capped at MAX_SESSION_RECORD_BYTES, the
|
|
39
|
+
// blob-reference walk is iterative (a hostile record cannot exhaust the call stack), and directory
|
|
40
|
+
// enumeration stops as soon as it has read more names than the entry budget could allow.
|
|
25
41
|
const { createHash } = require('crypto');
|
|
26
42
|
const fs = require('fs');
|
|
27
43
|
const path = require('path');
|
|
28
|
-
const { OMP_SESSION_LIMITS } = require('./omp-session-limits');
|
|
44
|
+
const { OMP_SESSION_LIMITS, MAX_SESSION_RECORD_BYTES } = require('./omp-session-limits');
|
|
29
45
|
const { resolveOmpBlobsDir } = require('./omp-blob-root');
|
|
30
46
|
|
|
31
47
|
const BLOB_REF_PREFIX = 'blob:sha256:';
|
|
@@ -34,6 +50,48 @@ const SESSION_FILE_NAME_PATTERN = /^[^/\\]+\.jsonl$/u;
|
|
|
34
50
|
const NEWLINE = 0x0a;
|
|
35
51
|
const STREAM_CHUNK_BYTES = 1 << 16;
|
|
36
52
|
|
|
53
|
+
// Raw-byte names are a POSIX property. On Windows, fs does not accept Buffer paths at all.
|
|
54
|
+
const RAW_NAME_BYTES_SUPPORTED = process.platform !== 'win32';
|
|
55
|
+
const FORWARD_SLASH = 0x2f;
|
|
56
|
+
const BACKSLASH = 0x5c;
|
|
57
|
+
const NUL = 0x00;
|
|
58
|
+
const DOT = 0x2e;
|
|
59
|
+
// The manifest's relative-path separator is always '/', on every platform, so a manifest computed
|
|
60
|
+
// for the same tree is the same manifest everywhere.
|
|
61
|
+
const MANIFEST_SEPARATOR = Buffer.from('/', 'utf8');
|
|
62
|
+
const PATH_SEPARATOR_BYTES = Buffer.from(path.sep, 'utf8');
|
|
63
|
+
|
|
64
|
+
/** A filesystem path in the form this platform's fs takes losslessly: raw bytes on POSIX, the
|
|
65
|
+
* original string on Windows. */
|
|
66
|
+
function toPathRef(pathString) {
|
|
67
|
+
return RAW_NAME_BYTES_SUPPORTED ? Buffer.from(pathString, 'utf8') : pathString;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** A directory entry's name as raw bytes (already bytes on POSIX; UTF-8 of the OS's UTF-16 name on
|
|
71
|
+
* Windows, which is the lossless encoding of that name). */
|
|
72
|
+
function nameToBytes(name) {
|
|
73
|
+
return Buffer.isBuffer(name) ? name : Buffer.from(name, 'utf8');
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Extend a pinned directory path with one child name, without a string round trip on POSIX. */
|
|
77
|
+
function joinChildRef(dirRef, nameBytes) {
|
|
78
|
+
if (!RAW_NAME_BYTES_SUPPORTED) return path.join(dirRef, nameBytes.toString('utf8'));
|
|
79
|
+
return Buffer.concat([dirRef, PATH_SEPARATOR_BYTES, nameBytes]);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* A printable rendering for error messages. Names that are not valid UTF-8 cannot be shown as text
|
|
84
|
+
* without lying about their bytes, so those carry the exact hex alongside — an operator reading a
|
|
85
|
+
* refused resume needs to be able to identify the actual file.
|
|
86
|
+
*/
|
|
87
|
+
function describeRef(ref) {
|
|
88
|
+
if (!Buffer.isBuffer(ref)) return String(ref);
|
|
89
|
+
const text = ref.toString('utf8');
|
|
90
|
+
return Buffer.from(text, 'utf8').equals(ref)
|
|
91
|
+
? text
|
|
92
|
+
: `${text} (raw bytes ${ref.toString('hex')})`;
|
|
93
|
+
}
|
|
94
|
+
|
|
37
95
|
const O_NOFOLLOW = fs.constants.O_NOFOLLOW ?? 0;
|
|
38
96
|
const O_DIRECTORY = fs.constants.O_DIRECTORY ?? 0;
|
|
39
97
|
// O_NONBLOCK is what keeps "reject sockets/devices/FIFOs" from being a liveness hole: opening a
|
|
@@ -72,7 +130,7 @@ function isSymlink(targetPath) {
|
|
|
72
130
|
|
|
73
131
|
function assertOwnerHeld(stat, targetPath) {
|
|
74
132
|
if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) {
|
|
75
|
-
fail('not-owner-held', `${targetPath} is not owned by the current user.`);
|
|
133
|
+
fail('not-owner-held', `${describeRef(targetPath)} is not owned by the current user.`);
|
|
76
134
|
}
|
|
77
135
|
}
|
|
78
136
|
|
|
@@ -94,22 +152,25 @@ function openPinned(targetPath, { directory = false, missingCode, notTypeCode })
|
|
|
94
152
|
// ENOTDIR on Linux, and "is a symlink" is a far more actionable message than "is not a
|
|
95
153
|
// directory" for the operator reading a refused resume.
|
|
96
154
|
if (error.code === 'ELOOP' || error.code === 'EMLINK' || isSymlink(targetPath)) {
|
|
97
|
-
fail('symlink-rejected', `${targetPath} is a symlink.`);
|
|
155
|
+
fail('symlink-rejected', `${describeRef(targetPath)} is a symlink.`);
|
|
98
156
|
}
|
|
99
157
|
if (error.code === 'ENOTDIR') {
|
|
100
|
-
fail(notTypeCode, `${targetPath} is not a directory.`);
|
|
158
|
+
fail(notTypeCode, `${describeRef(targetPath)} is not a directory.`);
|
|
101
159
|
}
|
|
102
160
|
if (error.code === 'EISDIR') {
|
|
103
|
-
fail(notTypeCode, `${targetPath} is a directory, not a regular file.`);
|
|
161
|
+
fail(notTypeCode, `${describeRef(targetPath)} is a directory, not a regular file.`);
|
|
104
162
|
}
|
|
105
|
-
fail(missingCode, `${targetPath} could not be opened: ${error.message}`);
|
|
163
|
+
fail(missingCode, `${describeRef(targetPath)} could not be opened: ${error.message}`);
|
|
106
164
|
}
|
|
107
165
|
let stat;
|
|
108
166
|
try {
|
|
109
167
|
stat = fs.fstatSync(fd);
|
|
110
168
|
} catch (error) {
|
|
111
169
|
fs.closeSync(fd);
|
|
112
|
-
fail(
|
|
170
|
+
fail(
|
|
171
|
+
missingCode,
|
|
172
|
+
`${describeRef(targetPath)} could not be stat'ed from its descriptor: ${error.message}`
|
|
173
|
+
);
|
|
113
174
|
}
|
|
114
175
|
// O_DIRECTORY/O_NOFOLLOW already excluded the wrong-type and symlink cases on platforms that
|
|
115
176
|
// implement them; re-assert from the descriptor so a platform lacking the flags still fails
|
|
@@ -118,7 +179,7 @@ function openPinned(targetPath, { directory = false, missingCode, notTypeCode })
|
|
|
118
179
|
fs.closeSync(fd);
|
|
119
180
|
fail(
|
|
120
181
|
notTypeCode,
|
|
121
|
-
`${targetPath} is not a ${directory ? 'real directory' : 'regular file'} (mode ${stat.mode.toString(8)}).`
|
|
182
|
+
`${describeRef(targetPath)} is not a ${directory ? 'real directory' : 'regular file'} (mode ${stat.mode.toString(8)}).`
|
|
122
183
|
);
|
|
123
184
|
}
|
|
124
185
|
try {
|
|
@@ -140,45 +201,86 @@ function withPinned(targetPath, options, body) {
|
|
|
140
201
|
}
|
|
141
202
|
|
|
142
203
|
/**
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* bracketing it with the identity comparison turns a substitution into a
|
|
146
|
-
* instead of a silent traversal of the replacement tree.
|
|
204
|
+
* Enumerate a directory whose identity is already pinned, re-pinning afterwards and comparing
|
|
205
|
+
* identity. Directory enumeration has no descriptor-taking form in Node, so this is the one
|
|
206
|
+
* unavoidable name lookup — bracketing it with the identity comparison turns a substitution into a
|
|
207
|
+
* hard verification failure instead of a silent traversal of the replacement tree.
|
|
208
|
+
*
|
|
209
|
+
* Names come back as raw bytes (`encoding: 'buffer'`) wherever the platform has them, and the sort
|
|
210
|
+
* is a byte-wise comparison, so the traversal order is a property of the bytes on disk rather than
|
|
211
|
+
* of how they happen to decode.
|
|
212
|
+
*
|
|
213
|
+
* `maxNames` is the largest number of entries this level could legitimately hold given the entry
|
|
214
|
+
* budget already consumed. Reading is abandoned the moment that is exceeded, so a directory with
|
|
215
|
+
* ten million entries costs the budget, not the directory: `opendir` streams, unlike `readdir`,
|
|
216
|
+
* which would materialize every name before any bound could be applied.
|
|
147
217
|
*/
|
|
148
|
-
function readdirPinned(
|
|
149
|
-
|
|
218
|
+
function readdirPinned(dirRef, expectedIdentity, maxNames) {
|
|
219
|
+
const children = [];
|
|
220
|
+
let dir;
|
|
150
221
|
try {
|
|
151
|
-
|
|
222
|
+
dir = fs.opendirSync(dirRef, RAW_NAME_BYTES_SUPPORTED ? { encoding: 'buffer' } : {});
|
|
152
223
|
} catch (error) {
|
|
153
|
-
fail('artifact-read-failed', `${
|
|
224
|
+
fail('artifact-read-failed', `${describeRef(dirRef)} could not be listed: ${error.message}`);
|
|
225
|
+
}
|
|
226
|
+
try {
|
|
227
|
+
for (;;) {
|
|
228
|
+
const entry = dir.readSync();
|
|
229
|
+
if (entry === null || entry === undefined) break;
|
|
230
|
+
if (children.length >= maxNames) {
|
|
231
|
+
fail('artifact-entries-exceeded', 'Artifact tree exceeds maxArtifactEntries.');
|
|
232
|
+
}
|
|
233
|
+
children.push({
|
|
234
|
+
name: nameToBytes(entry.name),
|
|
235
|
+
directory: entry.isDirectory(),
|
|
236
|
+
file: entry.isFile(),
|
|
237
|
+
symlink: entry.isSymbolicLink(),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
} finally {
|
|
241
|
+
try {
|
|
242
|
+
dir.closeSync();
|
|
243
|
+
} catch {
|
|
244
|
+
// The enumeration result is what matters; a close failure cannot invalidate it.
|
|
245
|
+
}
|
|
154
246
|
}
|
|
155
247
|
withPinned(
|
|
156
|
-
|
|
248
|
+
dirRef,
|
|
157
249
|
{ directory: true, missingCode: 'partition-missing', notTypeCode: 'not-a-directory' },
|
|
158
250
|
({ stat }) => {
|
|
159
251
|
if (!sameIdentity(identityOf(stat), expectedIdentity)) {
|
|
160
|
-
fail(
|
|
252
|
+
fail(
|
|
253
|
+
'identity-substituted',
|
|
254
|
+
`${describeRef(dirRef)} was substituted while it was being listed.`
|
|
255
|
+
);
|
|
161
256
|
}
|
|
162
257
|
}
|
|
163
258
|
);
|
|
164
|
-
return children.sort((a, b) => (a.name
|
|
259
|
+
return children.sort((a, b) => Buffer.compare(a.name, b.name));
|
|
165
260
|
}
|
|
166
261
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
262
|
+
/**
|
|
263
|
+
* Validate a direct-child name by *byte*, not by decoded text: a path separator is a byte, and a
|
|
264
|
+
* name that decodes to something harmless can still carry one. `.`/`..` are compared as bytes for
|
|
265
|
+
* the same reason.
|
|
266
|
+
*/
|
|
267
|
+
function assertDirectChildNameBytes(nameBytes, code = 'invalid-relative-path') {
|
|
268
|
+
const invalid =
|
|
269
|
+
nameBytes.length === 0 ||
|
|
270
|
+
nameBytes.includes(FORWARD_SLASH) ||
|
|
271
|
+
nameBytes.includes(BACKSLASH) ||
|
|
272
|
+
nameBytes.includes(NUL) ||
|
|
273
|
+
(nameBytes.length <= 2 && nameBytes.every((byte) => byte === DOT));
|
|
274
|
+
if (invalid) {
|
|
275
|
+
fail(code, `Invalid direct-child name: ${describeRef(nameBytes)}`);
|
|
177
276
|
}
|
|
178
277
|
}
|
|
179
278
|
|
|
180
279
|
function assertSessionFileName(name) {
|
|
181
|
-
|
|
280
|
+
if (typeof name !== 'string') {
|
|
281
|
+
fail('invalid-session-file-name', `Session file name must be a string: ${String(name)}`);
|
|
282
|
+
}
|
|
283
|
+
assertDirectChildNameBytes(Buffer.from(name, 'utf8'), 'invalid-session-file-name');
|
|
182
284
|
if (!SESSION_FILE_NAME_PATTERN.test(name)) {
|
|
183
285
|
fail(
|
|
184
286
|
'invalid-session-file-name',
|
|
@@ -205,36 +307,57 @@ function streamDescriptor(fd, maxBytes, onChunk, overflow) {
|
|
|
205
307
|
return observed;
|
|
206
308
|
}
|
|
207
309
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
310
|
+
/**
|
|
311
|
+
* Walk one parsed record for nested canonical blob references.
|
|
312
|
+
*
|
|
313
|
+
* Iterative on purpose. A JSONL record is attacker-controlled, and a recursive walk over a deeply
|
|
314
|
+
* nested value would exhaust the call stack — a RangeError thrown from outside any handler rather
|
|
315
|
+
* than a verification failure. An explicit stack is bounded by the parsed value, which is in turn
|
|
316
|
+
* bounded by MAX_SESSION_RECORD_BYTES. (`JSON.parse` itself refuses over-deep input with a
|
|
317
|
+
* SyntaxError/RangeError, which the caller already converts into a record-unparseable failure.)
|
|
318
|
+
*/
|
|
319
|
+
function collectCanonicalBlobRefs(root, sink, limits) {
|
|
320
|
+
const stack = [root];
|
|
321
|
+
while (stack.length > 0) {
|
|
322
|
+
const value = stack.pop();
|
|
323
|
+
if (typeof value === 'string') {
|
|
324
|
+
if (!value.startsWith(BLOB_REF_PREFIX)) continue;
|
|
325
|
+
if (!CANONICAL_BLOB_REF_PATTERN.test(value)) {
|
|
326
|
+
// OMP's parseBlobRef only warns and falls back to treating a malformed ref as literal data
|
|
327
|
+
// (blob-store.ts). Zeroshot cannot: a continuation whose externalized payload is
|
|
328
|
+
// unaddressable is not a continuation we can prove, so this fails closed.
|
|
329
|
+
fail(
|
|
330
|
+
'blob-reference-noncanonical',
|
|
331
|
+
`Non-canonical blob reference ${JSON.stringify(value)}.`
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
sink.add(value);
|
|
335
|
+
if (sink.size > limits.maxBlobReferences) {
|
|
336
|
+
fail('blob-references-exceeded', 'Session exceeds maxBlobReferences.');
|
|
337
|
+
}
|
|
338
|
+
continue;
|
|
216
339
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
340
|
+
if (Array.isArray(value)) {
|
|
341
|
+
for (const item of value) stack.push(item);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
if (value !== null && typeof value === 'object') {
|
|
345
|
+
for (const key of Object.keys(value)) stack.push(value[key]);
|
|
220
346
|
}
|
|
221
|
-
return;
|
|
222
|
-
}
|
|
223
|
-
if (Array.isArray(value)) {
|
|
224
|
-
for (const item of value) collectCanonicalBlobRefs(item, sink, limits);
|
|
225
|
-
return;
|
|
226
|
-
}
|
|
227
|
-
if (value !== null && typeof value === 'object') {
|
|
228
|
-
for (const key of Object.keys(value)) collectCanonicalBlobRefs(value[key], sink, limits);
|
|
229
347
|
}
|
|
230
348
|
}
|
|
231
349
|
|
|
232
350
|
/**
|
|
233
351
|
* Stream the session JSONL from its pinned descriptor: bound bytes/records, hash the raw content,
|
|
234
352
|
* parse each record, verify the session header record, and collect every canonical nested blob
|
|
235
|
-
* reference.
|
|
236
|
-
*
|
|
237
|
-
*
|
|
353
|
+
* reference.
|
|
354
|
+
*
|
|
355
|
+
* A record is buffered only until its terminating newline, and never past
|
|
356
|
+
* MAX_SESSION_RECORD_BYTES. That second bound is what makes this safe against a hostile transcript:
|
|
357
|
+
* `maxSessionBytes` bounds the file, but a 256 MiB file containing no newline is a *single* record,
|
|
358
|
+
* and buffering it would cost the raw bytes plus a UTF-16 string plus a parsed value. The record
|
|
359
|
+
* bound is checked before each append, i.e. before the copy it would authorize, so an over-long
|
|
360
|
+
* record is refused rather than accumulated. See src/omp-session-limits.js for the derivation.
|
|
238
361
|
*/
|
|
239
362
|
function streamSessionJsonl(fd, describePath, limits) {
|
|
240
363
|
const hash = createHash('sha256');
|
|
@@ -244,14 +367,28 @@ function streamSessionJsonl(fd, describePath, limits) {
|
|
|
244
367
|
let pending = [];
|
|
245
368
|
let pendingBytes = 0;
|
|
246
369
|
|
|
370
|
+
function appendPending(slice) {
|
|
371
|
+
if (pendingBytes + slice.length > MAX_SESSION_RECORD_BYTES) {
|
|
372
|
+
fail(
|
|
373
|
+
'session-record-bytes-exceeded',
|
|
374
|
+
`${describePath} record ${records + 1} exceeds the ${MAX_SESSION_RECORD_BYTES}-byte per-record bound.`
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
pending.push(Buffer.from(slice));
|
|
378
|
+
pendingBytes += slice.length;
|
|
379
|
+
}
|
|
380
|
+
|
|
247
381
|
function consumeRecord() {
|
|
248
382
|
records += 1;
|
|
249
383
|
if (records > limits.maxSessionRecords) {
|
|
250
384
|
fail('session-records-exceeded', `${describePath} exceeds maxSessionRecords.`);
|
|
251
385
|
}
|
|
252
|
-
|
|
386
|
+
// One buffer is the common case (a record that arrived inside a single read); concatenating
|
|
387
|
+
// only when it spanned reads keeps the extra copy off the normal path.
|
|
388
|
+
const raw = pending.length === 1 ? pending[0] : Buffer.concat(pending, pendingBytes);
|
|
253
389
|
pending = [];
|
|
254
390
|
pendingBytes = 0;
|
|
391
|
+
const line = raw.toString('utf8');
|
|
255
392
|
if (line.trim().length === 0) return;
|
|
256
393
|
let parsed;
|
|
257
394
|
try {
|
|
@@ -273,15 +410,11 @@ function streamSessionJsonl(fd, describePath, limits) {
|
|
|
273
410
|
let start = 0;
|
|
274
411
|
for (let i = 0; i < chunk.length; i += 1) {
|
|
275
412
|
if (chunk[i] !== NEWLINE) continue;
|
|
276
|
-
|
|
277
|
-
pendingBytes += i - start;
|
|
413
|
+
appendPending(chunk.subarray(start, i));
|
|
278
414
|
start = i + 1;
|
|
279
415
|
consumeRecord();
|
|
280
416
|
}
|
|
281
|
-
if (start < chunk.length)
|
|
282
|
-
pending.push(Buffer.from(chunk.subarray(start)));
|
|
283
|
-
pendingBytes += chunk.length - start;
|
|
284
|
-
}
|
|
417
|
+
if (start < chunk.length) appendPending(chunk.subarray(start));
|
|
285
418
|
hash.update(chunk);
|
|
286
419
|
},
|
|
287
420
|
() => fail('session-bytes-exceeded', `${describePath} observed bytes exceed maxSessionBytes.`)
|
|
@@ -360,32 +493,44 @@ function verifyBlobReferences(blobRefs, limits, blobsDirOptions) {
|
|
|
360
493
|
}
|
|
361
494
|
|
|
362
495
|
/** Walk the partition subtree (everything except the session file itself), descriptor-pinning
|
|
363
|
-
* every entry, and return the manifest entries in a deterministic order.
|
|
364
|
-
|
|
496
|
+
* every entry, and return the manifest entries in a deterministic order. Relative paths are raw
|
|
497
|
+
* bytes throughout — assembled from the bytes the directory reported, bounded by their own byte
|
|
498
|
+
* length, and never re-encoded from a decoded string. */
|
|
499
|
+
function collectArtifactEntries(partitionRef, partitionIdentity, sessionFileName, limits) {
|
|
365
500
|
const entries = [];
|
|
366
501
|
const aggregate = { count: 0, bytes: 0 };
|
|
502
|
+
const sessionFileNameBytes = Buffer.from(sessionFileName, 'utf8');
|
|
367
503
|
|
|
368
|
-
function visit(
|
|
504
|
+
function visit(absDirRef, dirIdentity, relDir, depth) {
|
|
369
505
|
if (depth > limits.maxArtifactDepth) {
|
|
370
|
-
fail(
|
|
506
|
+
fail(
|
|
507
|
+
'artifact-depth-exceeded',
|
|
508
|
+
`Artifact tree exceeds maxArtifactDepth at ${relDir ? describeRef(relDir) : '.'}.`
|
|
509
|
+
);
|
|
371
510
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
511
|
+
// The most names this level could hold without the entry budget already being blown. The
|
|
512
|
+
// session file is skipped rather than counted, so depth 0 may hold exactly one more.
|
|
513
|
+
const maxNames = limits.maxArtifactEntries - aggregate.count + (depth === 0 ? 1 : 0);
|
|
514
|
+
for (const child of readdirPinned(absDirRef, dirIdentity, maxNames)) {
|
|
515
|
+
if (depth === 0 && child.name.equals(sessionFileNameBytes)) continue;
|
|
516
|
+
assertDirectChildNameBytes(child.name);
|
|
517
|
+
const absPath = joinChildRef(absDirRef, child.name);
|
|
518
|
+
const relPath = relDir ? Buffer.concat([relDir, MANIFEST_SEPARATOR, child.name]) : child.name;
|
|
519
|
+
if (relPath.length > limits.maxRelativePathBytes) {
|
|
520
|
+
fail(
|
|
521
|
+
'relative-path-bytes-exceeded',
|
|
522
|
+
`${describeRef(relPath)} exceeds maxRelativePathBytes.`
|
|
523
|
+
);
|
|
379
524
|
}
|
|
380
525
|
aggregate.count += 1;
|
|
381
526
|
if (aggregate.count > limits.maxArtifactEntries) {
|
|
382
527
|
fail('artifact-entries-exceeded', 'Artifact tree exceeds maxArtifactEntries.');
|
|
383
528
|
}
|
|
384
529
|
|
|
385
|
-
if (child.
|
|
386
|
-
fail('symlink-rejected', `${relPath} is a symlink.`);
|
|
530
|
+
if (child.symlink) {
|
|
531
|
+
fail('symlink-rejected', `${describeRef(relPath)} is a symlink.`);
|
|
387
532
|
}
|
|
388
|
-
if (child.
|
|
533
|
+
if (child.directory) {
|
|
389
534
|
const childIdentity = withPinned(
|
|
390
535
|
absPath,
|
|
391
536
|
{ directory: true, missingCode: 'artifact-missing', notTypeCode: 'not-a-directory' },
|
|
@@ -395,10 +540,10 @@ function collectArtifactEntries(partitionPath, partitionIdentity, sessionFileNam
|
|
|
395
540
|
visit(absPath, childIdentity, relPath, depth + 1);
|
|
396
541
|
continue;
|
|
397
542
|
}
|
|
398
|
-
if (!child.
|
|
543
|
+
if (!child.file) {
|
|
399
544
|
fail(
|
|
400
545
|
'not-a-regular-single-link-file',
|
|
401
|
-
`${relPath} is neither a regular file nor a directory.`
|
|
546
|
+
`${describeRef(relPath)} is neither a regular file nor a directory.`
|
|
402
547
|
);
|
|
403
548
|
}
|
|
404
549
|
|
|
@@ -407,17 +552,27 @@ function collectArtifactEntries(partitionPath, partitionIdentity, sessionFileNam
|
|
|
407
552
|
{ missingCode: 'artifact-missing', notTypeCode: 'not-a-regular-single-link-file' },
|
|
408
553
|
({ fd, stat }) => {
|
|
409
554
|
if (stat.nlink > 1) {
|
|
410
|
-
fail(
|
|
555
|
+
fail(
|
|
556
|
+
'not-a-regular-single-link-file',
|
|
557
|
+
`${describeRef(relPath)} has more than one hard link.`
|
|
558
|
+
);
|
|
411
559
|
}
|
|
412
560
|
if (stat.size > limits.maxArtifactFileBytes) {
|
|
413
|
-
fail(
|
|
561
|
+
fail(
|
|
562
|
+
'artifact-file-bytes-exceeded',
|
|
563
|
+
`${describeRef(relPath)} exceeds maxArtifactFileBytes.`
|
|
564
|
+
);
|
|
414
565
|
}
|
|
415
566
|
const hash = createHash('sha256');
|
|
416
567
|
const observed = streamDescriptor(
|
|
417
568
|
fd,
|
|
418
569
|
limits.maxArtifactFileBytes,
|
|
419
570
|
(chunk) => hash.update(chunk),
|
|
420
|
-
() =>
|
|
571
|
+
() =>
|
|
572
|
+
fail(
|
|
573
|
+
'artifact-file-bytes-exceeded',
|
|
574
|
+
`${describeRef(relPath)} exceeds maxArtifactFileBytes.`
|
|
575
|
+
)
|
|
421
576
|
);
|
|
422
577
|
return {
|
|
423
578
|
relPath,
|
|
@@ -430,24 +585,32 @@ function collectArtifactEntries(partitionPath, partitionIdentity, sessionFileNam
|
|
|
430
585
|
|
|
431
586
|
aggregate.bytes += entry.size;
|
|
432
587
|
if (aggregate.bytes > limits.maxArtifactAggregateBytes) {
|
|
433
|
-
fail(
|
|
588
|
+
fail(
|
|
589
|
+
'artifact-aggregate-bytes-exceeded',
|
|
590
|
+
'Artifact tree exceeds maxArtifactAggregateBytes.'
|
|
591
|
+
);
|
|
434
592
|
}
|
|
435
593
|
entries.push(entry);
|
|
436
594
|
}
|
|
437
595
|
}
|
|
438
596
|
|
|
439
|
-
visit(
|
|
597
|
+
visit(partitionRef, partitionIdentity, null, 0);
|
|
440
598
|
return entries;
|
|
441
599
|
}
|
|
442
600
|
|
|
601
|
+
/** `<uint32 big-endian byte length><bytes>`. Length-prefixing is what makes the manifest
|
|
602
|
+
* unambiguous: without it `a/b` + `c` and `a` + `b/c` would hash identically. */
|
|
443
603
|
function lengthPrefixed(value) {
|
|
444
|
-
const buf = Buffer.from(String(value), 'utf8');
|
|
604
|
+
const buf = Buffer.isBuffer(value) ? value : Buffer.from(String(value), 'utf8');
|
|
445
605
|
const len = Buffer.alloc(4);
|
|
446
606
|
len.writeUInt32BE(buf.length, 0);
|
|
447
607
|
return Buffer.concat([len, buf]);
|
|
448
608
|
}
|
|
449
609
|
|
|
450
610
|
function hashManifestEntry(hash, { relPath, type, size, contentDigest }) {
|
|
611
|
+
// relPath is hashed as the raw bytes the filesystem reported. Two files whose names are both
|
|
612
|
+
// invalid UTF-8 decode to the same replacement-character string, so hashing the decoded form
|
|
613
|
+
// would give distinct trees identical manifests.
|
|
451
614
|
hash.update(lengthPrefixed(relPath));
|
|
452
615
|
hash.update(lengthPrefixed(type));
|
|
453
616
|
hash.update(lengthPrefixed(size));
|
|
@@ -472,8 +635,9 @@ function verifyPartitionContents(
|
|
|
472
635
|
) {
|
|
473
636
|
assertSessionFileName(sessionFileName);
|
|
474
637
|
|
|
638
|
+
const partitionRef = toPathRef(partitionPath);
|
|
475
639
|
const partitionIdentity = withPinned(
|
|
476
|
-
|
|
640
|
+
partitionRef,
|
|
477
641
|
{ directory: true, missingCode: 'partition-missing', notTypeCode: 'not-a-directory' },
|
|
478
642
|
({ stat }) => identityOf(stat)
|
|
479
643
|
);
|
|
@@ -486,7 +650,7 @@ function verifyPartitionContents(
|
|
|
486
650
|
|
|
487
651
|
const sessionFilePath = path.join(partitionPath, sessionFileName);
|
|
488
652
|
const session = withPinned(
|
|
489
|
-
|
|
653
|
+
joinChildRef(partitionRef, Buffer.from(sessionFileName, 'utf8')),
|
|
490
654
|
{ missingCode: 'session-file-missing', notTypeCode: 'not-a-regular-file' },
|
|
491
655
|
({ fd, stat }) => {
|
|
492
656
|
if (stat.nlink > 1) {
|
|
@@ -505,7 +669,7 @@ function verifyPartitionContents(
|
|
|
505
669
|
const header = parseSessionHeader(session.header, sessionFilePath);
|
|
506
670
|
const blobs = verifyBlobReferences(session.blobRefs, limits, blobsDirOptions);
|
|
507
671
|
const artifactEntries = collectArtifactEntries(
|
|
508
|
-
|
|
672
|
+
partitionRef,
|
|
509
673
|
partitionIdentity,
|
|
510
674
|
sessionFileName,
|
|
511
675
|
limits
|
|
@@ -541,7 +705,7 @@ function verifyPartitionContents(
|
|
|
541
705
|
* this point in a fresh session's lifecycle. */
|
|
542
706
|
function checkPartitionPathReady(partitionPath, { expectedPartitionIdentity = null } = {}) {
|
|
543
707
|
const identity = withPinned(
|
|
544
|
-
partitionPath,
|
|
708
|
+
toPathRef(partitionPath),
|
|
545
709
|
{ directory: true, missingCode: 'partition-missing', notTypeCode: 'not-a-directory' },
|
|
546
710
|
({ stat }) => identityOf(stat)
|
|
547
711
|
);
|