@byline/core 6.12.1 → 6.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/@types/extraction-types.d.ts +423 -0
  2. package/dist/@types/extraction-types.js +8 -0
  3. package/dist/@types/index.d.ts +1 -0
  4. package/dist/@types/index.js +1 -0
  5. package/dist/extraction/artifact.d.ts +46 -0
  6. package/dist/extraction/artifact.js +121 -0
  7. package/dist/extraction/artifact.test.node.d.ts +8 -0
  8. package/dist/extraction/artifact.test.node.js +194 -0
  9. package/dist/extraction/errors.d.ts +47 -0
  10. package/dist/extraction/errors.js +57 -0
  11. package/dist/extraction/errors.test.node.d.ts +8 -0
  12. package/dist/extraction/errors.test.node.js +68 -0
  13. package/dist/extraction/evidence.d.ts +22 -0
  14. package/dist/extraction/evidence.js +54 -0
  15. package/dist/extraction/evidence.test.node.d.ts +8 -0
  16. package/dist/extraction/evidence.test.node.js +138 -0
  17. package/dist/extraction/index.d.ts +14 -0
  18. package/dist/extraction/index.js +17 -0
  19. package/dist/extraction/internal.d.ts +30 -0
  20. package/dist/extraction/internal.js +93 -0
  21. package/dist/extraction/outcomes.d.ts +20 -0
  22. package/dist/extraction/outcomes.js +30 -0
  23. package/dist/extraction/outcomes.test.node.d.ts +8 -0
  24. package/dist/extraction/outcomes.test.node.js +55 -0
  25. package/dist/extraction/page-map.d.ts +40 -0
  26. package/dist/extraction/page-map.js +190 -0
  27. package/dist/extraction/page-map.test.node.d.ts +8 -0
  28. package/dist/extraction/page-map.test.node.js +371 -0
  29. package/dist/extraction/producer.d.ts +62 -0
  30. package/dist/extraction/producer.js +76 -0
  31. package/dist/extraction/producer.test.node.d.ts +8 -0
  32. package/dist/extraction/producer.test.node.js +230 -0
  33. package/dist/extraction/root-graph-guard.test.node.d.ts +8 -0
  34. package/dist/extraction/root-graph-guard.test.node.js +293 -0
  35. package/dist/extraction/work.d.ts +55 -0
  36. package/dist/extraction/work.js +102 -0
  37. package/dist/extraction/work.test.node.d.ts +8 -0
  38. package/dist/extraction/work.test.node.js +409 -0
  39. package/dist/index.d.ts +1 -0
  40. package/dist/index.js +1 -0
  41. package/dist/storage/stored-files-server.d.ts +1 -0
  42. package/dist/storage/stored-files-server.js +9 -24
  43. package/dist/storage/stored-files-server.test.node.js +6 -0
  44. package/dist/storage/verified-read.d.ts +131 -0
  45. package/dist/storage/verified-read.js +328 -0
  46. package/dist/storage/verified-read.test.node.d.ts +8 -0
  47. package/dist/storage/verified-read.test.node.js +645 -0
  48. package/package.json +2 -2
@@ -0,0 +1,645 @@
1
+ /**
2
+ * This Source Code is subject to the terms of the Mozilla Public
3
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
4
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
+ *
6
+ * Copyright (c) Infonomic Company Limited
7
+ */
8
+ import { createHash } from 'node:crypto';
9
+ import { mkdtempSync, readdirSync, rmSync, statSync } from 'node:fs';
10
+ import { tmpdir } from 'node:os';
11
+ import { join } from 'node:path';
12
+ import { Readable } from 'node:stream';
13
+ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
14
+ import { isSourceVerificationEvidence } from '../extraction/evidence.js';
15
+ import { ERR_STORAGE, ERR_STORAGE_PRECONDITION } from '../lib/errors.js';
16
+ import { probeVerifiedSource, readVerifiedSource, VERIFIED_SOURCE_SPOOL_PREFIX, } from './verified-read.js';
17
+ const BYTES = Buffer.from('the verified source bytes');
18
+ const HASH = createHash('sha256').update(BYTES).digest('hex');
19
+ const OTHER_BYTES = Buffer.from('different source bytes');
20
+ const OTHER_HASH = createHash('sha256').update(OTHER_BYTES).digest('hex');
21
+ const SAVED = { kind: 'test-etag', value: 'saved' };
22
+ const CURRENT = { kind: 'test-etag', value: 'current' };
23
+ const precondition = (reason) => ERR_STORAGE_PRECONDITION({ message: `precondition: ${reason}`, details: { reason } });
24
+ /** A provider whose `download` follows a script, recording every call. */
25
+ function scripted(steps) {
26
+ const calls = [];
27
+ const head = vi.fn();
28
+ const provider = {
29
+ providerName: 'test',
30
+ upload: () => Promise.reject(new Error('never writes')),
31
+ delete: () => Promise.reject(new Error('never writes')),
32
+ getUrl: () => '',
33
+ head,
34
+ download: async (_path, options) => {
35
+ calls.push(options);
36
+ const step = steps[calls.length - 1];
37
+ if (step === undefined)
38
+ throw new Error(`unexpected read ${calls.length}`);
39
+ if ('reject' in step)
40
+ throw step.reject;
41
+ if ('none' in step)
42
+ return null;
43
+ const { bytes, ...read } = step.read;
44
+ return {
45
+ body: read.body ?? Readable.from(bytes === undefined ? [] : [bytes]),
46
+ validator: read.validator ?? CURRENT,
47
+ ...(read.size !== undefined && { size: read.size }),
48
+ };
49
+ },
50
+ };
51
+ return { provider, calls, head };
52
+ }
53
+ const source = (change) => ({
54
+ storagePath: 'files/a.pdf',
55
+ storageProvider: 'test',
56
+ fileHash: HASH,
57
+ storageValidator: SAVED,
58
+ ...change,
59
+ });
60
+ const readOf = (bytes = BYTES) => ({ read: { bytes } });
61
+ describe('readVerifiedSource', () => {
62
+ let spoolDirectory;
63
+ const spoolFiles = () => readdirSync(spoolDirectory).filter((name) => name.startsWith(VERIFIED_SOURCE_SPOOL_PREFIX));
64
+ beforeEach(() => {
65
+ spoolDirectory = mkdtempSync(join(tmpdir(), 'byline-verified-read-test-'));
66
+ vi.stubEnv('TMPDIR', spoolDirectory);
67
+ });
68
+ afterEach(() => {
69
+ vi.unstubAllEnvs();
70
+ rmSync(spoolDirectory, { recursive: true, force: true });
71
+ });
72
+ describe('rule 1: no fileHash', () => {
73
+ it.each([
74
+ ['absent', undefined],
75
+ ['empty', ''],
76
+ ])('reports %s as missing-hash without any read', async (_label, fileHash) => {
77
+ const { provider, calls } = scripted([]);
78
+ const result = await readVerifiedSource(provider, source({ fileHash }));
79
+ expect(result).toEqual({
80
+ status: 'identity-problem',
81
+ problem: 'missing-hash',
82
+ storagePath: 'files/a.pdf',
83
+ });
84
+ expect(calls).toEqual([]);
85
+ });
86
+ });
87
+ describe('rule 2: provider', () => {
88
+ it('reports a value stored by another provider', async () => {
89
+ const { provider, calls } = scripted([]);
90
+ const result = await readVerifiedSource(provider, source({ storageProvider: 'other' }));
91
+ expect(result).toEqual({ status: 'unavailable', reason: 'provider-mismatch' });
92
+ expect(calls).toEqual([]);
93
+ });
94
+ it('reports a provider with no download', async () => {
95
+ const { provider } = scripted([]);
96
+ delete provider.download;
97
+ expect(await readVerifiedSource(provider, source())).toEqual({
98
+ status: 'unavailable',
99
+ reason: 'download-unsupported',
100
+ });
101
+ });
102
+ });
103
+ describe('rule 3: the first read', () => {
104
+ it('expects the saved validator', async () => {
105
+ const { provider, calls } = scripted([readOf()]);
106
+ const result = await readVerifiedSource(provider, source());
107
+ expect(calls[0]?.expect).toEqual(SAVED);
108
+ expect(result).toMatchObject({ status: 'verified' });
109
+ if (result.status === 'verified') {
110
+ expect(result.evidence.readBasis).toBe('expected');
111
+ await result.body.dispose();
112
+ }
113
+ });
114
+ it.each([
115
+ ['no saved validator', { storageValidator: undefined }, undefined],
116
+ ['knownChanged', {}, { knownChanged: true }],
117
+ ])('reads without an expectation with %s', async (_label, change, options) => {
118
+ const { provider, calls } = scripted([readOf()]);
119
+ const result = await readVerifiedSource(provider, source(change), options);
120
+ expect(calls[0]?.expect).toBeUndefined();
121
+ expect(result).toMatchObject({
122
+ status: 'verified',
123
+ evidence: { readBasis: 'unconditional' },
124
+ });
125
+ expect(result.status === 'verified' && 'preconditionFallback' in result.evidence).toBe(false);
126
+ if (result.status === 'verified')
127
+ await result.body.dispose();
128
+ });
129
+ it('passes the signal to the read', async () => {
130
+ const { provider, calls } = scripted([readOf()]);
131
+ const { signal } = new AbortController();
132
+ const result = await readVerifiedSource(provider, source(), { signal });
133
+ expect(calls[0]?.signal).toBe(signal);
134
+ if (result.status === 'verified')
135
+ await result.body.dispose();
136
+ });
137
+ });
138
+ describe('rule 4: missing objects', () => {
139
+ it('reports a missing precondition without a second read', async () => {
140
+ const { provider, calls } = scripted([{ reject: precondition('missing') }]);
141
+ expect(await readVerifiedSource(provider, source())).toEqual({
142
+ status: 'identity-problem',
143
+ problem: 'missing-object',
144
+ storagePath: 'files/a.pdf',
145
+ expectedHash: HASH,
146
+ });
147
+ expect(calls).toHaveLength(1);
148
+ });
149
+ it('reports a null result', async () => {
150
+ const { provider } = scripted([{ none: true }]);
151
+ expect(await readVerifiedSource(provider, source())).toMatchObject({
152
+ status: 'identity-problem',
153
+ problem: 'missing-object',
154
+ });
155
+ });
156
+ it('never invents an observed hash', async () => {
157
+ const { provider } = scripted([{ none: true }]);
158
+ const result = await readVerifiedSource(provider, source());
159
+ expect(result).not.toHaveProperty('observedHash');
160
+ expect(result).not.toHaveProperty('observedValidator');
161
+ });
162
+ });
163
+ describe('rule 5: the single fallback read', () => {
164
+ it.each(['changed', 'unsupported-kind', 'unestablishable'])('falls back once on %s, keeping the reason', async (reason) => {
165
+ const { provider, calls } = scripted([{ reject: precondition(reason) }, readOf()]);
166
+ const result = await readVerifiedSource(provider, source());
167
+ expect(calls).toHaveLength(2);
168
+ expect(calls[1]?.expect).toBeUndefined();
169
+ expect(result).toMatchObject({
170
+ status: 'verified',
171
+ evidence: { readBasis: 'fallback', preconditionFallback: reason },
172
+ });
173
+ if (result.status === 'verified')
174
+ await result.body.dispose();
175
+ });
176
+ it('treats an unrecognised reason as unestablishable', async () => {
177
+ const { provider } = scripted([{ reject: precondition('made-up') }, readOf()]);
178
+ const result = await readVerifiedSource(provider, source());
179
+ expect(result).toMatchObject({
180
+ status: 'verified',
181
+ evidence: { preconditionFallback: 'unestablishable' },
182
+ });
183
+ if (result.status === 'verified')
184
+ await result.body.dispose();
185
+ });
186
+ it('makes exactly two reads when both hit a precondition', async () => {
187
+ const cause = precondition('changed');
188
+ const { provider, calls } = scripted([{ reject: precondition('changed') }, { reject: cause }]);
189
+ expect(await readVerifiedSource(provider, source())).toEqual({
190
+ status: 'unavailable',
191
+ reason: 'read-failed',
192
+ cause,
193
+ });
194
+ expect(calls).toHaveLength(2);
195
+ });
196
+ it('reports missing on the fallback with the fallback reason', async () => {
197
+ const { provider } = scripted([
198
+ { reject: precondition('changed') },
199
+ { reject: precondition('missing') },
200
+ ]);
201
+ expect(await readVerifiedSource(provider, source())).toMatchObject({
202
+ status: 'identity-problem',
203
+ problem: 'missing-object',
204
+ preconditionFallback: 'changed',
205
+ });
206
+ });
207
+ it('reports null on the fallback as missing-object', async () => {
208
+ const { provider, calls } = scripted([{ reject: precondition('changed') }, { none: true }]);
209
+ expect(await readVerifiedSource(provider, source())).toMatchObject({
210
+ status: 'identity-problem',
211
+ problem: 'missing-object',
212
+ preconditionFallback: 'changed',
213
+ });
214
+ expect(calls).toHaveLength(2);
215
+ });
216
+ it('reports any other fallback failure as read-failed', async () => {
217
+ const cause = new Error('socket closed');
218
+ const { provider } = scripted([{ reject: precondition('changed') }, { reject: cause }]);
219
+ expect(await readVerifiedSource(provider, source())).toEqual({
220
+ status: 'unavailable',
221
+ reason: 'read-failed',
222
+ cause,
223
+ });
224
+ });
225
+ });
226
+ describe('rule 6: other failures', () => {
227
+ it('reports read-failed with the original error, without a second read', async () => {
228
+ const cause = ERR_STORAGE({ message: 'no validator', details: { reason: 'x' } });
229
+ const { provider, calls } = scripted([{ reject: cause }]);
230
+ expect(await readVerifiedSource(provider, source())).toEqual({
231
+ status: 'unavailable',
232
+ reason: 'read-failed',
233
+ cause,
234
+ });
235
+ expect(calls).toHaveLength(1);
236
+ });
237
+ it('reports aborted when the signal fired', async () => {
238
+ const controller = new AbortController();
239
+ const cause = new Error('aborted by the provider');
240
+ const { provider } = scripted([]);
241
+ provider.download = async () => {
242
+ controller.abort();
243
+ throw cause;
244
+ };
245
+ expect(await readVerifiedSource(provider, source(), { signal: controller.signal })).toEqual({
246
+ status: 'unavailable',
247
+ reason: 'aborted',
248
+ cause,
249
+ });
250
+ });
251
+ it('does not read when the signal has already fired', async () => {
252
+ const controller = new AbortController();
253
+ controller.abort();
254
+ const { provider, calls } = scripted([]);
255
+ expect(await readVerifiedSource(provider, source(), { signal: controller.signal })).toMatchObject({ status: 'unavailable', reason: 'aborted' });
256
+ expect(calls).toEqual([]);
257
+ });
258
+ });
259
+ describe('rule 7: complete bodies only', () => {
260
+ function failingBody(after, error) {
261
+ return Readable.from((async function* () {
262
+ yield after;
263
+ throw error;
264
+ })());
265
+ }
266
+ it.each([
267
+ ['in memory', undefined],
268
+ ['while spooling', { memoryThreshold: 0 }],
269
+ ])('reports a body that errors midway %s and leaves no spool', async (_label, options) => {
270
+ const cause = new Error('connection reset');
271
+ const { provider } = scripted([{ read: { body: failingBody(BYTES, cause) } }]);
272
+ const result = await readVerifiedSource(provider, source(), options);
273
+ expect(result).toEqual({ status: 'unavailable', reason: 'read-failed', cause });
274
+ expect(spoolFiles()).toEqual([]);
275
+ });
276
+ it('reports a body shorter than its reported size as unavailable, not a mismatch', async () => {
277
+ const { provider } = scripted([{ read: { bytes: BYTES, size: BYTES.length + 1 } }]);
278
+ const result = await readVerifiedSource(provider, source());
279
+ expect(result).toMatchObject({ status: 'unavailable', reason: 'read-failed' });
280
+ expect(result).not.toHaveProperty('observedHash');
281
+ });
282
+ it.each([
283
+ ['in memory', undefined],
284
+ ['while spooling', { memoryThreshold: 0 }],
285
+ ])('reports an abort during the body %s and leaves no spool', async (_label, options) => {
286
+ const controller = new AbortController();
287
+ // One chunk, then it waits like a real transfer. A body that pushed from every
288
+ // read() would keep the event loop busy, and the abort below might never run.
289
+ let pushed = false;
290
+ const body = new Readable({
291
+ read() {
292
+ if (pushed)
293
+ return;
294
+ pushed = true;
295
+ this.push(BYTES);
296
+ setImmediate(() => controller.abort());
297
+ },
298
+ });
299
+ const { provider } = scripted([{ read: { body } }]);
300
+ const result = await readVerifiedSource(provider, source(), {
301
+ ...options,
302
+ signal: controller.signal,
303
+ });
304
+ expect(result).toMatchObject({ status: 'unavailable', reason: 'aborted' });
305
+ expect(body.destroyed).toBe(true);
306
+ expect(spoolFiles()).toEqual([]);
307
+ });
308
+ });
309
+ describe('an abort that fires as the read completes', () => {
310
+ it.each([
311
+ ['in memory', undefined],
312
+ ['while spooling', { memoryThreshold: 0 }],
313
+ ])('reports an abort at the end of the body %s', async (_label, options) => {
314
+ const controller = new AbortController();
315
+ const body = Readable.from([BYTES]);
316
+ body.on('end', () => controller.abort());
317
+ const { provider } = scripted([{ read: { body } }]);
318
+ const result = await readVerifiedSource(provider, source(), {
319
+ ...options,
320
+ signal: controller.signal,
321
+ });
322
+ expect(result).toMatchObject({ status: 'unavailable', reason: 'aborted' });
323
+ expect(result).not.toHaveProperty('body');
324
+ expect(spoolFiles()).toEqual([]);
325
+ });
326
+ it.each([
327
+ ['in memory', undefined],
328
+ ['while spooling', { memoryThreshold: 0 }],
329
+ ])('reports an abort during finalization %s', async (_label, options) => {
330
+ const controller = new AbortController();
331
+ // The validator is first read after the body ends, so this fires after the
332
+ // end-of-body check and before the body is published.
333
+ const { provider } = scripted([readOf()]);
334
+ const download = provider.download?.bind(provider);
335
+ provider.download = async (path, downloadOptions) => {
336
+ const result = await download?.(path, downloadOptions);
337
+ return result == null
338
+ ? null
339
+ : Object.defineProperty(result, 'validator', {
340
+ get() {
341
+ controller.abort();
342
+ return CURRENT;
343
+ },
344
+ });
345
+ };
346
+ const result = await readVerifiedSource(provider, source(), {
347
+ ...options,
348
+ signal: controller.signal,
349
+ });
350
+ expect(result).toMatchObject({ status: 'unavailable', reason: 'aborted' });
351
+ expect(result).not.toHaveProperty('body');
352
+ expect(spoolFiles()).toEqual([]);
353
+ });
354
+ });
355
+ describe('a provider that reuses its chunk buffer', () => {
356
+ const expectedBytes = Buffer.from('AB');
357
+ const expectedHash = createHash('sha256').update(expectedBytes).digest('hex');
358
+ /** Yields one buffer twice, changing it in between, as a pooling stream might. */
359
+ function reusingBody() {
360
+ const buffer = Buffer.from('A');
361
+ return Readable.from((async function* () {
362
+ yield buffer;
363
+ await new Promise((resolve) => setImmediate(resolve));
364
+ buffer[0] = 0x42;
365
+ yield buffer;
366
+ })());
367
+ }
368
+ it.each([
369
+ ['in memory', undefined],
370
+ ['crossing the threshold', { memoryThreshold: 1 }],
371
+ ['spooled from the start', { memoryThreshold: 0 }],
372
+ ])('returns the bytes it hashed %s', async (_label, options) => {
373
+ const { provider } = scripted([{ read: { body: reusingBody() } }]);
374
+ const result = await readVerifiedSource(provider, source({ fileHash: expectedHash }), options);
375
+ if (result.status !== 'verified')
376
+ throw new Error(`expected verified: ${result.status}`);
377
+ const bytes = Buffer.from(await result.body.bytes());
378
+ expect(bytes.equals(expectedBytes)).toBe(true);
379
+ expect(createHash('sha256').update(bytes).digest('hex')).toBe(result.evidence.contentHash);
380
+ const chunks = [];
381
+ for await (const chunk of result.body.stream()) {
382
+ chunks.push(chunk);
383
+ }
384
+ expect(createHash('sha256').update(Buffer.concat(chunks)).digest('hex')).toBe(result.evidence.contentHash);
385
+ await result.body.dispose();
386
+ });
387
+ });
388
+ describe('rule 8: a different hash', () => {
389
+ it.each([
390
+ ['in memory', undefined],
391
+ ['while spooling', { memoryThreshold: 0 }],
392
+ ])('reports %s what was observed and exposes nothing', async (_label, options) => {
393
+ const { provider } = scripted([{ read: { bytes: OTHER_BYTES } }]);
394
+ const result = await readVerifiedSource(provider, source(), options);
395
+ expect(result).toEqual({
396
+ status: 'identity-problem',
397
+ problem: 'hash-mismatch',
398
+ storagePath: 'files/a.pdf',
399
+ expectedHash: HASH,
400
+ observedHash: OTHER_HASH,
401
+ observedValidator: CURRENT,
402
+ });
403
+ expect(result).not.toHaveProperty('body');
404
+ expect(spoolFiles()).toEqual([]);
405
+ });
406
+ it('keeps the fallback reason', async () => {
407
+ const { provider } = scripted([
408
+ { reject: precondition('changed') },
409
+ { read: { bytes: OTHER_BYTES } },
410
+ ]);
411
+ expect(await readVerifiedSource(provider, source())).toMatchObject({
412
+ problem: 'hash-mismatch',
413
+ preconditionFallback: 'changed',
414
+ });
415
+ });
416
+ });
417
+ describe('rule 9: evidence', () => {
418
+ it('binds the validator from this read, never the saved one', async () => {
419
+ const { provider } = scripted([readOf()]);
420
+ const result = await readVerifiedSource(provider, source());
421
+ expect(result.status).toBe('verified');
422
+ if (result.status !== 'verified')
423
+ return;
424
+ expect(result.evidence.validator).toEqual(CURRENT);
425
+ expect(result.evidence.expectedHash).toBe(HASH);
426
+ expect(result.evidence.contentHash).toBe(HASH);
427
+ expect(result.evidence.size).toBe(BYTES.length);
428
+ expect(isSourceVerificationEvidence(result.evidence, { expectedHash: HASH })).toBe(true);
429
+ await result.body.dispose();
430
+ });
431
+ it('reports a provider whose validator is unusable instead of writing bad evidence', async () => {
432
+ const { provider } = scripted([
433
+ { read: { bytes: BYTES, validator: { kind: '', value: '' } } },
434
+ ]);
435
+ expect(await readVerifiedSource(provider, source())).toMatchObject({
436
+ status: 'unavailable',
437
+ reason: 'read-failed',
438
+ });
439
+ expect(spoolFiles()).toEqual([]);
440
+ });
441
+ });
442
+ describe('rule 10: read-only', () => {
443
+ it('never writes to storage or changes the saved value', async () => {
444
+ const value = source();
445
+ const snapshot = structuredClone(value);
446
+ const { provider } = scripted([{ reject: precondition('changed') }, readOf()]);
447
+ const result = await readVerifiedSource(provider, value);
448
+ expect(value).toEqual(snapshot);
449
+ if (result.status === 'verified')
450
+ await result.body.dispose();
451
+ });
452
+ });
453
+ describe('the verified body', () => {
454
+ const collect = async (stream) => {
455
+ const chunks = [];
456
+ for await (const chunk of stream)
457
+ chunks.push(chunk);
458
+ return Buffer.concat(chunks);
459
+ };
460
+ it('stays in memory at or below the threshold', async () => {
461
+ const { provider } = scripted([readOf()]);
462
+ const result = await readVerifiedSource(provider, source(), {
463
+ memoryThreshold: BYTES.length,
464
+ });
465
+ expect(result.status).toBe('verified');
466
+ if (result.status !== 'verified')
467
+ return;
468
+ expect(spoolFiles()).toEqual([]);
469
+ expect(result.body.size).toBe(BYTES.length);
470
+ expect(Buffer.from(await result.body.bytes()).equals(BYTES)).toBe(true);
471
+ await result.body.dispose();
472
+ });
473
+ it('spools above the threshold to a private file', async () => {
474
+ const { provider } = scripted([readOf()]);
475
+ const result = await readVerifiedSource(provider, source(), {
476
+ memoryThreshold: BYTES.length - 1,
477
+ });
478
+ expect(result.status).toBe('verified');
479
+ if (result.status !== 'verified')
480
+ return;
481
+ const [spool] = spoolFiles();
482
+ expect(spool).toBeDefined();
483
+ expect(statSync(join(spoolDirectory, spool)).mode & 0o777).toBe(0o600);
484
+ expect(Buffer.from(await result.body.bytes()).equals(BYTES)).toBe(true);
485
+ await result.body.dispose();
486
+ });
487
+ it('crosses the threshold in the middle of a body', async () => {
488
+ const body = Readable.from([BYTES.subarray(0, 5), BYTES.subarray(5, 10), BYTES.subarray(10)]);
489
+ const { provider } = scripted([{ read: { body } }]);
490
+ const result = await readVerifiedSource(provider, source(), { memoryThreshold: 7 });
491
+ expect(result.status).toBe('verified');
492
+ if (result.status !== 'verified')
493
+ return;
494
+ expect(spoolFiles()).toHaveLength(1);
495
+ expect((await collect(result.body.stream())).equals(BYTES)).toBe(true);
496
+ await result.body.dispose();
497
+ });
498
+ it.each([
499
+ ['memory', undefined],
500
+ ['a spool', { memoryThreshold: 0 }],
501
+ ])('opens more than one stream over %s', async (_label, options) => {
502
+ const { provider } = scripted([readOf()]);
503
+ const result = await readVerifiedSource(provider, source(), options);
504
+ expect(result.status).toBe('verified');
505
+ if (result.status !== 'verified')
506
+ return;
507
+ expect((await collect(result.body.stream())).equals(BYTES)).toBe(true);
508
+ expect((await collect(result.body.stream())).equals(BYTES)).toBe(true);
509
+ await result.body.dispose();
510
+ });
511
+ it('gives a caller a copy, so verified bytes cannot be changed in place', async () => {
512
+ const { provider } = scripted([readOf()]);
513
+ const result = await readVerifiedSource(provider, source());
514
+ if (result.status !== 'verified')
515
+ throw new Error('expected verified');
516
+ const first = await result.body.bytes();
517
+ first.fill(0);
518
+ expect(Buffer.from(await result.body.bytes()).equals(BYTES)).toBe(true);
519
+ await result.body.dispose();
520
+ });
521
+ it('gives each stream a copy, so a provider cannot change verified bytes in place', async () => {
522
+ const { provider } = scripted([readOf()]);
523
+ const result = await readVerifiedSource(provider, source());
524
+ if (result.status !== 'verified')
525
+ throw new Error('expected verified');
526
+ const reader = result.body.stream().getReader();
527
+ const { value } = await reader.read();
528
+ value?.fill(0);
529
+ expect((await collect(result.body.stream())).equals(BYTES)).toBe(true);
530
+ await result.body.dispose();
531
+ });
532
+ it('verifies an empty object', async () => {
533
+ const empty = Buffer.alloc(0);
534
+ const emptyHash = createHash('sha256').update(empty).digest('hex');
535
+ const { provider } = scripted([{ read: { body: Readable.from([]) } }]);
536
+ const result = await readVerifiedSource(provider, source({ fileHash: emptyHash }));
537
+ expect(result.status).toBe('verified');
538
+ if (result.status !== 'verified')
539
+ return;
540
+ expect(result.body.size).toBe(0);
541
+ expect((await collect(result.body.stream())).length).toBe(0);
542
+ await result.body.dispose();
543
+ });
544
+ it('removes the spool on dispose, and dispose is idempotent', async () => {
545
+ const { provider } = scripted([readOf()]);
546
+ const result = await readVerifiedSource(provider, source(), { memoryThreshold: 0 });
547
+ if (result.status !== 'verified')
548
+ throw new Error('expected verified');
549
+ expect(spoolFiles()).toHaveLength(1);
550
+ await result.body.dispose();
551
+ await result.body.dispose();
552
+ expect(spoolFiles()).toEqual([]);
553
+ });
554
+ it.each([
555
+ ['memory', undefined],
556
+ ['a spool', { memoryThreshold: 0 }],
557
+ ])('refuses to read %s after dispose', async (_label, options) => {
558
+ const { provider } = scripted([readOf()]);
559
+ const result = await readVerifiedSource(provider, source(), options);
560
+ if (result.status !== 'verified')
561
+ throw new Error('expected verified');
562
+ await result.body.dispose();
563
+ expect(() => result.body.stream()).toThrow(/disposed/);
564
+ await expect(result.body.bytes()).rejects.toThrow(/disposed/);
565
+ });
566
+ it.each([-1, 1.5, Number.NaN])('rejects a memoryThreshold of %s', async (memoryThreshold) => {
567
+ const { provider, calls } = scripted([]);
568
+ await expect(readVerifiedSource(provider, source(), { memoryThreshold })).rejects.toThrow(TypeError);
569
+ expect(calls).toEqual([]);
570
+ });
571
+ });
572
+ });
573
+ describe('probeVerifiedSource', () => {
574
+ const evidence = {
575
+ expectedHash: HASH,
576
+ contentHash: HASH,
577
+ validator: CURRENT,
578
+ verifiedAt: '2026-11-04T00:00:00.000Z',
579
+ };
580
+ it('reports unchanged against the evidence validator', async () => {
581
+ const { provider, head } = scripted([]);
582
+ head.mockResolvedValue({ validator: CURRENT });
583
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
584
+ status: 'unchanged',
585
+ validator: CURRENT,
586
+ });
587
+ });
588
+ it('reports a different value as changed', async () => {
589
+ const { provider, head } = scripted([]);
590
+ const other = { kind: 'test-etag', value: 'other' };
591
+ head.mockResolvedValue({ validator: other });
592
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
593
+ status: 'changed',
594
+ validator: other,
595
+ });
596
+ });
597
+ it('reports a different kind as changed', async () => {
598
+ const { provider, head } = scripted([]);
599
+ const other = { kind: 'other-kind', value: 'current' };
600
+ head.mockResolvedValue({ validator: other });
601
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
602
+ status: 'changed',
603
+ validator: other,
604
+ });
605
+ });
606
+ it('reports a deleted object', async () => {
607
+ const { provider, head } = scripted([]);
608
+ head.mockResolvedValue(null);
609
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
610
+ status: 'missing-object',
611
+ });
612
+ });
613
+ it('reports a failing probe as unavailable, with its cause', async () => {
614
+ const { provider, head } = scripted([]);
615
+ const cause = new Error('head failed');
616
+ head.mockRejectedValue(cause);
617
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
618
+ status: 'unavailable',
619
+ cause,
620
+ });
621
+ });
622
+ it('reports a provider without head as unprobed', async () => {
623
+ const { provider } = scripted([]);
624
+ delete provider.head;
625
+ expect(await probeVerifiedSource(provider, 'a', evidence)).toEqual({
626
+ status: 'unprobed',
627
+ reason: 'head-unsupported',
628
+ });
629
+ });
630
+ it('does not call head without evidence, so a copied value cannot stand in', async () => {
631
+ const { provider, head } = scripted([]);
632
+ expect(await probeVerifiedSource(provider, 'a', null)).toEqual({
633
+ status: 'unprobed',
634
+ reason: 'no-evidence',
635
+ });
636
+ expect(head).not.toHaveBeenCalled();
637
+ });
638
+ it('treats malformed evidence as no evidence', async () => {
639
+ const { provider, head } = scripted([]);
640
+ expect(await probeVerifiedSource(provider, 'a', {
641
+ expectedHash: HASH,
642
+ })).toEqual({ status: 'unprobed', reason: 'no-evidence' });
643
+ expect(head).not.toHaveBeenCalled();
644
+ });
645
+ });
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@byline/core",
3
3
  "private": false,
4
4
  "license": "MPL-2.0",
5
- "version": "6.12.1",
5
+ "version": "6.13.0",
6
6
  "engines": {
7
7
  "node": ">=22.22.2"
8
8
  },
@@ -87,7 +87,7 @@
87
87
  "dist"
88
88
  ],
89
89
  "dependencies": {
90
- "@byline/auth": "6.12.1",
90
+ "@byline/auth": "6.13.0",
91
91
  "dotenv": "^18.0.9",
92
92
  "pino": "^10.4.0",
93
93
  "sharp": "^0.35.5",