@byline/core 6.12.0 → 6.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/@types/extraction-types.d.ts +423 -0
- package/dist/@types/extraction-types.js +8 -0
- package/dist/@types/index.d.ts +1 -0
- package/dist/@types/index.js +1 -0
- package/dist/extraction/artifact.d.ts +46 -0
- package/dist/extraction/artifact.js +121 -0
- package/dist/extraction/artifact.test.node.d.ts +8 -0
- package/dist/extraction/artifact.test.node.js +194 -0
- package/dist/extraction/errors.d.ts +47 -0
- package/dist/extraction/errors.js +57 -0
- package/dist/extraction/errors.test.node.d.ts +8 -0
- package/dist/extraction/errors.test.node.js +68 -0
- package/dist/extraction/evidence.d.ts +22 -0
- package/dist/extraction/evidence.js +54 -0
- package/dist/extraction/evidence.test.node.d.ts +8 -0
- package/dist/extraction/evidence.test.node.js +138 -0
- package/dist/extraction/index.d.ts +14 -0
- package/dist/extraction/index.js +17 -0
- package/dist/extraction/internal.d.ts +30 -0
- package/dist/extraction/internal.js +93 -0
- package/dist/extraction/outcomes.d.ts +20 -0
- package/dist/extraction/outcomes.js +30 -0
- package/dist/extraction/outcomes.test.node.d.ts +8 -0
- package/dist/extraction/outcomes.test.node.js +55 -0
- package/dist/extraction/page-map.d.ts +40 -0
- package/dist/extraction/page-map.js +190 -0
- package/dist/extraction/page-map.test.node.d.ts +8 -0
- package/dist/extraction/page-map.test.node.js +371 -0
- package/dist/extraction/producer.d.ts +62 -0
- package/dist/extraction/producer.js +76 -0
- package/dist/extraction/producer.test.node.d.ts +8 -0
- package/dist/extraction/producer.test.node.js +230 -0
- package/dist/extraction/root-graph-guard.test.node.d.ts +8 -0
- package/dist/extraction/root-graph-guard.test.node.js +293 -0
- package/dist/extraction/work.d.ts +55 -0
- package/dist/extraction/work.js +102 -0
- package/dist/extraction/work.test.node.d.ts +8 -0
- package/dist/extraction/work.test.node.js +409 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/storage/stored-files-server.d.ts +1 -0
- package/dist/storage/stored-files-server.js +9 -24
- package/dist/storage/stored-files-server.test.node.js +6 -0
- package/dist/storage/verified-read.d.ts +131 -0
- package/dist/storage/verified-read.js +328 -0
- package/dist/storage/verified-read.test.node.d.ts +8 -0
- package/dist/storage/verified-read.test.node.js +645 -0
- package/package.json +6 -6
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { validateExtractionArtifact } from './artifact.js';
|
|
10
|
+
const HASH = 'a'.repeat(64);
|
|
11
|
+
const CONFIG_HASH = 'b'.repeat(64);
|
|
12
|
+
function validArtifact() {
|
|
13
|
+
return {
|
|
14
|
+
schemaVersion: 1,
|
|
15
|
+
contentHash: HASH,
|
|
16
|
+
sourceVerified: true,
|
|
17
|
+
provider: 'tika',
|
|
18
|
+
providerVersion: '3.0.0',
|
|
19
|
+
producerConfigHash: CONFIG_HASH,
|
|
20
|
+
plainText: 'alpha',
|
|
21
|
+
metadata: {},
|
|
22
|
+
signals: { totalChars: 5 },
|
|
23
|
+
producedAt: '2026-11-04T00:00:00.000Z',
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function codes(value, options) {
|
|
27
|
+
const result = validateExtractionArtifact(value, options);
|
|
28
|
+
return result.ok ? [] : result.issues.map((issue) => issue.code);
|
|
29
|
+
}
|
|
30
|
+
describe('validateExtractionArtifact', () => {
|
|
31
|
+
it('accepts a complete artifact', () => {
|
|
32
|
+
expect(validateExtractionArtifact(validArtifact())).toEqual({ ok: true });
|
|
33
|
+
});
|
|
34
|
+
it('accepts the optional members', () => {
|
|
35
|
+
expect(validateExtractionArtifact({
|
|
36
|
+
...validArtifact(),
|
|
37
|
+
providerVersion: null,
|
|
38
|
+
markdown: '# alpha',
|
|
39
|
+
structure: { any: ['json'] },
|
|
40
|
+
pageCount: 1,
|
|
41
|
+
pageMap: {
|
|
42
|
+
version: 1,
|
|
43
|
+
assembly: { pageSeparator: '' },
|
|
44
|
+
pages: [{ index: 1, charStart: 0, charEnd: 5 }],
|
|
45
|
+
},
|
|
46
|
+
})).toEqual({ ok: true });
|
|
47
|
+
});
|
|
48
|
+
it.each([
|
|
49
|
+
['null', null],
|
|
50
|
+
['an array', []],
|
|
51
|
+
['a string', 'artifact'],
|
|
52
|
+
])('rejects %s', (_label, value) => {
|
|
53
|
+
expect(codes(value)).toContain('invalid-shape');
|
|
54
|
+
});
|
|
55
|
+
it('reports an unknown schema version as unsupported', () => {
|
|
56
|
+
expect(codes({ ...validArtifact(), schemaVersion: 2 })).toEqual(['unsupported-schema-version']);
|
|
57
|
+
});
|
|
58
|
+
it.each([
|
|
59
|
+
['a short content hash', { contentHash: 'abc' }, 'invalid-content-hash'],
|
|
60
|
+
['an upper-case content hash', { contentHash: 'A'.repeat(64) }, 'invalid-content-hash'],
|
|
61
|
+
['an empty content hash', { contentHash: '' }, 'invalid-content-hash'],
|
|
62
|
+
['a malformed configuration hash', { producerConfigHash: 'x' }, 'invalid-producer-config-hash'],
|
|
63
|
+
['an empty provider', { provider: '' }, 'invalid-provider'],
|
|
64
|
+
['a numeric provider version', { providerVersion: 3 }, 'invalid-provider-version'],
|
|
65
|
+
['absent plain text', { plainText: undefined }, 'invalid-plain-text'],
|
|
66
|
+
['non-string markdown', { markdown: 1 }, 'invalid-markdown'],
|
|
67
|
+
['a negative page count', { pageCount: -1 }, 'invalid-page-count'],
|
|
68
|
+
['a fractional page count', { pageCount: 1.5 }, 'invalid-page-count'],
|
|
69
|
+
['absent metadata', { metadata: undefined }, 'invalid-metadata'],
|
|
70
|
+
['absent signals', { signals: undefined }, 'invalid-signals'],
|
|
71
|
+
['a negative total', { signals: { totalChars: -1 } }, 'invalid-signals'],
|
|
72
|
+
['a date without a time', { producedAt: '2026-11-04' }, 'invalid-produced-at'],
|
|
73
|
+
['an unparseable date', { producedAt: 'yesterday' }, 'invalid-produced-at'],
|
|
74
|
+
['a non-boolean verified flag', { sourceVerified: 'yes' }, 'source-verified-mismatch'],
|
|
75
|
+
])('rejects %s', (_label, change, code) => {
|
|
76
|
+
expect(codes({ ...validArtifact(), ...change })).toContain(code);
|
|
77
|
+
});
|
|
78
|
+
it('requires sourceVerified to agree with contentHash', () => {
|
|
79
|
+
expect(codes({ ...validArtifact(), sourceVerified: false })).toContain('source-verified-mismatch');
|
|
80
|
+
expect(codes({ ...validArtifact(), contentHash: null })).toContain('source-verified-mismatch');
|
|
81
|
+
});
|
|
82
|
+
it('accepts a legacy import with no source identity', () => {
|
|
83
|
+
expect(validateExtractionArtifact({ ...validArtifact(), contentHash: null, sourceVerified: false })).toEqual({ ok: true });
|
|
84
|
+
});
|
|
85
|
+
it('reports page map issues against the artifact text and page count', () => {
|
|
86
|
+
const artifact = {
|
|
87
|
+
...validArtifact(),
|
|
88
|
+
pageCount: 2,
|
|
89
|
+
pageMap: {
|
|
90
|
+
version: 1,
|
|
91
|
+
assembly: { pageSeparator: '' },
|
|
92
|
+
pages: [{ index: 1, charStart: 0, charEnd: 5 }],
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
expect(codes(artifact)).toContain('invalid-page-map');
|
|
96
|
+
});
|
|
97
|
+
describe('structure', () => {
|
|
98
|
+
it('accepts any JSON value, because it is opaque to Byline', () => {
|
|
99
|
+
for (const structure of [null, 1, 'text', true, [1, { a: [] }], { a: { b: null } }]) {
|
|
100
|
+
expect(validateExtractionArtifact({ ...validArtifact(), structure })).toEqual({ ok: true });
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
const cycle = {};
|
|
104
|
+
cycle.self = cycle;
|
|
105
|
+
it.each([
|
|
106
|
+
['a bigint', 1n],
|
|
107
|
+
['a nested function', { a: () => 1 }],
|
|
108
|
+
['NaN', [Number.NaN]],
|
|
109
|
+
['a Date', new Date(0)],
|
|
110
|
+
['a sparse array', new Array(1)],
|
|
111
|
+
['an undefined member', { value: undefined }],
|
|
112
|
+
['a nested undefined member', { section: { value: undefined } }],
|
|
113
|
+
['an undefined member inside an array', [{ value: undefined }]],
|
|
114
|
+
['a cycle', cycle],
|
|
115
|
+
])('rejects %s', (_label, structure) => {
|
|
116
|
+
expect(codes({ ...validArtifact(), structure })).toContain('invalid-structure');
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
describe('metadata members', () => {
|
|
120
|
+
it('accepts recognised members and ignores unknown ones', () => {
|
|
121
|
+
expect(validateExtractionArtifact({
|
|
122
|
+
...validArtifact(),
|
|
123
|
+
metadata: {
|
|
124
|
+
contentType: 'application/pdf',
|
|
125
|
+
pages: 3,
|
|
126
|
+
title: 'T',
|
|
127
|
+
authors: ['A', 'B'],
|
|
128
|
+
language: 'en',
|
|
129
|
+
createdAt: '2026-01-01T00:00:00Z',
|
|
130
|
+
producer: 'P',
|
|
131
|
+
somethingElse: { any: 1 },
|
|
132
|
+
},
|
|
133
|
+
})).toEqual({ ok: true });
|
|
134
|
+
});
|
|
135
|
+
it.each([
|
|
136
|
+
['non-string authors', { authors: 42 }],
|
|
137
|
+
['an author that is not a string', { authors: ['A', 1] }],
|
|
138
|
+
['a hole in authors', { authors: new Array(1) }],
|
|
139
|
+
['non-numeric pages', { pages: 'bad' }],
|
|
140
|
+
['negative pages', { pages: -1 }],
|
|
141
|
+
['a non-string title', { title: 1 }],
|
|
142
|
+
['a non-string content type', { contentType: 1 }],
|
|
143
|
+
['a non-string language', { language: 1 }],
|
|
144
|
+
['a non-string creation date', { createdAt: 1 }],
|
|
145
|
+
['a creation date that is not ISO 8601', { createdAt: 'not-a-date' }],
|
|
146
|
+
['a raw PDF creation date', { createdAt: "D:20260101000000+07'00'" }],
|
|
147
|
+
['a non-string producer', { producer: 1 }],
|
|
148
|
+
])('rejects %s', (_label, metadata) => {
|
|
149
|
+
expect(codes({ ...validArtifact(), metadata })).toContain('invalid-metadata');
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
describe('signals members', () => {
|
|
153
|
+
it('accepts recognised members and ignores unknown ones', () => {
|
|
154
|
+
expect(validateExtractionArtifact({
|
|
155
|
+
...validArtifact(),
|
|
156
|
+
signals: {
|
|
157
|
+
totalChars: 5,
|
|
158
|
+
charsPerPage: [2, 3],
|
|
159
|
+
ocrCandidatePages: 0,
|
|
160
|
+
textLayerCoverage: 1,
|
|
161
|
+
somethingElse: true,
|
|
162
|
+
},
|
|
163
|
+
})).toEqual({ ok: true });
|
|
164
|
+
});
|
|
165
|
+
it.each([
|
|
166
|
+
['non-array charsPerPage', { charsPerPage: 'bad' }],
|
|
167
|
+
['a negative page count in charsPerPage', { charsPerPage: [-1] }],
|
|
168
|
+
['a hole in charsPerPage', { charsPerPage: new Array(1) }],
|
|
169
|
+
['a fractional OCR candidate count', { ocrCandidatePages: 1.5 }],
|
|
170
|
+
['coverage above one', { textLayerCoverage: 2 }],
|
|
171
|
+
['coverage below zero', { textLayerCoverage: -0.1 }],
|
|
172
|
+
['non-numeric coverage', { textLayerCoverage: '1' }],
|
|
173
|
+
])('rejects %s', (_label, change) => {
|
|
174
|
+
expect(codes({ ...validArtifact(), signals: { totalChars: 5, ...change } })).toContain('invalid-signals');
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
describe('an unrecorded producer configuration', () => {
|
|
178
|
+
const historical = { ...validArtifact(), providerVersion: null, producerConfigHash: null };
|
|
179
|
+
it('is rejected by default, so a new extraction cannot record null', () => {
|
|
180
|
+
expect(codes(historical)).toEqual(['unrecorded-producer']);
|
|
181
|
+
});
|
|
182
|
+
it('is rejected when the option is false', () => {
|
|
183
|
+
expect(codes(historical, { allowUnrecordedProducer: false })).toEqual(['unrecorded-producer']);
|
|
184
|
+
});
|
|
185
|
+
it('is accepted when reading a historical record', () => {
|
|
186
|
+
expect(validateExtractionArtifact(historical, { allowUnrecordedProducer: true })).toEqual({
|
|
187
|
+
ok: true,
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
it('does not relax a malformed configuration hash', () => {
|
|
191
|
+
expect(codes({ ...validArtifact(), producerConfigHash: 'x' }, { allowUnrecordedProducer: true })).toContain('invalid-producer-config-hash');
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import type { ExtractionFailureScope } from '../@types/extraction-types.js';
|
|
9
|
+
/**
|
|
10
|
+
* A typed provider failure.
|
|
11
|
+
*
|
|
12
|
+
* - `file`: the input caused it (corrupt, encrypted, unsupported format).
|
|
13
|
+
* Orchestration may record a per-file failure for this source and producer.
|
|
14
|
+
* - `infrastructure`: the service failed (timeout, unavailable, bad credentials).
|
|
15
|
+
* Orchestration must not record the file as bad and must preserve existing
|
|
16
|
+
* artifacts.
|
|
17
|
+
*
|
|
18
|
+
* `retryable` is independent policy: bad credentials are `infrastructure` but not
|
|
19
|
+
* usefully retryable until repaired.
|
|
20
|
+
*
|
|
21
|
+
* @provisional
|
|
22
|
+
*/
|
|
23
|
+
export declare class ExtractionProviderError extends Error {
|
|
24
|
+
readonly code = "ERR_EXTRACTION_PROVIDER";
|
|
25
|
+
readonly failureScope: ExtractionFailureScope;
|
|
26
|
+
readonly retryable: boolean;
|
|
27
|
+
constructor(message: string, options: {
|
|
28
|
+
failureScope: ExtractionFailureScope;
|
|
29
|
+
retryable: boolean;
|
|
30
|
+
cause?: unknown;
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Whether a value is a typed provider failure. It checks the `code` brand rather
|
|
35
|
+
* than `instanceof`, so an error thrown by a second copy of `@byline/core` still
|
|
36
|
+
* classifies.
|
|
37
|
+
*
|
|
38
|
+
* @provisional
|
|
39
|
+
*/
|
|
40
|
+
export declare function isExtractionProviderError(error: unknown): error is ExtractionProviderError;
|
|
41
|
+
/**
|
|
42
|
+
* Whether a value is a typed `file` failure. An untyped error or an abort is never
|
|
43
|
+
* evidence that the file is bad.
|
|
44
|
+
*
|
|
45
|
+
* @provisional
|
|
46
|
+
*/
|
|
47
|
+
export declare function isFileExtractionFailure(error: unknown): error is ExtractionProviderError;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
const ERROR_CODE = 'ERR_EXTRACTION_PROVIDER';
|
|
9
|
+
/**
|
|
10
|
+
* A typed provider failure.
|
|
11
|
+
*
|
|
12
|
+
* - `file`: the input caused it (corrupt, encrypted, unsupported format).
|
|
13
|
+
* Orchestration may record a per-file failure for this source and producer.
|
|
14
|
+
* - `infrastructure`: the service failed (timeout, unavailable, bad credentials).
|
|
15
|
+
* Orchestration must not record the file as bad and must preserve existing
|
|
16
|
+
* artifacts.
|
|
17
|
+
*
|
|
18
|
+
* `retryable` is independent policy: bad credentials are `infrastructure` but not
|
|
19
|
+
* usefully retryable until repaired.
|
|
20
|
+
*
|
|
21
|
+
* @provisional
|
|
22
|
+
*/
|
|
23
|
+
export class ExtractionProviderError extends Error {
|
|
24
|
+
code = ERROR_CODE;
|
|
25
|
+
failureScope;
|
|
26
|
+
retryable;
|
|
27
|
+
constructor(message, options) {
|
|
28
|
+
super(message, options.cause === undefined ? undefined : { cause: options.cause });
|
|
29
|
+
this.name = 'ExtractionProviderError';
|
|
30
|
+
this.failureScope = options.failureScope;
|
|
31
|
+
this.retryable = options.retryable;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Whether a value is a typed provider failure. It checks the `code` brand rather
|
|
36
|
+
* than `instanceof`, so an error thrown by a second copy of `@byline/core` still
|
|
37
|
+
* classifies.
|
|
38
|
+
*
|
|
39
|
+
* @provisional
|
|
40
|
+
*/
|
|
41
|
+
export function isExtractionProviderError(error) {
|
|
42
|
+
if (typeof error !== 'object' || error === null)
|
|
43
|
+
return false;
|
|
44
|
+
const candidate = error;
|
|
45
|
+
return (candidate.code === ERROR_CODE &&
|
|
46
|
+
(candidate.failureScope === 'file' || candidate.failureScope === 'infrastructure') &&
|
|
47
|
+
typeof candidate.retryable === 'boolean');
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Whether a value is a typed `file` failure. An untyped error or an abort is never
|
|
51
|
+
* evidence that the file is bad.
|
|
52
|
+
*
|
|
53
|
+
* @provisional
|
|
54
|
+
*/
|
|
55
|
+
export function isFileExtractionFailure(error) {
|
|
56
|
+
return isExtractionProviderError(error) && error.failureScope === 'file';
|
|
57
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { ExtractionProviderError, isExtractionProviderError, isFileExtractionFailure, } from './errors.js';
|
|
10
|
+
function fromAnotherCopy(failureScope) {
|
|
11
|
+
// Stands in for an error thrown by a second installed copy of @byline/core:
|
|
12
|
+
// the same shape and brand, but a different class, so `instanceof` is false.
|
|
13
|
+
class OtherExtractionProviderError extends Error {
|
|
14
|
+
code = 'ERR_EXTRACTION_PROVIDER';
|
|
15
|
+
failureScope = failureScope;
|
|
16
|
+
retryable = false;
|
|
17
|
+
}
|
|
18
|
+
return new OtherExtractionProviderError('from another copy');
|
|
19
|
+
}
|
|
20
|
+
describe('ExtractionProviderError', () => {
|
|
21
|
+
it('carries its brand, scope, retry policy and cause', () => {
|
|
22
|
+
const cause = new Error('socket closed');
|
|
23
|
+
const error = new ExtractionProviderError('service failed', {
|
|
24
|
+
failureScope: 'infrastructure',
|
|
25
|
+
retryable: true,
|
|
26
|
+
cause,
|
|
27
|
+
});
|
|
28
|
+
expect(error).toBeInstanceOf(Error);
|
|
29
|
+
expect(error.name).toBe('ExtractionProviderError');
|
|
30
|
+
expect(error.code).toBe('ERR_EXTRACTION_PROVIDER');
|
|
31
|
+
expect(error.failureScope).toBe('infrastructure');
|
|
32
|
+
expect(error.retryable).toBe(true);
|
|
33
|
+
expect(error.cause).toBe(cause);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
describe('isExtractionProviderError', () => {
|
|
37
|
+
it('accepts a typed error from this copy and from another copy', () => {
|
|
38
|
+
const typed = new ExtractionProviderError('bad', { failureScope: 'file', retryable: false });
|
|
39
|
+
expect(isExtractionProviderError(typed)).toBe(true);
|
|
40
|
+
expect(isExtractionProviderError(fromAnotherCopy('file'))).toBe(true);
|
|
41
|
+
});
|
|
42
|
+
it.each([
|
|
43
|
+
['an untyped error', new Error('boom')],
|
|
44
|
+
['an abort', new DOMException('aborted', 'AbortError')],
|
|
45
|
+
['a brand with an unknown scope', fromAnotherCopy('network')],
|
|
46
|
+
['a plain value', 'ERR_EXTRACTION_PROVIDER'],
|
|
47
|
+
['null', null],
|
|
48
|
+
])('rejects %s', (_label, value) => {
|
|
49
|
+
expect(isExtractionProviderError(value)).toBe(false);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
describe('isFileExtractionFailure', () => {
|
|
53
|
+
it('is true only for a typed file failure', () => {
|
|
54
|
+
expect(isFileExtractionFailure(new ExtractionProviderError('corrupt', { failureScope: 'file', retryable: false }))).toBe(true);
|
|
55
|
+
expect(isFileExtractionFailure(fromAnotherCopy('file'))).toBe(true);
|
|
56
|
+
});
|
|
57
|
+
it.each([
|
|
58
|
+
[
|
|
59
|
+
'an infrastructure failure',
|
|
60
|
+
new ExtractionProviderError('down', { failureScope: 'infrastructure', retryable: true }),
|
|
61
|
+
],
|
|
62
|
+
['an infrastructure failure from another copy', fromAnotherCopy('infrastructure')],
|
|
63
|
+
['an untyped error', new Error('corrupt')],
|
|
64
|
+
['an abort', new DOMException('aborted', 'AbortError')],
|
|
65
|
+
])('is false for %s', (_label, value) => {
|
|
66
|
+
expect(isFileExtractionFailure(value)).toBe(false);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import type { SourceVerificationEvidence } from '../@types/extraction-types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Whether a stored record is valid verification evidence, optionally for a given
|
|
11
|
+
* hash. Unknown extra members are ignored, so a later minor can add members without
|
|
12
|
+
* invalidating stored evidence.
|
|
13
|
+
*
|
|
14
|
+
* Records written before `readBasis` existed stay valid, with or without a fallback
|
|
15
|
+
* reason. The presence of a JSON key is not evidence, and a hash copied from the
|
|
16
|
+
* saved file value does not make a record valid.
|
|
17
|
+
*
|
|
18
|
+
* @provisional
|
|
19
|
+
*/
|
|
20
|
+
export declare function isSourceVerificationEvidence(value: unknown, options?: {
|
|
21
|
+
expectedHash?: string;
|
|
22
|
+
}): value is SourceVerificationEvidence;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { CONTENT_HASH_PATTERN, STORAGE_VALIDATOR_LIMITS } from '../storage/content-hash.js';
|
|
9
|
+
import { isIsoDateTime, isPlainObject } from './internal.js';
|
|
10
|
+
const isBoundedString = (value, limit) => typeof value === 'string' && value.length > 0 && value.length <= limit;
|
|
11
|
+
const FALLBACK_REASONS = ['changed', 'unsupported-kind', 'unestablishable'];
|
|
12
|
+
const READ_BASES = ['expected', 'unconditional', 'fallback'];
|
|
13
|
+
/**
|
|
14
|
+
* Whether a stored record is valid verification evidence, optionally for a given
|
|
15
|
+
* hash. Unknown extra members are ignored, so a later minor can add members without
|
|
16
|
+
* invalidating stored evidence.
|
|
17
|
+
*
|
|
18
|
+
* Records written before `readBasis` existed stay valid, with or without a fallback
|
|
19
|
+
* reason. The presence of a JSON key is not evidence, and a hash copied from the
|
|
20
|
+
* saved file value does not make a record valid.
|
|
21
|
+
*
|
|
22
|
+
* @provisional
|
|
23
|
+
*/
|
|
24
|
+
export function isSourceVerificationEvidence(value, options) {
|
|
25
|
+
if (!isPlainObject(value))
|
|
26
|
+
return false;
|
|
27
|
+
const { expectedHash, contentHash, validator, verifiedAt, size, readBasis, preconditionFallback, } = value;
|
|
28
|
+
if (typeof contentHash !== 'string' || !CONTENT_HASH_PATTERN.test(contentHash))
|
|
29
|
+
return false;
|
|
30
|
+
if (expectedHash !== contentHash)
|
|
31
|
+
return false;
|
|
32
|
+
if (options?.expectedHash !== undefined && expectedHash !== options.expectedHash)
|
|
33
|
+
return false;
|
|
34
|
+
if (!isPlainObject(validator) ||
|
|
35
|
+
!isBoundedString(validator.kind, STORAGE_VALIDATOR_LIMITS.kind) ||
|
|
36
|
+
!isBoundedString(validator.value, STORAGE_VALIDATOR_LIMITS.value)) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
if (!isIsoDateTime(verifiedAt))
|
|
40
|
+
return false;
|
|
41
|
+
if (size !== undefined && (!Number.isSafeInteger(size) || size < 0))
|
|
42
|
+
return false;
|
|
43
|
+
if (readBasis !== undefined && !READ_BASES.includes(readBasis))
|
|
44
|
+
return false;
|
|
45
|
+
if (preconditionFallback !== undefined && !FALLBACK_REASONS.includes(preconditionFallback)) {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
// A legacy record has no readBasis, and any fallback reason is valid on it.
|
|
49
|
+
if (readBasis === undefined)
|
|
50
|
+
return true;
|
|
51
|
+
return readBasis === 'fallback'
|
|
52
|
+
? preconditionFallback !== undefined
|
|
53
|
+
: preconditionFallback === undefined;
|
|
54
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { isSourceVerificationEvidence } from './evidence.js';
|
|
10
|
+
const HASH = 'a'.repeat(64);
|
|
11
|
+
/** The record a downstream library wrote before `size` and `readBasis` existed. */
|
|
12
|
+
function legacyRecord() {
|
|
13
|
+
return {
|
|
14
|
+
expectedHash: HASH,
|
|
15
|
+
contentHash: HASH,
|
|
16
|
+
validator: { kind: 's3-etag', value: '"abc"' },
|
|
17
|
+
verifiedAt: '2026-10-31T06:00:00.000Z',
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
describe('isSourceVerificationEvidence', () => {
|
|
21
|
+
describe('rule 1: plain objects', () => {
|
|
22
|
+
it.each([
|
|
23
|
+
['null', null],
|
|
24
|
+
['undefined', undefined],
|
|
25
|
+
['an array', []],
|
|
26
|
+
['a string', 'evidence'],
|
|
27
|
+
['a class instance', new (class Evidence {
|
|
28
|
+
})()],
|
|
29
|
+
])('rejects %s', (_label, value) => {
|
|
30
|
+
expect(isSourceVerificationEvidence(value)).toBe(false);
|
|
31
|
+
});
|
|
32
|
+
it('ignores unknown extra members, so a later minor can add some', () => {
|
|
33
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), futureMember: 1 })).toBe(true);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
describe('rule 2: hashes', () => {
|
|
37
|
+
it('accepts an existing record unchanged', () => {
|
|
38
|
+
expect(isSourceVerificationEvidence(legacyRecord())).toBe(true);
|
|
39
|
+
});
|
|
40
|
+
it.each([
|
|
41
|
+
['a malformed expected hash', { expectedHash: 'abc', contentHash: 'abc' }],
|
|
42
|
+
['an upper-case hash', { expectedHash: 'A'.repeat(64), contentHash: 'A'.repeat(64) }],
|
|
43
|
+
['an empty hash', { expectedHash: '', contentHash: '' }],
|
|
44
|
+
['hashes that differ', { contentHash: 'b'.repeat(64) }],
|
|
45
|
+
['a missing content hash', { contentHash: undefined }],
|
|
46
|
+
])('rejects %s', (_label, change) => {
|
|
47
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), ...change })).toBe(false);
|
|
48
|
+
});
|
|
49
|
+
it('rejects a copied-only record with nothing measured', () => {
|
|
50
|
+
expect(isSourceVerificationEvidence({ expectedHash: HASH })).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
it('checks the hash the caller expects', () => {
|
|
53
|
+
expect(isSourceVerificationEvidence(legacyRecord(), { expectedHash: HASH })).toBe(true);
|
|
54
|
+
expect(isSourceVerificationEvidence(legacyRecord(), { expectedHash: 'b'.repeat(64) })).toBe(false);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
describe('rule 3: validator', () => {
|
|
58
|
+
it.each([
|
|
59
|
+
['a missing validator', undefined],
|
|
60
|
+
['a string validator', 'etag'],
|
|
61
|
+
['a missing kind', { value: 'x' }],
|
|
62
|
+
['a missing value', { kind: 'x' }],
|
|
63
|
+
['an empty kind', { kind: '', value: 'x' }],
|
|
64
|
+
['an empty value', { kind: 'x', value: '' }],
|
|
65
|
+
['a kind over the limit', { kind: 'k'.repeat(33), value: 'x' }],
|
|
66
|
+
['a value over the limit', { kind: 'x', value: 'v'.repeat(1025) }],
|
|
67
|
+
])('rejects %s', (_label, validator) => {
|
|
68
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), validator })).toBe(false);
|
|
69
|
+
});
|
|
70
|
+
it('accepts values at the limit', () => {
|
|
71
|
+
expect(isSourceVerificationEvidence({
|
|
72
|
+
...legacyRecord(),
|
|
73
|
+
validator: { kind: 'k'.repeat(32), value: 'v'.repeat(1024) },
|
|
74
|
+
})).toBe(true);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
describe('rule 4: verifiedAt', () => {
|
|
78
|
+
it.each([
|
|
79
|
+
['a missing value', undefined],
|
|
80
|
+
['a number', 1_790_000_000],
|
|
81
|
+
['a date without a time', '2026-10-31'],
|
|
82
|
+
['text', 'yesterday'],
|
|
83
|
+
['an impossible date', '2026-13-45T00:00:00.000Z'],
|
|
84
|
+
])('rejects %s', (_label, verifiedAt) => {
|
|
85
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), verifiedAt })).toBe(false);
|
|
86
|
+
});
|
|
87
|
+
it('accepts an offset', () => {
|
|
88
|
+
expect(isSourceVerificationEvidence({
|
|
89
|
+
...legacyRecord(),
|
|
90
|
+
verifiedAt: '2026-10-31T13:00:00+07:00',
|
|
91
|
+
})).toBe(true);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
describe('rule 5: size', () => {
|
|
95
|
+
it.each([
|
|
96
|
+
['negative', -1],
|
|
97
|
+
['fractional', 1.5],
|
|
98
|
+
['a string', '10'],
|
|
99
|
+
['unsafe', Number.MAX_SAFE_INTEGER + 1],
|
|
100
|
+
])('rejects a size that is %s', (_label, size) => {
|
|
101
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), size })).toBe(false);
|
|
102
|
+
});
|
|
103
|
+
it('accepts zero', () => {
|
|
104
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), size: 0 })).toBe(true);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
describe('rules 6 and 7: known values', () => {
|
|
108
|
+
it('rejects an unknown readBasis', () => {
|
|
109
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), readBasis: 'guess' })).toBe(false);
|
|
110
|
+
});
|
|
111
|
+
it('rejects an unknown fallback reason', () => {
|
|
112
|
+
expect(isSourceVerificationEvidence({ ...legacyRecord(), preconditionFallback: 'other' })).toBe(false);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
describe('rule 8: readBasis and preconditionFallback', () => {
|
|
116
|
+
it.each([
|
|
117
|
+
// Legacy records written before readBasis existed.
|
|
118
|
+
[undefined, undefined, true],
|
|
119
|
+
[undefined, 'changed', true],
|
|
120
|
+
[undefined, 'unsupported-kind', true],
|
|
121
|
+
[undefined, 'unestablishable', true],
|
|
122
|
+
['fallback', 'changed', true],
|
|
123
|
+
['fallback', 'unsupported-kind', true],
|
|
124
|
+
['fallback', 'unestablishable', true],
|
|
125
|
+
['fallback', undefined, false],
|
|
126
|
+
['expected', undefined, true],
|
|
127
|
+
['unconditional', undefined, true],
|
|
128
|
+
['expected', 'changed', false],
|
|
129
|
+
['unconditional', 'unestablishable', false],
|
|
130
|
+
])('readBasis %s with fallback %s is %s', (readBasis, preconditionFallback, valid) => {
|
|
131
|
+
expect(isSourceVerificationEvidence({
|
|
132
|
+
...legacyRecord(),
|
|
133
|
+
...(readBasis != null && { readBasis }),
|
|
134
|
+
...(preconditionFallback != null && { preconditionFallback }),
|
|
135
|
+
})).toBe(valid);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export { type ExtractionArtifactIssue, type ExtractionArtifactIssueCode, validateExtractionArtifact, } from './artifact.js';
|
|
9
|
+
export { ExtractionProviderError, isExtractionProviderError, isFileExtractionFailure, } from './errors.js';
|
|
10
|
+
export { isSourceVerificationEvidence } from './evidence.js';
|
|
11
|
+
export { exclusionState } from './outcomes.js';
|
|
12
|
+
export { type PageMapIssue, type PageMapIssueCode, validatePageMap } from './page-map.js';
|
|
13
|
+
export { buildExtractionProducer, type ExtractionProducerIdentity, producerConfigHash, } from './producer.js';
|
|
14
|
+
export { decideExtractionWork, type ExistingExtraction } from './work.js';
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
// Browser-safe, provisional extraction contract helpers. The contract types live in
|
|
9
|
+
// `@types/extraction-types.ts`. The verified source read is server-only and lives in
|
|
10
|
+
// `@byline/core/stored-files`.
|
|
11
|
+
export { validateExtractionArtifact, } from './artifact.js';
|
|
12
|
+
export { ExtractionProviderError, isExtractionProviderError, isFileExtractionFailure, } from './errors.js';
|
|
13
|
+
export { isSourceVerificationEvidence } from './evidence.js';
|
|
14
|
+
export { exclusionState } from './outcomes.js';
|
|
15
|
+
export { validatePageMap } from './page-map.js';
|
|
16
|
+
export { buildExtractionProducer, producerConfigHash, } from './producer.js';
|
|
17
|
+
export { decideExtractionWork } from './work.js';
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
|
|
9
|
+
/** A string that both has the form of an ISO 8601 date-time and parses as one. */
|
|
10
|
+
export declare function isIsoDateTime(value: unknown): boolean;
|
|
11
|
+
export declare const isCount: (value: unknown) => boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Whether every slot of an array passes a check. It reads slots by index because
|
|
14
|
+
* `every` skips holes, and it stops at the first failure, so a very long sparse array
|
|
15
|
+
* cannot stall it.
|
|
16
|
+
*/
|
|
17
|
+
export declare function everySlot(value: unknown, check: (slot: unknown) => boolean): boolean;
|
|
18
|
+
/**
|
|
19
|
+
* Sorted keys and no insignificant whitespace, for hashing a configuration. `undefined`
|
|
20
|
+
* object members are dropped, so an unset option hashes like an absent one. Any other
|
|
21
|
+
* value JSON cannot represent exactly is rejected with a `TypeError` rather than coerced,
|
|
22
|
+
* because two different values must never canonicalise the same. That includes array
|
|
23
|
+
* holes and symbol keys.
|
|
24
|
+
*/
|
|
25
|
+
export declare function canonicalJson(value: unknown): string;
|
|
26
|
+
/**
|
|
27
|
+
* Whether a value is a strict `JsonValue`. Unlike `canonicalJson`, it rejects an
|
|
28
|
+
* `undefined` object member, which would be lost on persistence.
|
|
29
|
+
*/
|
|
30
|
+
export declare function isJsonValue(value: unknown): boolean;
|