@byline/core 6.12.1 → 6.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/@types/extraction-types.d.ts +423 -0
- package/dist/@types/extraction-types.js +8 -0
- package/dist/@types/index.d.ts +1 -0
- package/dist/@types/index.js +1 -0
- package/dist/extraction/artifact.d.ts +46 -0
- package/dist/extraction/artifact.js +121 -0
- package/dist/extraction/artifact.test.node.d.ts +8 -0
- package/dist/extraction/artifact.test.node.js +194 -0
- package/dist/extraction/errors.d.ts +47 -0
- package/dist/extraction/errors.js +57 -0
- package/dist/extraction/errors.test.node.d.ts +8 -0
- package/dist/extraction/errors.test.node.js +68 -0
- package/dist/extraction/evidence.d.ts +22 -0
- package/dist/extraction/evidence.js +54 -0
- package/dist/extraction/evidence.test.node.d.ts +8 -0
- package/dist/extraction/evidence.test.node.js +138 -0
- package/dist/extraction/index.d.ts +14 -0
- package/dist/extraction/index.js +17 -0
- package/dist/extraction/internal.d.ts +30 -0
- package/dist/extraction/internal.js +93 -0
- package/dist/extraction/outcomes.d.ts +20 -0
- package/dist/extraction/outcomes.js +30 -0
- package/dist/extraction/outcomes.test.node.d.ts +8 -0
- package/dist/extraction/outcomes.test.node.js +55 -0
- package/dist/extraction/page-map.d.ts +40 -0
- package/dist/extraction/page-map.js +190 -0
- package/dist/extraction/page-map.test.node.d.ts +8 -0
- package/dist/extraction/page-map.test.node.js +371 -0
- package/dist/extraction/producer.d.ts +62 -0
- package/dist/extraction/producer.js +76 -0
- package/dist/extraction/producer.test.node.d.ts +8 -0
- package/dist/extraction/producer.test.node.js +230 -0
- package/dist/extraction/root-graph-guard.test.node.d.ts +8 -0
- package/dist/extraction/root-graph-guard.test.node.js +293 -0
- package/dist/extraction/work.d.ts +55 -0
- package/dist/extraction/work.js +102 -0
- package/dist/extraction/work.test.node.d.ts +8 -0
- package/dist/extraction/work.test.node.js +409 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/storage/stored-files-server.d.ts +1 -0
- package/dist/storage/stored-files-server.js +9 -24
- package/dist/storage/stored-files-server.test.node.js +6 -0
- package/dist/storage/verified-read.d.ts +131 -0
- package/dist/storage/verified-read.js +328 -0
- package/dist/storage/verified-read.test.node.d.ts +8 -0
- package/dist/storage/verified-read.test.node.js +645 -0
- package/package.json +2 -2
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { CONTENT_HASH_PATTERN, sha256Hex } from '../storage/content-hash.js';
|
|
10
|
+
import { buildExtractionProducer, producerConfigHash } from './producer.js';
|
|
11
|
+
const encoder = new TextEncoder();
|
|
12
|
+
describe('producerConfigHash', () => {
|
|
13
|
+
it('hashes the canonical JSON of its inputs as lower-case SHA-256', async () => {
|
|
14
|
+
const hash = await producerConfigHash({
|
|
15
|
+
provider: 'tika',
|
|
16
|
+
configuration: { ocr: true, dpi: 300 },
|
|
17
|
+
hints: { language: 'en', filename: 'a.pdf' },
|
|
18
|
+
outputAffectingHints: ['language'],
|
|
19
|
+
configurationVersion: 'v1',
|
|
20
|
+
deploymentIdentifier: 'image-a',
|
|
21
|
+
});
|
|
22
|
+
const expected = '{"configuration":{"dpi":300,"ocr":true},"configurationVersion":"v1",' +
|
|
23
|
+
'"deploymentIdentifier":"image-a","hints":{"language":"en"},"provider":"tika"}';
|
|
24
|
+
expect(hash).toBe(await sha256Hex(encoder.encode(expected)));
|
|
25
|
+
expect(hash).toMatch(CONTENT_HASH_PATTERN);
|
|
26
|
+
});
|
|
27
|
+
it('does not depend on key order at any depth', async () => {
|
|
28
|
+
const a = await producerConfigHash({
|
|
29
|
+
provider: 'p',
|
|
30
|
+
configuration: { b: 1, a: { d: [1, { y: 2, x: 1 }], c: null } },
|
|
31
|
+
});
|
|
32
|
+
const b = await producerConfigHash({
|
|
33
|
+
provider: 'p',
|
|
34
|
+
configuration: { a: { c: null, d: [1, { x: 1, y: 2 }] }, b: 1 },
|
|
35
|
+
});
|
|
36
|
+
expect(a).toBe(b);
|
|
37
|
+
});
|
|
38
|
+
it('keeps array order significant', async () => {
|
|
39
|
+
const a = await producerConfigHash({ provider: 'p', configuration: { langs: ['th', 'en'] } });
|
|
40
|
+
const b = await producerConfigHash({ provider: 'p', configuration: { langs: ['en', 'th'] } });
|
|
41
|
+
expect(a).not.toBe(b);
|
|
42
|
+
});
|
|
43
|
+
it('drops undefined object members', async () => {
|
|
44
|
+
const a = await producerConfigHash({
|
|
45
|
+
provider: 'p',
|
|
46
|
+
configuration: { a: 1, b: undefined },
|
|
47
|
+
});
|
|
48
|
+
const b = await producerConfigHash({ provider: 'p', configuration: { a: 1 } });
|
|
49
|
+
expect(a).toBe(b);
|
|
50
|
+
});
|
|
51
|
+
it('drops nested undefined members, unlike strict JSON validation of artifact structure', async () => {
|
|
52
|
+
const a = await producerConfigHash({
|
|
53
|
+
provider: 'p',
|
|
54
|
+
configuration: { section: { a: 1, b: undefined } },
|
|
55
|
+
});
|
|
56
|
+
const b = await producerConfigHash({ provider: 'p', configuration: { section: { a: 1 } } });
|
|
57
|
+
expect(a).toBe(b);
|
|
58
|
+
});
|
|
59
|
+
it('keeps only the output-affecting hints that were supplied', async () => {
|
|
60
|
+
const withFilename = await producerConfigHash({
|
|
61
|
+
provider: 'p',
|
|
62
|
+
hints: { language: 'en', filename: 'a.pdf' },
|
|
63
|
+
outputAffectingHints: ['language', 'ocrLanguages'],
|
|
64
|
+
});
|
|
65
|
+
const withoutFilename = await producerConfigHash({
|
|
66
|
+
provider: 'p',
|
|
67
|
+
hints: { language: 'en' },
|
|
68
|
+
outputAffectingHints: ['language', 'ocrLanguages'],
|
|
69
|
+
});
|
|
70
|
+
const otherLanguage = await producerConfigHash({
|
|
71
|
+
provider: 'p',
|
|
72
|
+
hints: { language: 'th' },
|
|
73
|
+
outputAffectingHints: ['language'],
|
|
74
|
+
});
|
|
75
|
+
expect(withFilename).toBe(withoutFilename);
|
|
76
|
+
expect(otherLanguage).not.toBe(withoutFilename);
|
|
77
|
+
});
|
|
78
|
+
it('ignores every hint when the provider declares none', async () => {
|
|
79
|
+
const a = await producerConfigHash({ provider: 'p', hints: { language: 'en' } });
|
|
80
|
+
const b = await producerConfigHash({ provider: 'p' });
|
|
81
|
+
expect(a).toBe(b);
|
|
82
|
+
});
|
|
83
|
+
it.each([
|
|
84
|
+
['provider', { provider: 'q' }],
|
|
85
|
+
['configuration', { provider: 'p', configuration: { a: 2 } }],
|
|
86
|
+
['configuration version', { provider: 'p', configurationVersion: 'v2' }],
|
|
87
|
+
['deployment identifier', { provider: 'p', deploymentIdentifier: 'image-b' }],
|
|
88
|
+
])('changes with the %s', async (_label, change) => {
|
|
89
|
+
const inputs = {
|
|
90
|
+
provider: 'p',
|
|
91
|
+
configuration: { a: 1 },
|
|
92
|
+
configurationVersion: 'v1',
|
|
93
|
+
deploymentIdentifier: 'image-a',
|
|
94
|
+
};
|
|
95
|
+
const base = await producerConfigHash(inputs);
|
|
96
|
+
const changed = await producerConfigHash({ ...inputs, ...change });
|
|
97
|
+
expect(changed).not.toBe(base);
|
|
98
|
+
});
|
|
99
|
+
it('treats an absent and a null deployment identifier as the same', async () => {
|
|
100
|
+
const absent = await producerConfigHash({ provider: 'p' });
|
|
101
|
+
const nullIdentifier = await producerConfigHash({ provider: 'p', deploymentIdentifier: null });
|
|
102
|
+
expect(nullIdentifier).toBe(absent);
|
|
103
|
+
});
|
|
104
|
+
it.each([
|
|
105
|
+
['empty', ''],
|
|
106
|
+
['leading whitespace', ' image'],
|
|
107
|
+
['trailing whitespace', 'image '],
|
|
108
|
+
['only whitespace', ' '],
|
|
109
|
+
['not a string', 7],
|
|
110
|
+
])('rejects a deployment identifier that is %s', async (_label, deploymentIdentifier) => {
|
|
111
|
+
await expect(producerConfigHash({ provider: 'p', deploymentIdentifier: deploymentIdentifier })).rejects.toThrow(TypeError);
|
|
112
|
+
});
|
|
113
|
+
it.each([
|
|
114
|
+
['a function', () => 1],
|
|
115
|
+
['a symbol', Symbol('x')],
|
|
116
|
+
['a bigint', 1n],
|
|
117
|
+
['NaN', Number.NaN],
|
|
118
|
+
['Infinity', Number.POSITIVE_INFINITY],
|
|
119
|
+
['a Date', new Date(0)],
|
|
120
|
+
['a class instance', new (class Thing {
|
|
121
|
+
})()],
|
|
122
|
+
['undefined in an array', [undefined]],
|
|
123
|
+
])('rejects %s as a non-JSON value', async (_label, value) => {
|
|
124
|
+
await expect(producerConfigHash({ provider: 'p', configuration: { value } })).rejects.toThrow(TypeError);
|
|
125
|
+
});
|
|
126
|
+
it.each([
|
|
127
|
+
['one hole', () => new Array(1)],
|
|
128
|
+
['several holes', () => new Array(3)],
|
|
129
|
+
[
|
|
130
|
+
'a hole beside values',
|
|
131
|
+
() => {
|
|
132
|
+
const items = [1];
|
|
133
|
+
items[2] = 2;
|
|
134
|
+
return items;
|
|
135
|
+
},
|
|
136
|
+
],
|
|
137
|
+
['a very long sparse array', () => new Array(2 ** 32 - 1)],
|
|
138
|
+
])('rejects a sparse array with %s', async (_label, items) => {
|
|
139
|
+
await expect(producerConfigHash({ provider: 'p', configuration: { items: items() } })).rejects.toThrow(TypeError);
|
|
140
|
+
});
|
|
141
|
+
it('rejects a nested sparse array', async () => {
|
|
142
|
+
await expect(producerConfigHash({ provider: 'p', configuration: { a: { b: [new Array(1)] } } })).rejects.toThrow(TypeError);
|
|
143
|
+
});
|
|
144
|
+
it('rejects a sparse output-affecting hint array', async () => {
|
|
145
|
+
await expect(producerConfigHash({
|
|
146
|
+
provider: 'p',
|
|
147
|
+
hints: { ocrLanguages: new Array(1) },
|
|
148
|
+
outputAffectingHints: ['ocrLanguages'],
|
|
149
|
+
})).rejects.toThrow(TypeError);
|
|
150
|
+
});
|
|
151
|
+
it('rejects symbol keys, which JSON would silently drop', async () => {
|
|
152
|
+
await expect(producerConfigHash({ provider: 'p', configuration: { [Symbol('x')]: 1 } })).rejects.toThrow(TypeError);
|
|
153
|
+
});
|
|
154
|
+
it('rejects a cycle rather than overflowing the stack', async () => {
|
|
155
|
+
const cycle = {};
|
|
156
|
+
cycle.self = cycle;
|
|
157
|
+
await expect(producerConfigHash({ provider: 'p', configuration: cycle })).rejects.toThrow(TypeError);
|
|
158
|
+
});
|
|
159
|
+
it('accepts a shared object that is not a cycle', async () => {
|
|
160
|
+
const shared = { x: 1 };
|
|
161
|
+
await expect(producerConfigHash({ provider: 'p', configuration: { a: shared, b: shared } })).resolves.toMatch(CONTENT_HASH_PATTERN);
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
describe('buildExtractionProducer', () => {
|
|
165
|
+
const provider = {
|
|
166
|
+
name: 'tika',
|
|
167
|
+
capabilities: {
|
|
168
|
+
ocr: false,
|
|
169
|
+
markdown: false,
|
|
170
|
+
tables: false,
|
|
171
|
+
formulas: false,
|
|
172
|
+
charts: false,
|
|
173
|
+
coordinates: false,
|
|
174
|
+
pageMap: false,
|
|
175
|
+
structure: false,
|
|
176
|
+
formats: ['*/*'],
|
|
177
|
+
},
|
|
178
|
+
configuration: { ocr: false },
|
|
179
|
+
outputAffectingHints: ['language'],
|
|
180
|
+
extract: () => Promise.reject(new Error('not used')),
|
|
181
|
+
};
|
|
182
|
+
it('returns the identifier it hashed, with the same hash as a direct call', async () => {
|
|
183
|
+
const producer = await buildExtractionProducer({
|
|
184
|
+
provider,
|
|
185
|
+
providerVersion: '3.0.0',
|
|
186
|
+
hints: { language: 'th', filename: 'a.pdf' },
|
|
187
|
+
configurationVersion: 'v1',
|
|
188
|
+
deploymentIdentifier: 'image-a',
|
|
189
|
+
});
|
|
190
|
+
expect(producer).toEqual({
|
|
191
|
+
provider: 'tika',
|
|
192
|
+
providerVersion: '3.0.0',
|
|
193
|
+
deploymentIdentifier: 'image-a',
|
|
194
|
+
producerConfigHash: await producerConfigHash({
|
|
195
|
+
provider: 'tika',
|
|
196
|
+
configuration: { ocr: false },
|
|
197
|
+
hints: { language: 'th' },
|
|
198
|
+
outputAffectingHints: ['language'],
|
|
199
|
+
configurationVersion: 'v1',
|
|
200
|
+
deploymentIdentifier: 'image-a',
|
|
201
|
+
}),
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
it('gives a different hash for a different identifier', async () => {
|
|
205
|
+
const a = await buildExtractionProducer({
|
|
206
|
+
provider,
|
|
207
|
+
providerVersion: null,
|
|
208
|
+
deploymentIdentifier: 'image-a',
|
|
209
|
+
});
|
|
210
|
+
const b = await buildExtractionProducer({
|
|
211
|
+
provider,
|
|
212
|
+
providerVersion: null,
|
|
213
|
+
deploymentIdentifier: 'image-b',
|
|
214
|
+
});
|
|
215
|
+
expect(a.producerConfigHash).not.toBe(b.producerConfigHash);
|
|
216
|
+
});
|
|
217
|
+
it('normalises an absent identifier to null', async () => {
|
|
218
|
+
const producer = await buildExtractionProducer({ provider, providerVersion: null });
|
|
219
|
+
expect(producer.deploymentIdentifier).toBeNull();
|
|
220
|
+
expect(producer.providerVersion).toBeNull();
|
|
221
|
+
});
|
|
222
|
+
it('does not put the engine version in the hash', async () => {
|
|
223
|
+
const a = await buildExtractionProducer({ provider, providerVersion: '1.0.0' });
|
|
224
|
+
const b = await buildExtractionProducer({ provider, providerVersion: '2.0.0' });
|
|
225
|
+
expect(a.producerConfigHash).toBe(b.producerConfigHash);
|
|
226
|
+
});
|
|
227
|
+
it('rejects a malformed identifier', async () => {
|
|
228
|
+
await expect(buildExtractionProducer({ provider, providerVersion: null, deploymentIdentifier: ' a' })).rejects.toThrow(TypeError);
|
|
229
|
+
});
|
|
230
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Browser-safety guard for the `@byline/core` root entry.
|
|
10
|
+
*
|
|
11
|
+
* The root entry reaches browser bundles, so its module graph must not gain a Node
|
|
12
|
+
* built-in. Host framework protections such as `.server.*` file rules do not cover
|
|
13
|
+
* a published package, so the package checks itself here, against built output.
|
|
14
|
+
*
|
|
15
|
+
* Exactly one exception exists: `lib/logger.js` attempts a dynamic
|
|
16
|
+
* `import('node:async_hooks')` inside `try`/`catch` and falls back to a no-op store.
|
|
17
|
+
* The exception names the file, the specifier and the dynamic form together. A
|
|
18
|
+
* dynamic import is not safe merely because it is dynamic, and a static import of
|
|
19
|
+
* the same module in the same file would remove the fallback.
|
|
20
|
+
*
|
|
21
|
+
* Build `@byline/core` before running this test: it reads `dist`, so only a fresh build
|
|
22
|
+
* makes its result current. It fails when `dist/index.js` is missing or when the graph
|
|
23
|
+
* does not reach the extraction modules, but it is not a general stale-build detector.
|
|
24
|
+
*
|
|
25
|
+
* Its limits: it is a text walk rather than a parser, so it cannot see a computed
|
|
26
|
+
* `import(name)` or a template literal with a substitution, and it stops at package
|
|
27
|
+
* boundaries. It errs towards reporting: text in a string that looks like an import
|
|
28
|
+
* counts as one.
|
|
29
|
+
*/
|
|
30
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
31
|
+
import { builtinModules } from 'node:module';
|
|
32
|
+
import { tmpdir } from 'node:os';
|
|
33
|
+
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
34
|
+
import { fileURLToPath } from 'node:url';
|
|
35
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
36
|
+
const STATIC_IMPORT = /(?:^|[\n;])\s*(?:import|export)\s+(?:[^'"`;]*?\sfrom\s*)?['"]([^'"]+)['"]/g;
|
|
37
|
+
// The specifier may be any literal whose value is known statically: a quoted string, or
|
|
38
|
+
// a template literal with no substitution. A second argument may follow it.
|
|
39
|
+
const DYNAMIC_IMPORT = /\b(?:import|require)\(\s*(?:'([^']+)'|"([^"]+)"|`((?:[^`$]|\$(?!\{))+)`)\s*[,)]/g;
|
|
40
|
+
const BUILTINS = new Set(builtinModules);
|
|
41
|
+
function isNodeBuiltin(specifier) {
|
|
42
|
+
return specifier.startsWith('node:') || BUILTINS.has(specifier.split('/')[0] ?? specifier);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Removes comments, because built output keeps JSDoc whose examples can contain import
|
|
46
|
+
* expressions. It leaves string and template contents alone. A quoted string ends at
|
|
47
|
+
* its line break, so a quote inside a regular expression cannot hide later lines.
|
|
48
|
+
*/
|
|
49
|
+
function stripComments(source) {
|
|
50
|
+
let output = '';
|
|
51
|
+
let i = 0;
|
|
52
|
+
while (i < source.length) {
|
|
53
|
+
const char = source[i];
|
|
54
|
+
const next = source[i + 1];
|
|
55
|
+
if (char === '/' && next === '/') {
|
|
56
|
+
const end = source.indexOf('\n', i);
|
|
57
|
+
i = end === -1 ? source.length : end;
|
|
58
|
+
}
|
|
59
|
+
else if (char === '/' && next === '*') {
|
|
60
|
+
const end = source.indexOf('*/', i + 2);
|
|
61
|
+
i = end === -1 ? source.length : end + 2;
|
|
62
|
+
}
|
|
63
|
+
else if (char === "'" || char === '"' || char === '`') {
|
|
64
|
+
let end = i + 1;
|
|
65
|
+
while (end < source.length && source[end] !== char) {
|
|
66
|
+
if (char !== '`' && source[end] === '\n')
|
|
67
|
+
break;
|
|
68
|
+
end += source[end] === '\\' ? 2 : 1;
|
|
69
|
+
}
|
|
70
|
+
output += source.slice(i, end + 1);
|
|
71
|
+
i = end + 1;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
output += char;
|
|
75
|
+
i += 1;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return output;
|
|
79
|
+
}
|
|
80
|
+
/** Walks every relative static and dynamic import reachable from an entry file. */
|
|
81
|
+
function walkGraph(entry) {
|
|
82
|
+
const root = dirname(entry);
|
|
83
|
+
const seen = new Set();
|
|
84
|
+
const builtinImports = [];
|
|
85
|
+
const visit = (file) => {
|
|
86
|
+
if (seen.has(file))
|
|
87
|
+
return;
|
|
88
|
+
seen.add(file);
|
|
89
|
+
const source = stripComments(readFileSync(file, 'utf8'));
|
|
90
|
+
const name = relative(root, file).split(sep).join('/');
|
|
91
|
+
const scan = (pattern, form) => {
|
|
92
|
+
for (const match of source.matchAll(pattern)) {
|
|
93
|
+
const specifier = (match[1] ?? match[2] ?? match[3]);
|
|
94
|
+
if (specifier.startsWith('.')) {
|
|
95
|
+
visit(resolve(dirname(file), specifier));
|
|
96
|
+
}
|
|
97
|
+
else if (isNodeBuiltin(specifier)) {
|
|
98
|
+
builtinImports.push({ file: name, specifier, form });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
scan(STATIC_IMPORT, 'static');
|
|
103
|
+
scan(DYNAMIC_IMPORT, 'dynamic');
|
|
104
|
+
};
|
|
105
|
+
visit(entry);
|
|
106
|
+
return {
|
|
107
|
+
files: [...seen].map((file) => relative(root, file).split(sep).join('/')),
|
|
108
|
+
builtinImports,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const ALLOWED = [
|
|
112
|
+
{ file: 'lib/logger.js', specifier: 'node:async_hooks', form: 'dynamic' },
|
|
113
|
+
];
|
|
114
|
+
const sameImport = (a, b) => a.file === b.file && a.specifier === b.specifier && a.form === b.form;
|
|
115
|
+
/** Every failure the guard reports for a graph. */
|
|
116
|
+
function guardViolations(builtinImports, allowed = ALLOWED) {
|
|
117
|
+
const unexpected = builtinImports
|
|
118
|
+
.filter((found) => !allowed.some((entry) => sameImport(entry, found)))
|
|
119
|
+
.map((found) => `${found.file}: ${found.form} import of ${found.specifier}`);
|
|
120
|
+
const stale = allowed
|
|
121
|
+
.filter((entry) => !builtinImports.some((found) => sameImport(entry, found)))
|
|
122
|
+
.map((entry) => `stale allowlist entry: ${entry.file} ${entry.form} ${entry.specifier}`);
|
|
123
|
+
return [...unexpected, ...stale];
|
|
124
|
+
}
|
|
125
|
+
describe('root graph guard', () => {
|
|
126
|
+
const temporaryDirectories = [];
|
|
127
|
+
/** Writes a small built tree and returns the path of its `index.js`. */
|
|
128
|
+
function fixture(files) {
|
|
129
|
+
const directory = mkdtempSync(join(tmpdir(), 'byline-root-guard-'));
|
|
130
|
+
temporaryDirectories.push(directory);
|
|
131
|
+
for (const [name, source] of Object.entries(files)) {
|
|
132
|
+
const path = join(directory, name);
|
|
133
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
134
|
+
writeFileSync(path, source);
|
|
135
|
+
}
|
|
136
|
+
return join(directory, 'index.js');
|
|
137
|
+
}
|
|
138
|
+
afterEach(() => {
|
|
139
|
+
for (const directory of temporaryDirectories.splice(0)) {
|
|
140
|
+
rmSync(directory, { recursive: true, force: true });
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
const LOGGER = "try { await import('node:async_hooks') } catch {}\n";
|
|
144
|
+
describe('self-test', () => {
|
|
145
|
+
it('passes a tree with only the allowlisted import', () => {
|
|
146
|
+
const { builtinImports } = walkGraph(fixture({ 'index.js': "export * from './lib/logger.js'\n", 'lib/logger.js': LOGGER }));
|
|
147
|
+
expect(guardViolations(builtinImports)).toEqual([]);
|
|
148
|
+
});
|
|
149
|
+
it('catches a planted static import', () => {
|
|
150
|
+
const { builtinImports } = walkGraph(fixture({
|
|
151
|
+
'index.js': "export * from './lib/logger.js'\nexport { x } from './a/b.js'\n",
|
|
152
|
+
'lib/logger.js': LOGGER,
|
|
153
|
+
'a/b.js': "import { readFile } from 'node:fs/promises'\nexport const x = readFile\n",
|
|
154
|
+
}));
|
|
155
|
+
expect(guardViolations(builtinImports)).toEqual(['a/b.js: static import of node:fs/promises']);
|
|
156
|
+
});
|
|
157
|
+
it('catches a planted dynamic import', () => {
|
|
158
|
+
const { builtinImports } = walkGraph(fixture({
|
|
159
|
+
'index.js': "export * from './lib/logger.js'\nimport './a.js'\n",
|
|
160
|
+
'lib/logger.js': LOGGER,
|
|
161
|
+
'a.js': "const fs = await import('node:fs')\nexport { fs }\n",
|
|
162
|
+
}));
|
|
163
|
+
expect(guardViolations(builtinImports)).toEqual(['a.js: dynamic import of node:fs']);
|
|
164
|
+
});
|
|
165
|
+
it('catches a static import of the allowlisted module in the allowlisted file', () => {
|
|
166
|
+
const { builtinImports } = walkGraph(fixture({
|
|
167
|
+
'index.js': "export * from './lib/logger.js'\n",
|
|
168
|
+
'lib/logger.js': "import { AsyncLocalStorage } from 'node:async_hooks'\n",
|
|
169
|
+
}));
|
|
170
|
+
expect(guardViolations(builtinImports)).toEqual([
|
|
171
|
+
'lib/logger.js: static import of node:async_hooks',
|
|
172
|
+
'stale allowlist entry: lib/logger.js dynamic node:async_hooks',
|
|
173
|
+
]);
|
|
174
|
+
});
|
|
175
|
+
it('catches the allowlisted import in any other file', () => {
|
|
176
|
+
const { builtinImports } = walkGraph(fixture({
|
|
177
|
+
'index.js': "export * from './lib/logger.js'\nimport './other.js'\n",
|
|
178
|
+
'lib/logger.js': LOGGER,
|
|
179
|
+
'other.js': "await import('node:async_hooks')\n",
|
|
180
|
+
}));
|
|
181
|
+
expect(guardViolations(builtinImports)).toEqual([
|
|
182
|
+
'other.js: dynamic import of node:async_hooks',
|
|
183
|
+
]);
|
|
184
|
+
});
|
|
185
|
+
it('catches a bare built-in with no node: prefix', () => {
|
|
186
|
+
const { builtinImports } = walkGraph(fixture({
|
|
187
|
+
'index.js': "export * from './lib/logger.js'\nimport fs from 'fs'\nexport { fs }\n",
|
|
188
|
+
'lib/logger.js': LOGGER,
|
|
189
|
+
}));
|
|
190
|
+
expect(guardViolations(builtinImports)).toEqual(['index.js: static import of fs']);
|
|
191
|
+
});
|
|
192
|
+
it('ignores imports written in comments', () => {
|
|
193
|
+
const { builtinImports } = walkGraph(fixture({
|
|
194
|
+
'index.js': [
|
|
195
|
+
"/** Example: hooks: () => import('./missing.hooks.js') */",
|
|
196
|
+
"// await import('node:fs')",
|
|
197
|
+
"export * from './lib/logger.js'",
|
|
198
|
+
].join('\n'),
|
|
199
|
+
'lib/logger.js': LOGGER,
|
|
200
|
+
}));
|
|
201
|
+
expect(guardViolations(builtinImports)).toEqual([]);
|
|
202
|
+
});
|
|
203
|
+
it('still sees a real import after a string that looks like a comment', () => {
|
|
204
|
+
const { builtinImports } = walkGraph(fixture({
|
|
205
|
+
'index.js': [
|
|
206
|
+
"export * from './lib/logger.js'",
|
|
207
|
+
"const url = 'https://example.com/*'",
|
|
208
|
+
"const fs = await import('node:fs')",
|
|
209
|
+
'export { fs, url }',
|
|
210
|
+
].join('\n'),
|
|
211
|
+
'lib/logger.js': LOGGER,
|
|
212
|
+
}));
|
|
213
|
+
expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
|
|
214
|
+
});
|
|
215
|
+
it('fails on a relative import that does not resolve', () => {
|
|
216
|
+
expect(() => walkGraph(fixture({ 'index.js': "export * from './missing.js'\n" }))).toThrow(/ENOENT/);
|
|
217
|
+
});
|
|
218
|
+
it('catches a built-in named by a no-substitution template literal', () => {
|
|
219
|
+
const { builtinImports } = walkGraph(fixture({
|
|
220
|
+
'index.js': [
|
|
221
|
+
"export * from './lib/logger.js'",
|
|
222
|
+
'export const read = () => import(`node:fs`)',
|
|
223
|
+
].join('\n'),
|
|
224
|
+
'lib/logger.js': LOGGER,
|
|
225
|
+
}));
|
|
226
|
+
expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
|
|
227
|
+
});
|
|
228
|
+
it('follows a relative template literal import to a built-in one hop away', () => {
|
|
229
|
+
const { builtinImports } = walkGraph(fixture({
|
|
230
|
+
'index.js': [
|
|
231
|
+
"export * from './lib/logger.js'",
|
|
232
|
+
'export const load = () => import(`./hidden.js`)',
|
|
233
|
+
].join('\n'),
|
|
234
|
+
'lib/logger.js': LOGGER,
|
|
235
|
+
'hidden.js': "import fs from 'node:fs'\nexport { fs }\n",
|
|
236
|
+
}));
|
|
237
|
+
expect(guardViolations(builtinImports)).toEqual(['hidden.js: static import of node:fs']);
|
|
238
|
+
});
|
|
239
|
+
it('catches a dynamic import with an options argument', () => {
|
|
240
|
+
const { builtinImports } = walkGraph(fixture({
|
|
241
|
+
'index.js': [
|
|
242
|
+
"export * from './lib/logger.js'",
|
|
243
|
+
"export const load = () => import('node:fs', { with: {} })",
|
|
244
|
+
].join('\n'),
|
|
245
|
+
'lib/logger.js': LOGGER,
|
|
246
|
+
}));
|
|
247
|
+
expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
|
|
248
|
+
});
|
|
249
|
+
it('does not treat a template literal with a substitution as a known specifier', () => {
|
|
250
|
+
const { builtinImports } = walkGraph(fixture({
|
|
251
|
+
'index.js': 'export const load = (name) => import(`./${name}.js`)\n',
|
|
252
|
+
}));
|
|
253
|
+
expect(builtinImports).toEqual([]);
|
|
254
|
+
});
|
|
255
|
+
it('errs towards reporting text in a string that looks like an import', () => {
|
|
256
|
+
const { builtinImports } = walkGraph(fixture({
|
|
257
|
+
'index.js': "export const help = 'write import(`node:fs`) in your own code'\n",
|
|
258
|
+
}));
|
|
259
|
+
expect(builtinImports.map((found) => found.specifier)).toEqual(['node:fs']);
|
|
260
|
+
});
|
|
261
|
+
it('reports an allowlist entry that no longer matches', () => {
|
|
262
|
+
const { builtinImports } = walkGraph(fixture({ 'index.js': 'export const x = 1\n' }));
|
|
263
|
+
expect(guardViolations(builtinImports)).toEqual([
|
|
264
|
+
'stale allowlist entry: lib/logger.js dynamic node:async_hooks',
|
|
265
|
+
]);
|
|
266
|
+
});
|
|
267
|
+
it('does not treat a package with a built-in-like suffix as a built-in', () => {
|
|
268
|
+
const { builtinImports } = walkGraph(fixture({ 'index.js': "import x from 'fs-extra'\nexport { x }\n" }));
|
|
269
|
+
expect(builtinImports).toEqual([]);
|
|
270
|
+
});
|
|
271
|
+
});
|
|
272
|
+
describe('the built @byline/core root entry', () => {
|
|
273
|
+
const entry = fileURLToPath(new URL('../../dist/index.js', import.meta.url));
|
|
274
|
+
it('exists; build @byline/core before running this test', () => {
|
|
275
|
+
expect(existsSync(entry)).toBe(true);
|
|
276
|
+
});
|
|
277
|
+
it('reaches the extraction modules', () => {
|
|
278
|
+
const { files } = walkGraph(entry);
|
|
279
|
+
expect(files).toEqual(expect.arrayContaining([
|
|
280
|
+
'extraction/artifact.js',
|
|
281
|
+
'extraction/errors.js',
|
|
282
|
+
'extraction/evidence.js',
|
|
283
|
+
'extraction/outcomes.js',
|
|
284
|
+
'extraction/page-map.js',
|
|
285
|
+
'extraction/producer.js',
|
|
286
|
+
'extraction/work.js',
|
|
287
|
+
]));
|
|
288
|
+
});
|
|
289
|
+
it('holds no Node built-in beyond the allowlisted dynamic import', () => {
|
|
290
|
+
expect(guardViolations(walkGraph(entry).builtinImports)).toEqual([]);
|
|
291
|
+
});
|
|
292
|
+
});
|
|
293
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { type ExtractionProducerIdentity } from './producer.js';
|
|
9
|
+
import type { ExtractionArtifact, ExtractionWorkDecision } from '../@types/extraction-types.js';
|
|
10
|
+
/**
|
|
11
|
+
* The output and evidence a reference currently holds. Pass only this reference's
|
|
12
|
+
* own record: reuse here means "keep what this reference has", not a cache lookup.
|
|
13
|
+
* An artifact with a `null` `producerConfigHash` must never be returned as a cache
|
|
14
|
+
* match for a fully identified producer or for another reference.
|
|
15
|
+
*
|
|
16
|
+
* @provisional
|
|
17
|
+
*/
|
|
18
|
+
export interface ExistingExtraction {
|
|
19
|
+
state: 'complete' | 'failed' | 'identity-problem';
|
|
20
|
+
artifact?: Pick<ExtractionArtifact, 'contentHash' | 'sourceVerified' | 'provider' | 'providerVersion' | 'producerConfigHash' | 'schemaVersion'>;
|
|
21
|
+
evidence?: unknown;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Decides whether a reference needs extraction, before any bytes are read.
|
|
25
|
+
*
|
|
26
|
+
* - A reference with no `fileHash` is an identity problem. `force` does not bypass it.
|
|
27
|
+
* - `force` always yields `work`. It bypasses reuse, never the verified read.
|
|
28
|
+
* - `reuse` needs a complete, verified artifact for the reference's hash, valid
|
|
29
|
+
* evidence for that hash, the same provider and configuration, a version that
|
|
30
|
+
* passes the version rules, and the current schema. Follow it with
|
|
31
|
+
* `probeVerifiedSource`.
|
|
32
|
+
* - A `null` current engine version never yields a version reason, so an unknown
|
|
33
|
+
* version cannot cause repeated work.
|
|
34
|
+
* - An artifact with a `null` `producerConfigHash` is historical output whose
|
|
35
|
+
* configuration was never recorded. Only that hash and a `null` recorded version go
|
|
36
|
+
* uncompared, and `reuse` then reports `producerUnchecked: true`. A provider
|
|
37
|
+
* mismatch, two known versions that differ, and every other reason apply unchanged.
|
|
38
|
+
*
|
|
39
|
+
* The decision is synchronous and cannot recompute the hash, so it trusts that
|
|
40
|
+
* `producer.deploymentIdentifier` is the value hashed into `producerConfigHash`.
|
|
41
|
+
* `buildExtractionProducer` guarantees that. It checks the identifier's form and
|
|
42
|
+
* throws a `TypeError` when malformed.
|
|
43
|
+
*
|
|
44
|
+
* Hosts may add their own reasons and treat the union as the decision.
|
|
45
|
+
*
|
|
46
|
+
* @provisional
|
|
47
|
+
*/
|
|
48
|
+
export declare function decideExtractionWork(input: {
|
|
49
|
+
force?: boolean;
|
|
50
|
+
reference: {
|
|
51
|
+
fileHash?: string | null;
|
|
52
|
+
};
|
|
53
|
+
producer: ExtractionProducerIdentity;
|
|
54
|
+
existing?: ExistingExtraction | null;
|
|
55
|
+
}): ExtractionWorkDecision;
|