@byline/core 6.12.1 → 6.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/@types/extraction-types.d.ts +423 -0
  2. package/dist/@types/extraction-types.js +8 -0
  3. package/dist/@types/index.d.ts +1 -0
  4. package/dist/@types/index.js +1 -0
  5. package/dist/extraction/artifact.d.ts +46 -0
  6. package/dist/extraction/artifact.js +121 -0
  7. package/dist/extraction/artifact.test.node.d.ts +8 -0
  8. package/dist/extraction/artifact.test.node.js +194 -0
  9. package/dist/extraction/errors.d.ts +47 -0
  10. package/dist/extraction/errors.js +57 -0
  11. package/dist/extraction/errors.test.node.d.ts +8 -0
  12. package/dist/extraction/errors.test.node.js +68 -0
  13. package/dist/extraction/evidence.d.ts +22 -0
  14. package/dist/extraction/evidence.js +54 -0
  15. package/dist/extraction/evidence.test.node.d.ts +8 -0
  16. package/dist/extraction/evidence.test.node.js +138 -0
  17. package/dist/extraction/index.d.ts +14 -0
  18. package/dist/extraction/index.js +17 -0
  19. package/dist/extraction/internal.d.ts +30 -0
  20. package/dist/extraction/internal.js +93 -0
  21. package/dist/extraction/outcomes.d.ts +20 -0
  22. package/dist/extraction/outcomes.js +30 -0
  23. package/dist/extraction/outcomes.test.node.d.ts +8 -0
  24. package/dist/extraction/outcomes.test.node.js +55 -0
  25. package/dist/extraction/page-map.d.ts +40 -0
  26. package/dist/extraction/page-map.js +190 -0
  27. package/dist/extraction/page-map.test.node.d.ts +8 -0
  28. package/dist/extraction/page-map.test.node.js +371 -0
  29. package/dist/extraction/producer.d.ts +62 -0
  30. package/dist/extraction/producer.js +76 -0
  31. package/dist/extraction/producer.test.node.d.ts +8 -0
  32. package/dist/extraction/producer.test.node.js +230 -0
  33. package/dist/extraction/root-graph-guard.test.node.d.ts +8 -0
  34. package/dist/extraction/root-graph-guard.test.node.js +293 -0
  35. package/dist/extraction/work.d.ts +55 -0
  36. package/dist/extraction/work.js +102 -0
  37. package/dist/extraction/work.test.node.d.ts +8 -0
  38. package/dist/extraction/work.test.node.js +409 -0
  39. package/dist/index.d.ts +1 -0
  40. package/dist/index.js +1 -0
  41. package/dist/storage/stored-files-server.d.ts +1 -0
  42. package/dist/storage/stored-files-server.js +9 -24
  43. package/dist/storage/stored-files-server.test.node.js +6 -0
  44. package/dist/storage/verified-read.d.ts +131 -0
  45. package/dist/storage/verified-read.js +328 -0
  46. package/dist/storage/verified-read.test.node.d.ts +8 -0
  47. package/dist/storage/verified-read.test.node.js +645 -0
  48. package/package.json +2 -2
@@ -0,0 +1,230 @@
1
+ /**
2
+ * This Source Code is subject to the terms of the Mozilla Public
3
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
4
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
+ *
6
+ * Copyright (c) Infonomic Company Limited
7
+ */
8
+ import { describe, expect, it } from 'vitest';
9
+ import { CONTENT_HASH_PATTERN, sha256Hex } from '../storage/content-hash.js';
10
+ import { buildExtractionProducer, producerConfigHash } from './producer.js';
11
+ const encoder = new TextEncoder();
12
+ describe('producerConfigHash', () => {
13
+ it('hashes the canonical JSON of its inputs as lower-case SHA-256', async () => {
14
+ const hash = await producerConfigHash({
15
+ provider: 'tika',
16
+ configuration: { ocr: true, dpi: 300 },
17
+ hints: { language: 'en', filename: 'a.pdf' },
18
+ outputAffectingHints: ['language'],
19
+ configurationVersion: 'v1',
20
+ deploymentIdentifier: 'image-a',
21
+ });
22
+ const expected = '{"configuration":{"dpi":300,"ocr":true},"configurationVersion":"v1",' +
23
+ '"deploymentIdentifier":"image-a","hints":{"language":"en"},"provider":"tika"}';
24
+ expect(hash).toBe(await sha256Hex(encoder.encode(expected)));
25
+ expect(hash).toMatch(CONTENT_HASH_PATTERN);
26
+ });
27
+ it('does not depend on key order at any depth', async () => {
28
+ const a = await producerConfigHash({
29
+ provider: 'p',
30
+ configuration: { b: 1, a: { d: [1, { y: 2, x: 1 }], c: null } },
31
+ });
32
+ const b = await producerConfigHash({
33
+ provider: 'p',
34
+ configuration: { a: { c: null, d: [1, { x: 1, y: 2 }] }, b: 1 },
35
+ });
36
+ expect(a).toBe(b);
37
+ });
38
+ it('keeps array order significant', async () => {
39
+ const a = await producerConfigHash({ provider: 'p', configuration: { langs: ['th', 'en'] } });
40
+ const b = await producerConfigHash({ provider: 'p', configuration: { langs: ['en', 'th'] } });
41
+ expect(a).not.toBe(b);
42
+ });
43
+ it('drops undefined object members', async () => {
44
+ const a = await producerConfigHash({
45
+ provider: 'p',
46
+ configuration: { a: 1, b: undefined },
47
+ });
48
+ const b = await producerConfigHash({ provider: 'p', configuration: { a: 1 } });
49
+ expect(a).toBe(b);
50
+ });
51
+ it('drops nested undefined members, unlike strict JSON validation of artifact structure', async () => {
52
+ const a = await producerConfigHash({
53
+ provider: 'p',
54
+ configuration: { section: { a: 1, b: undefined } },
55
+ });
56
+ const b = await producerConfigHash({ provider: 'p', configuration: { section: { a: 1 } } });
57
+ expect(a).toBe(b);
58
+ });
59
+ it('keeps only the output-affecting hints that were supplied', async () => {
60
+ const withFilename = await producerConfigHash({
61
+ provider: 'p',
62
+ hints: { language: 'en', filename: 'a.pdf' },
63
+ outputAffectingHints: ['language', 'ocrLanguages'],
64
+ });
65
+ const withoutFilename = await producerConfigHash({
66
+ provider: 'p',
67
+ hints: { language: 'en' },
68
+ outputAffectingHints: ['language', 'ocrLanguages'],
69
+ });
70
+ const otherLanguage = await producerConfigHash({
71
+ provider: 'p',
72
+ hints: { language: 'th' },
73
+ outputAffectingHints: ['language'],
74
+ });
75
+ expect(withFilename).toBe(withoutFilename);
76
+ expect(otherLanguage).not.toBe(withoutFilename);
77
+ });
78
+ it('ignores every hint when the provider declares none', async () => {
79
+ const a = await producerConfigHash({ provider: 'p', hints: { language: 'en' } });
80
+ const b = await producerConfigHash({ provider: 'p' });
81
+ expect(a).toBe(b);
82
+ });
83
+ it.each([
84
+ ['provider', { provider: 'q' }],
85
+ ['configuration', { provider: 'p', configuration: { a: 2 } }],
86
+ ['configuration version', { provider: 'p', configurationVersion: 'v2' }],
87
+ ['deployment identifier', { provider: 'p', deploymentIdentifier: 'image-b' }],
88
+ ])('changes with the %s', async (_label, change) => {
89
+ const inputs = {
90
+ provider: 'p',
91
+ configuration: { a: 1 },
92
+ configurationVersion: 'v1',
93
+ deploymentIdentifier: 'image-a',
94
+ };
95
+ const base = await producerConfigHash(inputs);
96
+ const changed = await producerConfigHash({ ...inputs, ...change });
97
+ expect(changed).not.toBe(base);
98
+ });
99
+ it('treats an absent and a null deployment identifier as the same', async () => {
100
+ const absent = await producerConfigHash({ provider: 'p' });
101
+ const nullIdentifier = await producerConfigHash({ provider: 'p', deploymentIdentifier: null });
102
+ expect(nullIdentifier).toBe(absent);
103
+ });
104
+ it.each([
105
+ ['empty', ''],
106
+ ['leading whitespace', ' image'],
107
+ ['trailing whitespace', 'image '],
108
+ ['only whitespace', ' '],
109
+ ['not a string', 7],
110
+ ])('rejects a deployment identifier that is %s', async (_label, deploymentIdentifier) => {
111
+ await expect(producerConfigHash({ provider: 'p', deploymentIdentifier: deploymentIdentifier })).rejects.toThrow(TypeError);
112
+ });
113
+ it.each([
114
+ ['a function', () => 1],
115
+ ['a symbol', Symbol('x')],
116
+ ['a bigint', 1n],
117
+ ['NaN', Number.NaN],
118
+ ['Infinity', Number.POSITIVE_INFINITY],
119
+ ['a Date', new Date(0)],
120
+ ['a class instance', new (class Thing {
121
+ })()],
122
+ ['undefined in an array', [undefined]],
123
+ ])('rejects %s as a non-JSON value', async (_label, value) => {
124
+ await expect(producerConfigHash({ provider: 'p', configuration: { value } })).rejects.toThrow(TypeError);
125
+ });
126
+ it.each([
127
+ ['one hole', () => new Array(1)],
128
+ ['several holes', () => new Array(3)],
129
+ [
130
+ 'a hole beside values',
131
+ () => {
132
+ const items = [1];
133
+ items[2] = 2;
134
+ return items;
135
+ },
136
+ ],
137
+ ['a very long sparse array', () => new Array(2 ** 32 - 1)],
138
+ ])('rejects a sparse array with %s', async (_label, items) => {
139
+ await expect(producerConfigHash({ provider: 'p', configuration: { items: items() } })).rejects.toThrow(TypeError);
140
+ });
141
+ it('rejects a nested sparse array', async () => {
142
+ await expect(producerConfigHash({ provider: 'p', configuration: { a: { b: [new Array(1)] } } })).rejects.toThrow(TypeError);
143
+ });
144
+ it('rejects a sparse output-affecting hint array', async () => {
145
+ await expect(producerConfigHash({
146
+ provider: 'p',
147
+ hints: { ocrLanguages: new Array(1) },
148
+ outputAffectingHints: ['ocrLanguages'],
149
+ })).rejects.toThrow(TypeError);
150
+ });
151
+ it('rejects symbol keys, which JSON would silently drop', async () => {
152
+ await expect(producerConfigHash({ provider: 'p', configuration: { [Symbol('x')]: 1 } })).rejects.toThrow(TypeError);
153
+ });
154
+ it('rejects a cycle rather than overflowing the stack', async () => {
155
+ const cycle = {};
156
+ cycle.self = cycle;
157
+ await expect(producerConfigHash({ provider: 'p', configuration: cycle })).rejects.toThrow(TypeError);
158
+ });
159
+ it('accepts a shared object that is not a cycle', async () => {
160
+ const shared = { x: 1 };
161
+ await expect(producerConfigHash({ provider: 'p', configuration: { a: shared, b: shared } })).resolves.toMatch(CONTENT_HASH_PATTERN);
162
+ });
163
+ });
164
+ describe('buildExtractionProducer', () => {
165
+ const provider = {
166
+ name: 'tika',
167
+ capabilities: {
168
+ ocr: false,
169
+ markdown: false,
170
+ tables: false,
171
+ formulas: false,
172
+ charts: false,
173
+ coordinates: false,
174
+ pageMap: false,
175
+ structure: false,
176
+ formats: ['*/*'],
177
+ },
178
+ configuration: { ocr: false },
179
+ outputAffectingHints: ['language'],
180
+ extract: () => Promise.reject(new Error('not used')),
181
+ };
182
+ it('returns the identifier it hashed, with the same hash as a direct call', async () => {
183
+ const producer = await buildExtractionProducer({
184
+ provider,
185
+ providerVersion: '3.0.0',
186
+ hints: { language: 'th', filename: 'a.pdf' },
187
+ configurationVersion: 'v1',
188
+ deploymentIdentifier: 'image-a',
189
+ });
190
+ expect(producer).toEqual({
191
+ provider: 'tika',
192
+ providerVersion: '3.0.0',
193
+ deploymentIdentifier: 'image-a',
194
+ producerConfigHash: await producerConfigHash({
195
+ provider: 'tika',
196
+ configuration: { ocr: false },
197
+ hints: { language: 'th' },
198
+ outputAffectingHints: ['language'],
199
+ configurationVersion: 'v1',
200
+ deploymentIdentifier: 'image-a',
201
+ }),
202
+ });
203
+ });
204
+ it('gives a different hash for a different identifier', async () => {
205
+ const a = await buildExtractionProducer({
206
+ provider,
207
+ providerVersion: null,
208
+ deploymentIdentifier: 'image-a',
209
+ });
210
+ const b = await buildExtractionProducer({
211
+ provider,
212
+ providerVersion: null,
213
+ deploymentIdentifier: 'image-b',
214
+ });
215
+ expect(a.producerConfigHash).not.toBe(b.producerConfigHash);
216
+ });
217
+ it('normalises an absent identifier to null', async () => {
218
+ const producer = await buildExtractionProducer({ provider, providerVersion: null });
219
+ expect(producer.deploymentIdentifier).toBeNull();
220
+ expect(producer.providerVersion).toBeNull();
221
+ });
222
+ it('does not put the engine version in the hash', async () => {
223
+ const a = await buildExtractionProducer({ provider, providerVersion: '1.0.0' });
224
+ const b = await buildExtractionProducer({ provider, providerVersion: '2.0.0' });
225
+ expect(a.producerConfigHash).toBe(b.producerConfigHash);
226
+ });
227
+ it('rejects a malformed identifier', async () => {
228
+ await expect(buildExtractionProducer({ provider, providerVersion: null, deploymentIdentifier: ' a' })).rejects.toThrow(TypeError);
229
+ });
230
+ });
@@ -0,0 +1,8 @@
1
+ /**
2
+ * This Source Code is subject to the terms of the Mozilla Public
3
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
4
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
+ *
6
+ * Copyright (c) Infonomic Company Limited
7
+ */
8
+ export {};
@@ -0,0 +1,293 @@
1
+ /**
2
+ * This Source Code is subject to the terms of the Mozilla Public
3
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
4
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
+ *
6
+ * Copyright (c) Infonomic Company Limited
7
+ */
8
+ /**
9
+ * Browser-safety guard for the `@byline/core` root entry.
10
+ *
11
+ * The root entry reaches browser bundles, so its module graph must not gain a Node
12
+ * built-in. Host framework protections such as `.server.*` file rules do not cover
13
+ * a published package, so the package checks itself here, against built output.
14
+ *
15
+ * Exactly one exception exists: `lib/logger.js` attempts a dynamic
16
+ * `import('node:async_hooks')` inside `try`/`catch` and falls back to a no-op store.
17
+ * The exception names the file, the specifier and the dynamic form together. A
18
+ * dynamic import is not safe merely because it is dynamic, and a static import of
19
+ * the same module in the same file would remove the fallback.
20
+ *
21
+ * Build `@byline/core` before running this test: it reads `dist`, so only a fresh build
22
+ * makes its result current. It fails when `dist/index.js` is missing or when the graph
23
+ * does not reach the extraction modules, but it is not a general stale-build detector.
24
+ *
25
+ * Its limits: it is a text walk rather than a parser, so it cannot see a computed
26
+ * `import(name)` or a template literal with a substitution, and it stops at package
27
+ * boundaries. It errs towards reporting: text in a string that looks like an import
28
+ * counts as one.
29
+ */
30
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
31
+ import { builtinModules } from 'node:module';
32
+ import { tmpdir } from 'node:os';
33
+ import { dirname, join, relative, resolve, sep } from 'node:path';
34
+ import { fileURLToPath } from 'node:url';
35
+ import { afterEach, describe, expect, it } from 'vitest';
36
+ const STATIC_IMPORT = /(?:^|[\n;])\s*(?:import|export)\s+(?:[^'"`;]*?\sfrom\s*)?['"]([^'"]+)['"]/g;
37
+ // The specifier may be any literal whose value is known statically: a quoted string, or
38
+ // a template literal with no substitution. A second argument may follow it.
39
+ const DYNAMIC_IMPORT = /\b(?:import|require)\(\s*(?:'([^']+)'|"([^"]+)"|`((?:[^`$]|\$(?!\{))+)`)\s*[,)]/g;
40
+ const BUILTINS = new Set(builtinModules);
41
+ function isNodeBuiltin(specifier) {
42
+ return specifier.startsWith('node:') || BUILTINS.has(specifier.split('/')[0] ?? specifier);
43
+ }
44
+ /**
45
+ * Removes comments, because built output keeps JSDoc whose examples can contain import
46
+ * expressions. It leaves string and template contents alone. A quoted string ends at
47
+ * its line break, so a quote inside a regular expression cannot hide later lines.
48
+ */
49
+ function stripComments(source) {
50
+ let output = '';
51
+ let i = 0;
52
+ while (i < source.length) {
53
+ const char = source[i];
54
+ const next = source[i + 1];
55
+ if (char === '/' && next === '/') {
56
+ const end = source.indexOf('\n', i);
57
+ i = end === -1 ? source.length : end;
58
+ }
59
+ else if (char === '/' && next === '*') {
60
+ const end = source.indexOf('*/', i + 2);
61
+ i = end === -1 ? source.length : end + 2;
62
+ }
63
+ else if (char === "'" || char === '"' || char === '`') {
64
+ let end = i + 1;
65
+ while (end < source.length && source[end] !== char) {
66
+ if (char !== '`' && source[end] === '\n')
67
+ break;
68
+ end += source[end] === '\\' ? 2 : 1;
69
+ }
70
+ output += source.slice(i, end + 1);
71
+ i = end + 1;
72
+ }
73
+ else {
74
+ output += char;
75
+ i += 1;
76
+ }
77
+ }
78
+ return output;
79
+ }
80
+ /** Walks every relative static and dynamic import reachable from an entry file. */
81
+ function walkGraph(entry) {
82
+ const root = dirname(entry);
83
+ const seen = new Set();
84
+ const builtinImports = [];
85
+ const visit = (file) => {
86
+ if (seen.has(file))
87
+ return;
88
+ seen.add(file);
89
+ const source = stripComments(readFileSync(file, 'utf8'));
90
+ const name = relative(root, file).split(sep).join('/');
91
+ const scan = (pattern, form) => {
92
+ for (const match of source.matchAll(pattern)) {
93
+ const specifier = (match[1] ?? match[2] ?? match[3]);
94
+ if (specifier.startsWith('.')) {
95
+ visit(resolve(dirname(file), specifier));
96
+ }
97
+ else if (isNodeBuiltin(specifier)) {
98
+ builtinImports.push({ file: name, specifier, form });
99
+ }
100
+ }
101
+ };
102
+ scan(STATIC_IMPORT, 'static');
103
+ scan(DYNAMIC_IMPORT, 'dynamic');
104
+ };
105
+ visit(entry);
106
+ return {
107
+ files: [...seen].map((file) => relative(root, file).split(sep).join('/')),
108
+ builtinImports,
109
+ };
110
+ }
111
+ const ALLOWED = [
112
+ { file: 'lib/logger.js', specifier: 'node:async_hooks', form: 'dynamic' },
113
+ ];
114
+ const sameImport = (a, b) => a.file === b.file && a.specifier === b.specifier && a.form === b.form;
115
+ /** Every failure the guard reports for a graph. */
116
+ function guardViolations(builtinImports, allowed = ALLOWED) {
117
+ const unexpected = builtinImports
118
+ .filter((found) => !allowed.some((entry) => sameImport(entry, found)))
119
+ .map((found) => `${found.file}: ${found.form} import of ${found.specifier}`);
120
+ const stale = allowed
121
+ .filter((entry) => !builtinImports.some((found) => sameImport(entry, found)))
122
+ .map((entry) => `stale allowlist entry: ${entry.file} ${entry.form} ${entry.specifier}`);
123
+ return [...unexpected, ...stale];
124
+ }
125
+ describe('root graph guard', () => {
126
+ const temporaryDirectories = [];
127
+ /** Writes a small built tree and returns the path of its `index.js`. */
128
+ function fixture(files) {
129
+ const directory = mkdtempSync(join(tmpdir(), 'byline-root-guard-'));
130
+ temporaryDirectories.push(directory);
131
+ for (const [name, source] of Object.entries(files)) {
132
+ const path = join(directory, name);
133
+ mkdirSync(dirname(path), { recursive: true });
134
+ writeFileSync(path, source);
135
+ }
136
+ return join(directory, 'index.js');
137
+ }
138
+ afterEach(() => {
139
+ for (const directory of temporaryDirectories.splice(0)) {
140
+ rmSync(directory, { recursive: true, force: true });
141
+ }
142
+ });
143
+ const LOGGER = "try { await import('node:async_hooks') } catch {}\n";
144
+ describe('self-test', () => {
145
+ it('passes a tree with only the allowlisted import', () => {
146
+ const { builtinImports } = walkGraph(fixture({ 'index.js': "export * from './lib/logger.js'\n", 'lib/logger.js': LOGGER }));
147
+ expect(guardViolations(builtinImports)).toEqual([]);
148
+ });
149
+ it('catches a planted static import', () => {
150
+ const { builtinImports } = walkGraph(fixture({
151
+ 'index.js': "export * from './lib/logger.js'\nexport { x } from './a/b.js'\n",
152
+ 'lib/logger.js': LOGGER,
153
+ 'a/b.js': "import { readFile } from 'node:fs/promises'\nexport const x = readFile\n",
154
+ }));
155
+ expect(guardViolations(builtinImports)).toEqual(['a/b.js: static import of node:fs/promises']);
156
+ });
157
+ it('catches a planted dynamic import', () => {
158
+ const { builtinImports } = walkGraph(fixture({
159
+ 'index.js': "export * from './lib/logger.js'\nimport './a.js'\n",
160
+ 'lib/logger.js': LOGGER,
161
+ 'a.js': "const fs = await import('node:fs')\nexport { fs }\n",
162
+ }));
163
+ expect(guardViolations(builtinImports)).toEqual(['a.js: dynamic import of node:fs']);
164
+ });
165
+ it('catches a static import of the allowlisted module in the allowlisted file', () => {
166
+ const { builtinImports } = walkGraph(fixture({
167
+ 'index.js': "export * from './lib/logger.js'\n",
168
+ 'lib/logger.js': "import { AsyncLocalStorage } from 'node:async_hooks'\n",
169
+ }));
170
+ expect(guardViolations(builtinImports)).toEqual([
171
+ 'lib/logger.js: static import of node:async_hooks',
172
+ 'stale allowlist entry: lib/logger.js dynamic node:async_hooks',
173
+ ]);
174
+ });
175
+ it('catches the allowlisted import in any other file', () => {
176
+ const { builtinImports } = walkGraph(fixture({
177
+ 'index.js': "export * from './lib/logger.js'\nimport './other.js'\n",
178
+ 'lib/logger.js': LOGGER,
179
+ 'other.js': "await import('node:async_hooks')\n",
180
+ }));
181
+ expect(guardViolations(builtinImports)).toEqual([
182
+ 'other.js: dynamic import of node:async_hooks',
183
+ ]);
184
+ });
185
+ it('catches a bare built-in with no node: prefix', () => {
186
+ const { builtinImports } = walkGraph(fixture({
187
+ 'index.js': "export * from './lib/logger.js'\nimport fs from 'fs'\nexport { fs }\n",
188
+ 'lib/logger.js': LOGGER,
189
+ }));
190
+ expect(guardViolations(builtinImports)).toEqual(['index.js: static import of fs']);
191
+ });
192
+ it('ignores imports written in comments', () => {
193
+ const { builtinImports } = walkGraph(fixture({
194
+ 'index.js': [
195
+ "/** Example: hooks: () => import('./missing.hooks.js') */",
196
+ "// await import('node:fs')",
197
+ "export * from './lib/logger.js'",
198
+ ].join('\n'),
199
+ 'lib/logger.js': LOGGER,
200
+ }));
201
+ expect(guardViolations(builtinImports)).toEqual([]);
202
+ });
203
+ it('still sees a real import after a string that looks like a comment', () => {
204
+ const { builtinImports } = walkGraph(fixture({
205
+ 'index.js': [
206
+ "export * from './lib/logger.js'",
207
+ "const url = 'https://example.com/*'",
208
+ "const fs = await import('node:fs')",
209
+ 'export { fs, url }',
210
+ ].join('\n'),
211
+ 'lib/logger.js': LOGGER,
212
+ }));
213
+ expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
214
+ });
215
+ it('fails on a relative import that does not resolve', () => {
216
+ expect(() => walkGraph(fixture({ 'index.js': "export * from './missing.js'\n" }))).toThrow(/ENOENT/);
217
+ });
218
+ it('catches a built-in named by a no-substitution template literal', () => {
219
+ const { builtinImports } = walkGraph(fixture({
220
+ 'index.js': [
221
+ "export * from './lib/logger.js'",
222
+ 'export const read = () => import(`node:fs`)',
223
+ ].join('\n'),
224
+ 'lib/logger.js': LOGGER,
225
+ }));
226
+ expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
227
+ });
228
+ it('follows a relative template literal import to a built-in one hop away', () => {
229
+ const { builtinImports } = walkGraph(fixture({
230
+ 'index.js': [
231
+ "export * from './lib/logger.js'",
232
+ 'export const load = () => import(`./hidden.js`)',
233
+ ].join('\n'),
234
+ 'lib/logger.js': LOGGER,
235
+ 'hidden.js': "import fs from 'node:fs'\nexport { fs }\n",
236
+ }));
237
+ expect(guardViolations(builtinImports)).toEqual(['hidden.js: static import of node:fs']);
238
+ });
239
+ it('catches a dynamic import with an options argument', () => {
240
+ const { builtinImports } = walkGraph(fixture({
241
+ 'index.js': [
242
+ "export * from './lib/logger.js'",
243
+ "export const load = () => import('node:fs', { with: {} })",
244
+ ].join('\n'),
245
+ 'lib/logger.js': LOGGER,
246
+ }));
247
+ expect(guardViolations(builtinImports)).toEqual(['index.js: dynamic import of node:fs']);
248
+ });
249
+ it('does not treat a template literal with a substitution as a known specifier', () => {
250
+ const { builtinImports } = walkGraph(fixture({
251
+ 'index.js': 'export const load = (name) => import(`./${name}.js`)\n',
252
+ }));
253
+ expect(builtinImports).toEqual([]);
254
+ });
255
+ it('errs towards reporting text in a string that looks like an import', () => {
256
+ const { builtinImports } = walkGraph(fixture({
257
+ 'index.js': "export const help = 'write import(`node:fs`) in your own code'\n",
258
+ }));
259
+ expect(builtinImports.map((found) => found.specifier)).toEqual(['node:fs']);
260
+ });
261
+ it('reports an allowlist entry that no longer matches', () => {
262
+ const { builtinImports } = walkGraph(fixture({ 'index.js': 'export const x = 1\n' }));
263
+ expect(guardViolations(builtinImports)).toEqual([
264
+ 'stale allowlist entry: lib/logger.js dynamic node:async_hooks',
265
+ ]);
266
+ });
267
+ it('does not treat a package with a built-in-like suffix as a built-in', () => {
268
+ const { builtinImports } = walkGraph(fixture({ 'index.js': "import x from 'fs-extra'\nexport { x }\n" }));
269
+ expect(builtinImports).toEqual([]);
270
+ });
271
+ });
272
+ describe('the built @byline/core root entry', () => {
273
+ const entry = fileURLToPath(new URL('../../dist/index.js', import.meta.url));
274
+ it('exists; build @byline/core before running this test', () => {
275
+ expect(existsSync(entry)).toBe(true);
276
+ });
277
+ it('reaches the extraction modules', () => {
278
+ const { files } = walkGraph(entry);
279
+ expect(files).toEqual(expect.arrayContaining([
280
+ 'extraction/artifact.js',
281
+ 'extraction/errors.js',
282
+ 'extraction/evidence.js',
283
+ 'extraction/outcomes.js',
284
+ 'extraction/page-map.js',
285
+ 'extraction/producer.js',
286
+ 'extraction/work.js',
287
+ ]));
288
+ });
289
+ it('holds no Node built-in beyond the allowlisted dynamic import', () => {
290
+ expect(guardViolations(walkGraph(entry).builtinImports)).toEqual([]);
291
+ });
292
+ });
293
+ });
@@ -0,0 +1,55 @@
1
+ /**
2
+ * This Source Code is subject to the terms of the Mozilla Public
3
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
4
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
+ *
6
+ * Copyright (c) Infonomic Company Limited
7
+ */
8
+ import { type ExtractionProducerIdentity } from './producer.js';
9
+ import type { ExtractionArtifact, ExtractionWorkDecision } from '../@types/extraction-types.js';
10
+ /**
11
+ * The output and evidence a reference currently holds. Pass only this reference's
12
+ * own record: reuse here means "keep what this reference has", not a cache lookup.
13
+ * An artifact with a `null` `producerConfigHash` must never be returned as a cache
14
+ * match for a fully identified producer or for another reference.
15
+ *
16
+ * @provisional
17
+ */
18
+ export interface ExistingExtraction {
19
+ state: 'complete' | 'failed' | 'identity-problem';
20
+ artifact?: Pick<ExtractionArtifact, 'contentHash' | 'sourceVerified' | 'provider' | 'providerVersion' | 'producerConfigHash' | 'schemaVersion'>;
21
+ evidence?: unknown;
22
+ }
23
+ /**
24
+ * Decides whether a reference needs extraction, before any bytes are read.
25
+ *
26
+ * - A reference with no `fileHash` is an identity problem. `force` does not bypass it.
27
+ * - `force` always yields `work`. It bypasses reuse, never the verified read.
28
+ * - `reuse` needs a complete, verified artifact for the reference's hash, valid
29
+ * evidence for that hash, the same provider and configuration, a version that
30
+ * passes the version rules, and the current schema. Follow it with
31
+ * `probeVerifiedSource`.
32
+ * - A `null` current engine version never yields a version reason, so an unknown
33
+ * version cannot cause repeated work.
34
+ * - An artifact with a `null` `producerConfigHash` is historical output whose
35
+ * configuration was never recorded. Only that hash and a `null` recorded version go
36
+ * uncompared, and `reuse` then reports `producerUnchecked: true`. A provider
37
+ * mismatch, two known versions that differ, and every other reason apply unchanged.
38
+ *
39
+ * The decision is synchronous and cannot recompute the hash, so it trusts that
40
+ * `producer.deploymentIdentifier` is the value hashed into `producerConfigHash`.
41
+ * `buildExtractionProducer` guarantees that. It checks the identifier's form and
42
+ * throws a `TypeError` when malformed.
43
+ *
44
+ * Hosts may add their own reasons and treat the union as the decision.
45
+ *
46
+ * @provisional
47
+ */
48
+ export declare function decideExtractionWork(input: {
49
+ force?: boolean;
50
+ reference: {
51
+ fileHash?: string | null;
52
+ };
53
+ producer: ExtractionProducerIdentity;
54
+ existing?: ExistingExtraction | null;
55
+ }): ExtractionWorkDecision;