@hraness/sponge 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +39 -17
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +10 -46
- package/dist/src/digest/editorial.js +73 -59
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +53 -32
- package/dist/src/digest/pipeline.js +357 -125
- package/dist/src/digest/schema.d.ts +44 -17
- package/dist/src/digest/schema.js +132 -44
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +31 -24
- package/dist/src/importers/jungle-digest.js +217 -115
- package/dist/src/importers/jungle-store.d.ts +20 -11
- package/dist/src/importers/jungle-store.js +136 -41
- package/dist/src/importers/jungle.d.ts +32 -16
- package/dist/src/importers/jungle.js +145 -32
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +108 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -77
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
package/dist/src/pdf/errors.js
CHANGED
|
@@ -8,6 +8,17 @@ export function pdfFailure(value, requestId) {
|
|
|
8
8
|
return failure(requestId, 'unclassified');
|
|
9
9
|
if (record.captureError)
|
|
10
10
|
return upstreamFailure(value, requestId);
|
|
11
|
-
const codes = {
|
|
11
|
+
const codes = {
|
|
12
|
+
'invalid-input': 'invalid-input',
|
|
13
|
+
encrypted: 'unsupported',
|
|
14
|
+
'limit-exceeded': 'limit-exceeded',
|
|
15
|
+
'tool-unavailable': 'runtime-missing',
|
|
16
|
+
'extraction-failed': 'unavailable',
|
|
17
|
+
timeout: 'unavailable',
|
|
18
|
+
cancelled: 'cancelled',
|
|
19
|
+
integrity: 'integrity',
|
|
20
|
+
'unsupported-schema': 'schema-newer',
|
|
21
|
+
unknown: 'unclassified',
|
|
22
|
+
};
|
|
12
23
|
return failure(requestId, typeof record.code === 'string' && Object.hasOwn(codes, record.code) ? codes[record.code] : 'unclassified');
|
|
13
24
|
}
|
|
@@ -2,7 +2,7 @@ import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
|
|
|
2
2
|
import { type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
|
|
3
3
|
import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
|
|
4
4
|
import type { LibraryStore, AttemptToken } from '../store/index.js';
|
|
5
|
-
|
|
5
|
+
interface PdfResult {
|
|
6
6
|
ok: true;
|
|
7
7
|
itemId: string;
|
|
8
8
|
revisionId: string;
|
|
@@ -22,3 +22,4 @@ export interface PdfOptions {
|
|
|
22
22
|
}
|
|
23
23
|
/** One attempt. Durable queue policy and process recovery live above this function. */
|
|
24
24
|
export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;
|
|
25
|
+
export {};
|
package/dist/src/pdf/handler.js
CHANGED
|
@@ -1,84 +1,83 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
1
|
import { join } from 'node:path';
|
|
3
2
|
import { validateCapture } from '../../contracts/validate.js';
|
|
4
3
|
import { capturePdf } from './capture.js';
|
|
5
|
-
import {
|
|
4
|
+
import { runAttempt, classifyErrors } from '../ingest/attempt.js';
|
|
5
|
+
import { sha256 } from '../util/hash.js';
|
|
6
6
|
import { bundleBytes, bundleText } from '../app/bundle-files.js';
|
|
7
7
|
import { selectedMetadata } from '../metadata/article.js';
|
|
8
8
|
import { localizeMarkdown } from '../app/markdown.js';
|
|
9
|
-
|
|
9
|
+
import { SpongeError } from '../errors.js';
|
|
10
|
+
const classify = classifyErrors([
|
|
11
|
+
'integrity',
|
|
12
|
+
'limit-exceeded',
|
|
13
|
+
'cancelled',
|
|
14
|
+
'invalid-input',
|
|
15
|
+
'owner-busy',
|
|
16
|
+
'unavailable',
|
|
17
|
+
]);
|
|
10
18
|
/** One attempt. Durable queue policy and process recovery live above this function. */
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
let cleanup = 'settled';
|
|
15
|
-
try {
|
|
16
|
-
const source = store.sourceForJob(token.jobId);
|
|
19
|
+
export function ingestPdf(store, token, options) {
|
|
20
|
+
return runAttempt(store, token, classify, async (attempt) => {
|
|
21
|
+
const source = store.queue.source(token.jobId);
|
|
17
22
|
if (!source || source.kind !== 'pdf')
|
|
18
|
-
throw
|
|
19
|
-
const result = await (options.capture ?? capturePdf)({
|
|
23
|
+
throw new SpongeError('invalid-input');
|
|
24
|
+
const result = await (options.capture ?? capturePdf)({
|
|
25
|
+
runProcess: options.runProcess,
|
|
26
|
+
runtime: options.runtime,
|
|
27
|
+
stagingRoot: join(store.root, 'staging'),
|
|
28
|
+
sourceUrl: source.locator,
|
|
29
|
+
requestId: token.attemptId,
|
|
30
|
+
connection: options.connection,
|
|
31
|
+
signal: options.signal,
|
|
32
|
+
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
33
|
+
});
|
|
20
34
|
if (!result.ok) {
|
|
21
|
-
|
|
22
|
-
|
|
35
|
+
if (result.code === 'cleanup-required')
|
|
36
|
+
attempt.cleanup = 'pending';
|
|
37
|
+
return result;
|
|
23
38
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
throw Object.assign(Error('cancelled'), { code: 'cancelled' });
|
|
44
|
-
}
|
|
45
|
-
const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
|
|
46
|
-
const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
|
|
47
|
-
const status = localized.missingImages ? 'partial' : capture.status;
|
|
48
|
-
// Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
|
|
49
|
-
const bundleHash = createHash('sha256').update(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256])).digest('hex');
|
|
50
|
-
await options.checkpoint?.('promoting');
|
|
51
|
-
const revision = store.commitRevision({ ...token, itemId: source.itemId, bundleHash, markdown: localized.markdown, title, extractor: capture.extractor, status, metadata: capture.metadata, assets, warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])] });
|
|
52
|
-
committed = true;
|
|
53
|
-
outcome = { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
|
|
39
|
+
attempt.hold(result);
|
|
40
|
+
await options.checkpoint?.('extracting');
|
|
41
|
+
const capture = result.capture;
|
|
42
|
+
validateCapture(capture);
|
|
43
|
+
if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
|
|
44
|
+
throw new SpongeError('integrity');
|
|
45
|
+
const original = await bundleText(result.bundleRoot, capture.markdown);
|
|
46
|
+
const allAssets = [...capture.assets, capture.sourcePdf];
|
|
47
|
+
const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
|
|
48
|
+
const assets = [];
|
|
49
|
+
for (const asset of allAssets) {
|
|
50
|
+
if (options.signal?.aborted)
|
|
51
|
+
throw new SpongeError('cancelled');
|
|
52
|
+
const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
|
|
53
|
+
assets.push({
|
|
54
|
+
...blob,
|
|
55
|
+
mediaType: asset.mediaType,
|
|
56
|
+
relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
|
|
57
|
+
});
|
|
54
58
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
outcome = failure(token.attemptId, code && known.has(code) ? code : 'unclassified', 'storage');
|
|
59
|
-
}
|
|
60
|
-
if (captured)
|
|
61
|
-
try {
|
|
62
|
-
await captured.dispose();
|
|
59
|
+
if (options.signal?.aborted) {
|
|
60
|
+
store.queue.cancel(token.jobId);
|
|
61
|
+
throw new SpongeError('cancelled');
|
|
63
62
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
return {
|
|
83
|
-
|
|
63
|
+
const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
|
|
64
|
+
const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
|
|
65
|
+
const status = localized.missingImages ? 'partial' : capture.status;
|
|
66
|
+
// Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
|
|
67
|
+
const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256]));
|
|
68
|
+
await options.checkpoint?.('promoting');
|
|
69
|
+
const revision = store.revisions.commit({
|
|
70
|
+
...token,
|
|
71
|
+
itemId: source.itemId,
|
|
72
|
+
bundleHash,
|
|
73
|
+
markdown: localized.markdown,
|
|
74
|
+
title,
|
|
75
|
+
extractor: capture.extractor,
|
|
76
|
+
status,
|
|
77
|
+
metadata: capture.metadata,
|
|
78
|
+
assets,
|
|
79
|
+
warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
|
|
80
|
+
});
|
|
81
|
+
return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
|
|
82
|
+
});
|
|
84
83
|
}
|
package/dist/src/pdf/index.d.ts
CHANGED
|
@@ -1,4 +1,2 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export { ingestPdf, type PdfOptions, type PdfOutcome, type PdfResult } from './handler.js';
|
|
3
|
-
export { pdfFailure } from './errors.js';
|
|
1
|
+
export { ingestPdf } from './handler.js';
|
|
4
2
|
export { pdfSource } from './source.js';
|
package/dist/src/pdf/index.js
CHANGED
package/dist/src/pdf/input.js
CHANGED
|
@@ -2,30 +2,31 @@ import { open } from 'node:fs/promises';
|
|
|
2
2
|
import { constants } from 'node:fs';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { limits } from '../../contracts/index.js';
|
|
5
|
+
import { SpongeError } from '../errors.js';
|
|
5
6
|
/** Copy a bounded stable regular file; never pass the user's source to a converter. */
|
|
6
7
|
export async function copyPdfInput(source, target, signal) {
|
|
7
8
|
const input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
8
9
|
try {
|
|
9
10
|
const before = await input.stat();
|
|
10
11
|
if (!before.isFile())
|
|
11
|
-
throw
|
|
12
|
+
throw new SpongeError('invalid-input');
|
|
12
13
|
if (before.size > limits.pdfBytes)
|
|
13
|
-
throw
|
|
14
|
+
throw new SpongeError('limit-exceeded');
|
|
14
15
|
const output = await open(target, 'wx', 0o600), hash = createHash('sha256');
|
|
15
16
|
let total = 0;
|
|
16
17
|
try {
|
|
17
18
|
const buffer = Buffer.alloc(1024 ** 2);
|
|
18
19
|
for (;;) {
|
|
19
20
|
if (signal?.aborted)
|
|
20
|
-
throw
|
|
21
|
+
throw new SpongeError('cancelled');
|
|
21
22
|
const { bytesRead } = await input.read(buffer, 0, buffer.length, null);
|
|
22
23
|
if (!bytesRead)
|
|
23
24
|
break;
|
|
24
25
|
total += bytesRead;
|
|
25
26
|
if (total > limits.pdfBytes)
|
|
26
|
-
throw
|
|
27
|
+
throw new SpongeError('limit-exceeded');
|
|
27
28
|
if (total === bytesRead && buffer.subarray(0, 5).toString() !== '%PDF-')
|
|
28
|
-
throw
|
|
29
|
+
throw new SpongeError('invalid-input');
|
|
29
30
|
const bytes = buffer.subarray(0, bytesRead);
|
|
30
31
|
hash.update(bytes);
|
|
31
32
|
await output.writeFile(bytes);
|
|
@@ -36,8 +37,12 @@ export async function copyPdfInput(source, target, signal) {
|
|
|
36
37
|
await output.close();
|
|
37
38
|
}
|
|
38
39
|
const after = await input.stat();
|
|
39
|
-
if (total < 5 ||
|
|
40
|
-
|
|
40
|
+
if (total < 5 ||
|
|
41
|
+
total !== before.size ||
|
|
42
|
+
after.size !== before.size ||
|
|
43
|
+
after.mtimeMs !== before.mtimeMs ||
|
|
44
|
+
after.ctimeMs !== before.ctimeMs)
|
|
45
|
+
throw new SpongeError('integrity');
|
|
41
46
|
return hash.digest('hex');
|
|
42
47
|
}
|
|
43
48
|
finally {
|
|
@@ -1,64 +1,123 @@
|
|
|
1
1
|
import { limits } from '../../contracts/index.js';
|
|
2
2
|
import { validateCapture, validateAsset } from '../../contracts/validate.js';
|
|
3
|
-
import { readConfined
|
|
3
|
+
import { readConfined } from '../acquisition/files.js';
|
|
4
|
+
import { sha256 as hash } from '../util/hash.js';
|
|
4
5
|
import { object } from '../acquisition/normalize.js';
|
|
6
|
+
import { SpongeError } from '../errors.js';
|
|
5
7
|
export async function normalizePdfBundle(input) {
|
|
6
8
|
const verified = object(input.inspection), inspection = object(verified.inspection), summary = object(input.summary);
|
|
7
9
|
if (verified.ok !== true || inspection.schemaVersion !== 1 || !Array.isArray(inspection.assets))
|
|
8
|
-
throw
|
|
10
|
+
throw new SpongeError('integrity');
|
|
9
11
|
const manifestBytes = await readConfined(input.bundleRoot, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8')));
|
|
10
|
-
if (manifest.schemaVersion !== 1 ||
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
if (manifest.schemaVersion !== 1 ||
|
|
13
|
+
manifest.kind !== 'pdf' ||
|
|
14
|
+
!['complete', 'partial'].includes(String(manifest.status)))
|
|
15
|
+
throw new SpongeError('integrity');
|
|
16
|
+
function asset(raw, mediaType) {
|
|
17
|
+
const value = object(raw);
|
|
18
|
+
const result = {
|
|
19
|
+
relativePath: String(value.path),
|
|
20
|
+
bytes: Number(value.bytes),
|
|
21
|
+
sha256: String(value.sha256),
|
|
22
|
+
mediaType,
|
|
23
|
+
};
|
|
24
|
+
validateAsset(result);
|
|
25
|
+
return result;
|
|
26
|
+
}
|
|
27
|
+
function assertVerified(value, checked) {
|
|
28
|
+
const record = object(checked);
|
|
29
|
+
if (record.integrity !== 'verified' ||
|
|
30
|
+
record.path !== value.relativePath ||
|
|
31
|
+
record.sha256 !== value.sha256 ||
|
|
32
|
+
record.bytes !== value.bytes)
|
|
33
|
+
throw new SpongeError('integrity');
|
|
34
|
+
}
|
|
15
35
|
const markdown = asset(manifest.markdown, 'text/markdown'), sourcePdf = asset(manifest.source, 'application/pdf');
|
|
16
36
|
assertVerified(markdown, inspection.markdown);
|
|
17
37
|
assertVerified(sourcePdf, inspection.source);
|
|
18
38
|
if (input.sourceHash && sourcePdf.sha256 !== input.sourceHash)
|
|
19
|
-
throw
|
|
39
|
+
throw new SpongeError('integrity');
|
|
20
40
|
const source = object(manifest.source);
|
|
21
|
-
if (!input.sourceHash &&
|
|
22
|
-
|
|
41
|
+
if (!input.sourceHash &&
|
|
42
|
+
(typeof verified.expectedSourceUrl !== 'string' || source.requestedUrl !== verified.expectedSourceUrl))
|
|
43
|
+
throw new SpongeError('integrity');
|
|
23
44
|
const markdownBytes = await readConfined(input.bundleRoot, markdown.relativePath, limits.markdownBytes);
|
|
24
45
|
if (hash(markdownBytes) !== markdown.sha256 || markdownBytes.length !== markdown.bytes)
|
|
25
|
-
throw
|
|
46
|
+
throw new SpongeError('integrity');
|
|
26
47
|
if (!Array.isArray(manifest.images) || manifest.images.length > limits.imagesPerItem)
|
|
27
|
-
throw
|
|
48
|
+
throw new SpongeError('limit-exceeded');
|
|
28
49
|
const byPath = new Map();
|
|
29
50
|
for (const raw of manifest.images) {
|
|
30
51
|
const image = object(raw), record = object(image.asset), value = asset(record, String(record.mimeType));
|
|
31
52
|
if (!value.mediaType.startsWith('image/'))
|
|
32
|
-
throw
|
|
33
|
-
assertVerified(value, inspection.assets.find(raw => object(raw).path === value.relativePath));
|
|
53
|
+
throw new SpongeError('integrity');
|
|
54
|
+
assertVerified(value, inspection.assets.find((raw) => object(raw).path === value.relativePath));
|
|
34
55
|
const prior = byPath.get(value.relativePath);
|
|
35
56
|
if (prior && JSON.stringify(prior) !== JSON.stringify(value))
|
|
36
|
-
throw
|
|
57
|
+
throw new SpongeError('integrity');
|
|
37
58
|
byPath.set(value.relativePath, value);
|
|
38
59
|
}
|
|
39
60
|
if (byPath.size !== inspection.assets.length)
|
|
40
|
-
throw
|
|
61
|
+
throw new SpongeError('integrity');
|
|
41
62
|
const metadata = [], document = object(manifest.document), observedAt = String(manifest.capturedAt);
|
|
42
63
|
for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
|
|
43
64
|
const value = document[field];
|
|
44
65
|
if (typeof value === 'string' && value.length > 0 && value.length <= 16384)
|
|
45
66
|
metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
|
|
46
67
|
}
|
|
47
|
-
metadata.push({
|
|
68
|
+
metadata.push({
|
|
69
|
+
field: 'pageCount',
|
|
70
|
+
value: String(document.pageCount),
|
|
71
|
+
source: 'pdf',
|
|
72
|
+
observedAt,
|
|
73
|
+
rule: 'wordcell-pdf-v1',
|
|
74
|
+
});
|
|
48
75
|
if (manifest.quality) {
|
|
49
76
|
const quality = object(manifest.quality);
|
|
50
77
|
if (Array.isArray(quality.issues)) {
|
|
51
|
-
metadata.push({
|
|
78
|
+
metadata.push({
|
|
79
|
+
field: 'pdfQualityIssueCount',
|
|
80
|
+
value: String(quality.issues.length),
|
|
81
|
+
source: 'pdf',
|
|
82
|
+
observedAt,
|
|
83
|
+
rule: 'wordcell-pdf-v1',
|
|
84
|
+
});
|
|
52
85
|
if (quality.issues.length > 400)
|
|
53
|
-
metadata.push({
|
|
86
|
+
metadata.push({
|
|
87
|
+
field: 'pdfQualityIssuesTruncated',
|
|
88
|
+
value: 'true',
|
|
89
|
+
source: 'pdf',
|
|
90
|
+
observedAt,
|
|
91
|
+
rule: 'wordcell-pdf-v1',
|
|
92
|
+
});
|
|
54
93
|
for (const issue of quality.issues.slice(0, 400)) {
|
|
55
94
|
const row = object(issue);
|
|
56
|
-
metadata.push({
|
|
95
|
+
metadata.push({
|
|
96
|
+
field: 'pdfQuality',
|
|
97
|
+
value: JSON.stringify({ code: row.code, page: row.page }),
|
|
98
|
+
source: 'pdf',
|
|
99
|
+
observedAt,
|
|
100
|
+
rule: 'wordcell-pdf-v1',
|
|
101
|
+
});
|
|
57
102
|
}
|
|
58
103
|
}
|
|
59
104
|
}
|
|
60
105
|
const partial = manifest.status !== 'complete' || inspection.status !== 'complete' || summary.status !== 'complete';
|
|
61
|
-
const result = {
|
|
106
|
+
const result = {
|
|
107
|
+
schemaVersion: 1,
|
|
108
|
+
requestId: input.requestId,
|
|
109
|
+
kind: 'pdf',
|
|
110
|
+
status: partial ? 'partial' : 'complete',
|
|
111
|
+
sourceUrl: input.sourceUrl,
|
|
112
|
+
markdown,
|
|
113
|
+
sourcePdf,
|
|
114
|
+
assets: [...byPath.values()],
|
|
115
|
+
metadata,
|
|
116
|
+
warnings: partial ? ['partial'] : [],
|
|
117
|
+
extractor: { name: 'wordcell', version: input.wordcellVersion },
|
|
118
|
+
manifestSha256: hash(manifestBytes),
|
|
119
|
+
completeness: { text: partial ? 'partial' : 'complete', images: partial ? 'partial' : 'complete' },
|
|
120
|
+
};
|
|
62
121
|
validateCapture(result);
|
|
63
122
|
return result;
|
|
64
123
|
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import { SpongeError } from '../errors.js';
|
|
1
2
|
/** Local PDFs are bound to copied bytes; only remote sources use Wordcell's URL sanitizer. */
|
|
2
3
|
export function pdfSourceIdentity(sourceUrl, sanitize) {
|
|
3
4
|
const url = new URL(sourceUrl);
|
|
4
5
|
if (url.protocol === 'file:') {
|
|
5
6
|
if (url.host || url.username || url.password || url.search || url.hash)
|
|
6
|
-
throw
|
|
7
|
+
throw new SpongeError('integrity');
|
|
7
8
|
return {};
|
|
8
9
|
}
|
|
9
10
|
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password)
|
|
10
|
-
throw
|
|
11
|
+
throw new SpongeError('integrity');
|
|
11
12
|
return { expectedSourceUrl: sanitize(sourceUrl) };
|
|
12
13
|
}
|
package/dist/src/pdf/source.js
CHANGED
|
@@ -1,22 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { sha256 } from '../util/hash.js';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
3
|
import { pathToFileURL } from 'node:url';
|
|
4
|
-
import {
|
|
4
|
+
import { SpongeError } from '../errors.js';
|
|
5
5
|
/** PDF identity includes its kind and account realm. Local files stay separate from URLs. */
|
|
6
6
|
export function pdfSource(input, realm = 'public') {
|
|
7
|
-
if (!input ||
|
|
8
|
-
|
|
7
|
+
if (!input ||
|
|
8
|
+
Buffer.byteLength(input) > 8192 ||
|
|
9
|
+
!realm ||
|
|
10
|
+
Buffer.byteLength(realm) > 256 ||
|
|
11
|
+
/[\u0000-\u001f\u007f]/u.test(input + realm))
|
|
12
|
+
throw new SpongeError('invalid-input');
|
|
9
13
|
let url;
|
|
10
14
|
try {
|
|
11
15
|
url = /^[a-z][a-z0-9+.-]*:/i.test(input) ? new URL(input) : pathToFileURL(resolve(input));
|
|
12
16
|
}
|
|
13
17
|
catch {
|
|
14
|
-
throw new
|
|
18
|
+
throw new SpongeError('invalid-input');
|
|
15
19
|
}
|
|
16
20
|
if (!['file:', 'https:', 'http:'].includes(url.protocol) || url.username || url.password)
|
|
17
|
-
throw new
|
|
21
|
+
throw new SpongeError('invalid-input');
|
|
18
22
|
if (url.protocol === 'file:' && (url.host || url.hash || url.search || realm !== 'public'))
|
|
19
|
-
throw new
|
|
20
|
-
const locator = url.href, sourceKey =
|
|
23
|
+
throw new SpongeError('invalid-input');
|
|
24
|
+
const locator = url.href, sourceKey = sha256(JSON.stringify(['pdf', realm, locator]));
|
|
21
25
|
return { sourceKey, locator, realm };
|
|
22
26
|
}
|
|
@@ -1,14 +1,26 @@
|
|
|
1
1
|
// Public PDF verifier, executed only by the integrity-checked pinned Bun runtime.
|
|
2
2
|
import { pdfSourceIdentity } from './source-identity.js';
|
|
3
|
+
import { SpongeError } from '../errors.js';
|
|
3
4
|
const runtime = process.argv[2], bundle = process.argv[3];
|
|
4
5
|
try {
|
|
5
6
|
const { sanitizeArtifactUrl } = await import(import.meta.resolve('@hraness/wordcell/clip/persist', `${runtime}/package.json`));
|
|
6
7
|
const { verifyPdfCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/pdf', `${runtime}/package.json`));
|
|
7
|
-
const result = await verifyPdfCaptureBundle(bundle, {
|
|
8
|
+
const result = await verifyPdfCaptureBundle(bundle, {
|
|
9
|
+
maxDocumentBytes: 16 * 1024 ** 2,
|
|
10
|
+
maxSourceBytes: 100 * 1024 ** 2,
|
|
11
|
+
maxVerifiedAssets: 200,
|
|
12
|
+
maxAssetBytes: 20 * 1024 ** 2,
|
|
13
|
+
maxTotalAssetBytes: 500 * 1024 ** 2,
|
|
14
|
+
maxVerificationMs: 30000,
|
|
15
|
+
});
|
|
8
16
|
const { markdown, ...document } = result.inspection.markdown;
|
|
9
|
-
const output = JSON.stringify({
|
|
17
|
+
const output = JSON.stringify({
|
|
18
|
+
...result,
|
|
19
|
+
...pdfSourceIdentity(process.argv[4], sanitizeArtifactUrl),
|
|
20
|
+
inspection: { ...result.inspection, markdown: document },
|
|
21
|
+
});
|
|
10
22
|
if (Buffer.byteLength(output) > 65536)
|
|
11
|
-
throw
|
|
23
|
+
throw new SpongeError('limit-exceeded');
|
|
12
24
|
process.stdout.write(output);
|
|
13
25
|
}
|
|
14
26
|
catch {
|
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
import type { LibraryStore } from '../store/index.js';
|
|
2
|
-
import { type FileRecord } from './files.js';
|
|
3
|
-
export interface BackupManifest {
|
|
4
|
-
format: 'sponge-backup';
|
|
5
|
-
version: 1;
|
|
6
|
-
schemaVersion: number;
|
|
7
|
-
libraryId: string;
|
|
8
|
-
revisionEpoch: number;
|
|
9
|
-
files: FileRecord[];
|
|
10
|
-
}
|
|
11
2
|
export declare function backupLibrary(store: LibraryStore, output: string): Promise<{
|
|
12
3
|
output: string;
|
|
13
4
|
sha256: string;
|
|
14
5
|
bytes: number;
|
|
15
6
|
itemsSnapshotEpoch: number;
|
|
7
|
+
publishKey: boolean;
|
|
16
8
|
}>;
|
|
17
9
|
/** Versioned length-framed archive: no tar links, executable hooks or path extraction. */
|
|
18
10
|
export declare function restoreLibrary(input: string, into: string): Promise<{
|
|
@@ -20,4 +12,5 @@ export declare function restoreLibrary(input: string, into: string): Promise<{
|
|
|
20
12
|
libraryId: string;
|
|
21
13
|
incarnation: string;
|
|
22
14
|
heldJobs: number;
|
|
15
|
+
publishKey: boolean;
|
|
23
16
|
}>;
|