@hraness/sponge 0.2.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +23 -10
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +39 -17
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +10 -46
  64. package/dist/src/digest/editorial.js +73 -59
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +53 -32
  67. package/dist/src/digest/pipeline.js +357 -125
  68. package/dist/src/digest/schema.d.ts +44 -17
  69. package/dist/src/digest/schema.js +132 -44
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +31 -24
  82. package/dist/src/importers/jungle-digest.js +217 -115
  83. package/dist/src/importers/jungle-store.d.ts +20 -11
  84. package/dist/src/importers/jungle-store.js +136 -41
  85. package/dist/src/importers/jungle.d.ts +32 -16
  86. package/dist/src/importers/jungle.js +145 -32
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +108 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -77
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -8,6 +8,17 @@ export function pdfFailure(value, requestId) {
8
8
  return failure(requestId, 'unclassified');
9
9
  if (record.captureError)
10
10
  return upstreamFailure(value, requestId);
11
- const codes = { 'invalid-input': 'invalid-input', encrypted: 'unsupported', 'limit-exceeded': 'limit-exceeded', 'tool-unavailable': 'runtime-missing', 'extraction-failed': 'unavailable', timeout: 'unavailable', cancelled: 'cancelled', integrity: 'integrity', 'unsupported-schema': 'schema-newer', unknown: 'unclassified' };
11
+ const codes = {
12
+ 'invalid-input': 'invalid-input',
13
+ encrypted: 'unsupported',
14
+ 'limit-exceeded': 'limit-exceeded',
15
+ 'tool-unavailable': 'runtime-missing',
16
+ 'extraction-failed': 'unavailable',
17
+ timeout: 'unavailable',
18
+ cancelled: 'cancelled',
19
+ integrity: 'integrity',
20
+ 'unsupported-schema': 'schema-newer',
21
+ unknown: 'unclassified',
22
+ };
12
23
  return failure(requestId, typeof record.code === 'string' && Object.hasOwn(codes, record.code) ? codes[record.code] : 'unclassified');
13
24
  }
@@ -2,7 +2,7 @@ import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
2
2
  import { type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
3
3
  import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
4
4
  import type { LibraryStore, AttemptToken } from '../store/index.js';
5
- export interface PdfResult {
5
+ interface PdfResult {
6
6
  ok: true;
7
7
  itemId: string;
8
8
  revisionId: string;
@@ -22,3 +22,4 @@ export interface PdfOptions {
22
22
  }
23
23
  /** One attempt. Durable queue policy and process recovery live above this function. */
24
24
  export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;
25
+ export {};
@@ -1,84 +1,83 @@
1
- import { createHash } from 'node:crypto';
2
1
  import { join } from 'node:path';
3
2
  import { validateCapture } from '../../contracts/validate.js';
4
3
  import { capturePdf } from './capture.js';
5
- import { failure } from '../acquisition/errors.js';
4
+ import { runAttempt, classifyErrors } from '../ingest/attempt.js';
5
+ import { sha256 } from '../util/hash.js';
6
6
  import { bundleBytes, bundleText } from '../app/bundle-files.js';
7
7
  import { selectedMetadata } from '../metadata/article.js';
8
8
  import { localizeMarkdown } from '../app/markdown.js';
9
- const known = new Set(['integrity', 'limit-exceeded', 'cancelled', 'invalid-input', 'owner-busy', 'unavailable']);
9
+ import { SpongeError } from '../errors.js';
10
+ const classify = classifyErrors([
11
+ 'integrity',
12
+ 'limit-exceeded',
13
+ 'cancelled',
14
+ 'invalid-input',
15
+ 'owner-busy',
16
+ 'unavailable',
17
+ ]);
10
18
  /** One attempt. Durable queue policy and process recovery live above this function. */
11
- export async function ingestPdf(store, token, options) {
12
- let captured, committed = false;
13
- let outcome = failure(token.attemptId, 'unclassified', 'storage');
14
- let cleanup = 'settled';
15
- try {
16
- const source = store.sourceForJob(token.jobId);
19
+ export function ingestPdf(store, token, options) {
20
+ return runAttempt(store, token, classify, async (attempt) => {
21
+ const source = store.queue.source(token.jobId);
17
22
  if (!source || source.kind !== 'pdf')
18
- throw Object.assign(Error('unsupported'), { code: 'invalid-input' });
19
- const result = await (options.capture ?? capturePdf)({ runProcess: options.runProcess, runtime: options.runtime, stagingRoot: join(store.root, 'staging'), sourceUrl: source.locator, requestId: token.attemptId, connection: options.connection, signal: options.signal, allowPrivateNetwork: options.allowPrivateNetwork });
23
+ throw new SpongeError('invalid-input');
24
+ const result = await (options.capture ?? capturePdf)({
25
+ runProcess: options.runProcess,
26
+ runtime: options.runtime,
27
+ stagingRoot: join(store.root, 'staging'),
28
+ sourceUrl: source.locator,
29
+ requestId: token.attemptId,
30
+ connection: options.connection,
31
+ signal: options.signal,
32
+ allowPrivateNetwork: options.allowPrivateNetwork,
33
+ });
20
34
  if (!result.ok) {
21
- outcome = result;
22
- cleanup = result.code === 'cleanup-required' ? 'pending' : 'settled';
35
+ if (result.code === 'cleanup-required')
36
+ attempt.cleanup = 'pending';
37
+ return result;
23
38
  }
24
- else {
25
- captured = result;
26
- await options.checkpoint?.('extracting');
27
- const capture = result.capture;
28
- validateCapture(capture);
29
- if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
30
- throw Object.assign(Error('integrity'), { code: 'integrity' });
31
- const original = await bundleText(result.bundleRoot, capture.markdown);
32
- const allAssets = [...capture.assets, capture.sourcePdf];
33
- const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
34
- const assets = [];
35
- for (const asset of allAssets) {
36
- if (options.signal?.aborted)
37
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
38
- const blob = await store.putBlob(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
39
- assets.push({ ...blob, mediaType: asset.mediaType, relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image' });
40
- }
41
- if (options.signal?.aborted) {
42
- store.requestCancellation(token.jobId);
43
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
44
- }
45
- const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
46
- const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
47
- const status = localized.missingImages ? 'partial' : capture.status;
48
- // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
49
- const bundleHash = createHash('sha256').update(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256])).digest('hex');
50
- await options.checkpoint?.('promoting');
51
- const revision = store.commitRevision({ ...token, itemId: source.itemId, bundleHash, markdown: localized.markdown, title, extractor: capture.extractor, status, metadata: capture.metadata, assets, warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])] });
52
- committed = true;
53
- outcome = { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
39
+ attempt.hold(result);
40
+ await options.checkpoint?.('extracting');
41
+ const capture = result.capture;
42
+ validateCapture(capture);
43
+ if (capture.kind !== 'pdf' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
44
+ throw new SpongeError('integrity');
45
+ const original = await bundleText(result.bundleRoot, capture.markdown);
46
+ const allAssets = [...capture.assets, capture.sourcePdf];
47
+ const localized = localizeMarkdown(original.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1'), allAssets);
48
+ const assets = [];
49
+ for (const asset of allAssets) {
50
+ if (options.signal?.aborted)
51
+ throw new SpongeError('cancelled');
52
+ const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
53
+ assets.push({
54
+ ...blob,
55
+ mediaType: asset.mediaType,
56
+ relationship: asset === capture.sourcePdf ? 'source-pdf' : 'image',
57
+ });
54
58
  }
55
- }
56
- catch (error) {
57
- const code = error?.code;
58
- outcome = failure(token.attemptId, code && known.has(code) ? code : 'unclassified', 'storage');
59
- }
60
- if (captured)
61
- try {
62
- await captured.dispose();
59
+ if (options.signal?.aborted) {
60
+ store.queue.cancel(token.jobId);
61
+ throw new SpongeError('cancelled');
63
62
  }
64
- catch {
65
- cleanup = 'pending';
66
- }
67
- if (!committed)
68
- try {
69
- store.failAttempt(token, outcome.ok ? 'unclassified' : outcome.code, outcome.ok ? {} : { retryable: outcome.retryable, retryAfterMs: outcome.retryAfterMs });
70
- }
71
- catch {
72
- cleanup = 'pending';
73
- }
74
- if (cleanup === 'settled')
75
- try {
76
- store.settleAttempt(token);
77
- }
78
- catch {
79
- cleanup = 'pending';
80
- }
81
- if (cleanup === 'pending')
82
- return { ...failure(token.attemptId, 'cleanup-required', 'cleanup'), cleanup };
83
- return { ...outcome, cleanup };
63
+ const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
64
+ const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
65
+ const status = localized.missingImages ? 'partial' : capture.status;
66
+ // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
67
+ const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', capture.manifestSha256]));
68
+ await options.checkpoint?.('promoting');
69
+ const revision = store.revisions.commit({
70
+ ...token,
71
+ itemId: source.itemId,
72
+ bundleHash,
73
+ markdown: localized.markdown,
74
+ title,
75
+ extractor: capture.extractor,
76
+ status,
77
+ metadata: capture.metadata,
78
+ assets,
79
+ warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
80
+ });
81
+ return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
82
+ });
84
83
  }
@@ -1,4 +1,2 @@
1
- export { capturePdf, type PdfCaptureRequest, type CaptureOutcome } from './capture.js';
2
- export { ingestPdf, type PdfOptions, type PdfOutcome, type PdfResult } from './handler.js';
3
- export { pdfFailure } from './errors.js';
1
+ export { ingestPdf } from './handler.js';
4
2
  export { pdfSource } from './source.js';
@@ -1,4 +1,2 @@
1
- export { capturePdf } from './capture.js';
2
1
  export { ingestPdf } from './handler.js';
3
- export { pdfFailure } from './errors.js';
4
2
  export { pdfSource } from './source.js';
@@ -2,30 +2,31 @@ import { open } from 'node:fs/promises';
2
2
  import { constants } from 'node:fs';
3
3
  import { createHash } from 'node:crypto';
4
4
  import { limits } from '../../contracts/index.js';
5
+ import { SpongeError } from '../errors.js';
5
6
  /** Copy a bounded stable regular file; never pass the user's source to a converter. */
6
7
  export async function copyPdfInput(source, target, signal) {
7
8
  const input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
8
9
  try {
9
10
  const before = await input.stat();
10
11
  if (!before.isFile())
11
- throw Error('invalid-input');
12
+ throw new SpongeError('invalid-input');
12
13
  if (before.size > limits.pdfBytes)
13
- throw Error('limit-exceeded');
14
+ throw new SpongeError('limit-exceeded');
14
15
  const output = await open(target, 'wx', 0o600), hash = createHash('sha256');
15
16
  let total = 0;
16
17
  try {
17
18
  const buffer = Buffer.alloc(1024 ** 2);
18
19
  for (;;) {
19
20
  if (signal?.aborted)
20
- throw Error('cancelled');
21
+ throw new SpongeError('cancelled');
21
22
  const { bytesRead } = await input.read(buffer, 0, buffer.length, null);
22
23
  if (!bytesRead)
23
24
  break;
24
25
  total += bytesRead;
25
26
  if (total > limits.pdfBytes)
26
- throw Error('limit-exceeded');
27
+ throw new SpongeError('limit-exceeded');
27
28
  if (total === bytesRead && buffer.subarray(0, 5).toString() !== '%PDF-')
28
- throw Error('invalid-input');
29
+ throw new SpongeError('invalid-input');
29
30
  const bytes = buffer.subarray(0, bytesRead);
30
31
  hash.update(bytes);
31
32
  await output.writeFile(bytes);
@@ -36,8 +37,12 @@ export async function copyPdfInput(source, target, signal) {
36
37
  await output.close();
37
38
  }
38
39
  const after = await input.stat();
39
- if (total < 5 || total !== before.size || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
40
- throw Error('integrity');
40
+ if (total < 5 ||
41
+ total !== before.size ||
42
+ after.size !== before.size ||
43
+ after.mtimeMs !== before.mtimeMs ||
44
+ after.ctimeMs !== before.ctimeMs)
45
+ throw new SpongeError('integrity');
41
46
  return hash.digest('hex');
42
47
  }
43
48
  finally {
@@ -1,64 +1,123 @@
1
1
  import { limits } from '../../contracts/index.js';
2
2
  import { validateCapture, validateAsset } from '../../contracts/validate.js';
3
- import { readConfined, hash } from '../acquisition/files.js';
3
+ import { readConfined } from '../acquisition/files.js';
4
+ import { sha256 as hash } from '../util/hash.js';
4
5
  import { object } from '../acquisition/normalize.js';
6
+ import { SpongeError } from '../errors.js';
5
7
  export async function normalizePdfBundle(input) {
6
8
  const verified = object(input.inspection), inspection = object(verified.inspection), summary = object(input.summary);
7
9
  if (verified.ok !== true || inspection.schemaVersion !== 1 || !Array.isArray(inspection.assets))
8
- throw Error('integrity');
10
+ throw new SpongeError('integrity');
9
11
  const manifestBytes = await readConfined(input.bundleRoot, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8')));
10
- if (manifest.schemaVersion !== 1 || manifest.kind !== 'pdf' || !['complete', 'partial'].includes(String(manifest.status)))
11
- throw Error('integrity');
12
- function asset(raw, mediaType) { const value = object(raw); const result = { relativePath: String(value.path), bytes: Number(value.bytes), sha256: String(value.sha256), mediaType }; validateAsset(result); return result; }
13
- function assertVerified(value, checked) { const record = object(checked); if (record.integrity !== 'verified' || record.path !== value.relativePath || record.sha256 !== value.sha256 || record.bytes !== value.bytes)
14
- throw Error('integrity'); }
12
+ if (manifest.schemaVersion !== 1 ||
13
+ manifest.kind !== 'pdf' ||
14
+ !['complete', 'partial'].includes(String(manifest.status)))
15
+ throw new SpongeError('integrity');
16
+ function asset(raw, mediaType) {
17
+ const value = object(raw);
18
+ const result = {
19
+ relativePath: String(value.path),
20
+ bytes: Number(value.bytes),
21
+ sha256: String(value.sha256),
22
+ mediaType,
23
+ };
24
+ validateAsset(result);
25
+ return result;
26
+ }
27
+ function assertVerified(value, checked) {
28
+ const record = object(checked);
29
+ if (record.integrity !== 'verified' ||
30
+ record.path !== value.relativePath ||
31
+ record.sha256 !== value.sha256 ||
32
+ record.bytes !== value.bytes)
33
+ throw new SpongeError('integrity');
34
+ }
15
35
  const markdown = asset(manifest.markdown, 'text/markdown'), sourcePdf = asset(manifest.source, 'application/pdf');
16
36
  assertVerified(markdown, inspection.markdown);
17
37
  assertVerified(sourcePdf, inspection.source);
18
38
  if (input.sourceHash && sourcePdf.sha256 !== input.sourceHash)
19
- throw Error('integrity');
39
+ throw new SpongeError('integrity');
20
40
  const source = object(manifest.source);
21
- if (!input.sourceHash && (typeof verified.expectedSourceUrl !== 'string' || source.requestedUrl !== verified.expectedSourceUrl))
22
- throw Error('integrity');
41
+ if (!input.sourceHash &&
42
+ (typeof verified.expectedSourceUrl !== 'string' || source.requestedUrl !== verified.expectedSourceUrl))
43
+ throw new SpongeError('integrity');
23
44
  const markdownBytes = await readConfined(input.bundleRoot, markdown.relativePath, limits.markdownBytes);
24
45
  if (hash(markdownBytes) !== markdown.sha256 || markdownBytes.length !== markdown.bytes)
25
- throw Error('integrity');
46
+ throw new SpongeError('integrity');
26
47
  if (!Array.isArray(manifest.images) || manifest.images.length > limits.imagesPerItem)
27
- throw Error('limit-exceeded');
48
+ throw new SpongeError('limit-exceeded');
28
49
  const byPath = new Map();
29
50
  for (const raw of manifest.images) {
30
51
  const image = object(raw), record = object(image.asset), value = asset(record, String(record.mimeType));
31
52
  if (!value.mediaType.startsWith('image/'))
32
- throw Error('integrity');
33
- assertVerified(value, inspection.assets.find(raw => object(raw).path === value.relativePath));
53
+ throw new SpongeError('integrity');
54
+ assertVerified(value, inspection.assets.find((raw) => object(raw).path === value.relativePath));
34
55
  const prior = byPath.get(value.relativePath);
35
56
  if (prior && JSON.stringify(prior) !== JSON.stringify(value))
36
- throw Error('integrity');
57
+ throw new SpongeError('integrity');
37
58
  byPath.set(value.relativePath, value);
38
59
  }
39
60
  if (byPath.size !== inspection.assets.length)
40
- throw Error('integrity');
61
+ throw new SpongeError('integrity');
41
62
  const metadata = [], document = object(manifest.document), observedAt = String(manifest.capturedAt);
42
63
  for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
43
64
  const value = document[field];
44
65
  if (typeof value === 'string' && value.length > 0 && value.length <= 16384)
45
66
  metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
46
67
  }
47
- metadata.push({ field: 'pageCount', value: String(document.pageCount), source: 'pdf', observedAt, rule: 'wordcell-pdf-v1' });
68
+ metadata.push({
69
+ field: 'pageCount',
70
+ value: String(document.pageCount),
71
+ source: 'pdf',
72
+ observedAt,
73
+ rule: 'wordcell-pdf-v1',
74
+ });
48
75
  if (manifest.quality) {
49
76
  const quality = object(manifest.quality);
50
77
  if (Array.isArray(quality.issues)) {
51
- metadata.push({ field: 'pdfQualityIssueCount', value: String(quality.issues.length), source: 'pdf', observedAt, rule: 'wordcell-pdf-v1' });
78
+ metadata.push({
79
+ field: 'pdfQualityIssueCount',
80
+ value: String(quality.issues.length),
81
+ source: 'pdf',
82
+ observedAt,
83
+ rule: 'wordcell-pdf-v1',
84
+ });
52
85
  if (quality.issues.length > 400)
53
- metadata.push({ field: 'pdfQualityIssuesTruncated', value: 'true', source: 'pdf', observedAt, rule: 'wordcell-pdf-v1' });
86
+ metadata.push({
87
+ field: 'pdfQualityIssuesTruncated',
88
+ value: 'true',
89
+ source: 'pdf',
90
+ observedAt,
91
+ rule: 'wordcell-pdf-v1',
92
+ });
54
93
  for (const issue of quality.issues.slice(0, 400)) {
55
94
  const row = object(issue);
56
- metadata.push({ field: 'pdfQuality', value: JSON.stringify({ code: row.code, page: row.page }), source: 'pdf', observedAt, rule: 'wordcell-pdf-v1' });
95
+ metadata.push({
96
+ field: 'pdfQuality',
97
+ value: JSON.stringify({ code: row.code, page: row.page }),
98
+ source: 'pdf',
99
+ observedAt,
100
+ rule: 'wordcell-pdf-v1',
101
+ });
57
102
  }
58
103
  }
59
104
  }
60
105
  const partial = manifest.status !== 'complete' || inspection.status !== 'complete' || summary.status !== 'complete';
61
- const result = { schemaVersion: 1, requestId: input.requestId, kind: 'pdf', status: partial ? 'partial' : 'complete', sourceUrl: input.sourceUrl, markdown, sourcePdf, assets: [...byPath.values()], metadata, warnings: partial ? ['partial'] : [], extractor: { name: 'wordcell', version: input.wordcellVersion }, manifestSha256: hash(manifestBytes), completeness: { text: partial ? 'partial' : 'complete', images: partial ? 'partial' : 'complete' } };
106
+ const result = {
107
+ schemaVersion: 1,
108
+ requestId: input.requestId,
109
+ kind: 'pdf',
110
+ status: partial ? 'partial' : 'complete',
111
+ sourceUrl: input.sourceUrl,
112
+ markdown,
113
+ sourcePdf,
114
+ assets: [...byPath.values()],
115
+ metadata,
116
+ warnings: partial ? ['partial'] : [],
117
+ extractor: { name: 'wordcell', version: input.wordcellVersion },
118
+ manifestSha256: hash(manifestBytes),
119
+ completeness: { text: partial ? 'partial' : 'complete', images: partial ? 'partial' : 'complete' },
120
+ };
62
121
  validateCapture(result);
63
122
  return result;
64
123
  }
@@ -1,12 +1,13 @@
1
+ import { SpongeError } from '../errors.js';
1
2
  /** Local PDFs are bound to copied bytes; only remote sources use Wordcell's URL sanitizer. */
2
3
  export function pdfSourceIdentity(sourceUrl, sanitize) {
3
4
  const url = new URL(sourceUrl);
4
5
  if (url.protocol === 'file:') {
5
6
  if (url.host || url.username || url.password || url.search || url.hash)
6
- throw Error('integrity');
7
+ throw new SpongeError('integrity');
7
8
  return {};
8
9
  }
9
10
  if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password)
10
- throw Error('integrity');
11
+ throw new SpongeError('integrity');
11
12
  return { expectedSourceUrl: sanitize(sourceUrl) };
12
13
  }
@@ -1,22 +1,26 @@
1
- import { createHash } from 'node:crypto';
1
+ import { sha256 } from '../util/hash.js';
2
2
  import { resolve } from 'node:path';
3
3
  import { pathToFileURL } from 'node:url';
4
- import { InputError } from '../app/source.js';
4
+ import { SpongeError } from '../errors.js';
5
5
  /** PDF identity includes its kind and account realm. Local files stay separate from URLs. */
6
6
  export function pdfSource(input, realm = 'public') {
7
- if (!input || Buffer.byteLength(input) > 8192 || !realm || Buffer.byteLength(realm) > 256 || /[\u0000-\u001f\u007f]/u.test(input + realm))
8
- throw new InputError();
7
+ if (!input ||
8
+ Buffer.byteLength(input) > 8192 ||
9
+ !realm ||
10
+ Buffer.byteLength(realm) > 256 ||
11
+ /[\u0000-\u001f\u007f]/u.test(input + realm))
12
+ throw new SpongeError('invalid-input');
9
13
  let url;
10
14
  try {
11
15
  url = /^[a-z][a-z0-9+.-]*:/i.test(input) ? new URL(input) : pathToFileURL(resolve(input));
12
16
  }
13
17
  catch {
14
- throw new InputError();
18
+ throw new SpongeError('invalid-input');
15
19
  }
16
20
  if (!['file:', 'https:', 'http:'].includes(url.protocol) || url.username || url.password)
17
- throw new InputError();
21
+ throw new SpongeError('invalid-input');
18
22
  if (url.protocol === 'file:' && (url.host || url.hash || url.search || realm !== 'public'))
19
- throw new InputError();
20
- const locator = url.href, sourceKey = createHash('sha256').update(JSON.stringify(['pdf', realm, locator])).digest('hex');
23
+ throw new SpongeError('invalid-input');
24
+ const locator = url.href, sourceKey = sha256(JSON.stringify(['pdf', realm, locator]));
21
25
  return { sourceKey, locator, realm };
22
26
  }
@@ -1,14 +1,26 @@
1
1
  // Public PDF verifier, executed only by the integrity-checked pinned Bun runtime.
2
2
  import { pdfSourceIdentity } from './source-identity.js';
3
+ import { SpongeError } from '../errors.js';
3
4
  const runtime = process.argv[2], bundle = process.argv[3];
4
5
  try {
5
6
  const { sanitizeArtifactUrl } = await import(import.meta.resolve('@hraness/wordcell/clip/persist', `${runtime}/package.json`));
6
7
  const { verifyPdfCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/pdf', `${runtime}/package.json`));
7
- const result = await verifyPdfCaptureBundle(bundle, { maxDocumentBytes: 16 * 1024 ** 2, maxSourceBytes: 100 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxVerificationMs: 30000 });
8
+ const result = await verifyPdfCaptureBundle(bundle, {
9
+ maxDocumentBytes: 16 * 1024 ** 2,
10
+ maxSourceBytes: 100 * 1024 ** 2,
11
+ maxVerifiedAssets: 200,
12
+ maxAssetBytes: 20 * 1024 ** 2,
13
+ maxTotalAssetBytes: 500 * 1024 ** 2,
14
+ maxVerificationMs: 30000,
15
+ });
8
16
  const { markdown, ...document } = result.inspection.markdown;
9
- const output = JSON.stringify({ ...result, ...pdfSourceIdentity(process.argv[4], sanitizeArtifactUrl), inspection: { ...result.inspection, markdown: document } });
17
+ const output = JSON.stringify({
18
+ ...result,
19
+ ...pdfSourceIdentity(process.argv[4], sanitizeArtifactUrl),
20
+ inspection: { ...result.inspection, markdown: document },
21
+ });
10
22
  if (Buffer.byteLength(output) > 65536)
11
- throw Error('frame-limit');
23
+ throw new SpongeError('limit-exceeded');
12
24
  process.stdout.write(output);
13
25
  }
14
26
  catch {
@@ -1,18 +1,10 @@
1
1
  import type { LibraryStore } from '../store/index.js';
2
- import { type FileRecord } from './files.js';
3
- export interface BackupManifest {
4
- format: 'sponge-backup';
5
- version: 1;
6
- schemaVersion: number;
7
- libraryId: string;
8
- revisionEpoch: number;
9
- files: FileRecord[];
10
- }
11
2
  export declare function backupLibrary(store: LibraryStore, output: string): Promise<{
12
3
  output: string;
13
4
  sha256: string;
14
5
  bytes: number;
15
6
  itemsSnapshotEpoch: number;
7
+ publishKey: boolean;
16
8
  }>;
17
9
  /** Versioned length-framed archive: no tar links, executable hooks or path extraction. */
18
10
  export declare function restoreLibrary(input: string, into: string): Promise<{
@@ -20,4 +12,5 @@ export declare function restoreLibrary(input: string, into: string): Promise<{
20
12
  libraryId: string;
21
13
  incarnation: string;
22
14
  heldJobs: number;
15
+ publishKey: boolean;
23
16
  }>;