@hraness/sponge 0.2.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +23 -10
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +39 -17
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +10 -46
  64. package/dist/src/digest/editorial.js +73 -59
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +53 -32
  67. package/dist/src/digest/pipeline.js +357 -125
  68. package/dist/src/digest/schema.d.ts +44 -17
  69. package/dist/src/digest/schema.js +132 -44
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +31 -24
  82. package/dist/src/importers/jungle-digest.js +217 -115
  83. package/dist/src/importers/jungle-store.d.ts +20 -11
  84. package/dist/src/importers/jungle-store.js +136 -41
  85. package/dist/src/importers/jungle.d.ts +32 -16
  86. package/dist/src/importers/jungle.js +145 -32
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +108 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -77
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -1,19 +1,26 @@
1
+ import { SpongeError } from '../errors.js';
1
2
  // Executed only by the pinned Bun runtime. The import is Wordcell's public API.
2
3
  const runtime = process.argv[2], bundle = process.argv[3];
3
4
  try {
4
5
  const { verifyCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/clip/bundle-reader', `${runtime}/package.json`));
5
- const result = await verifyCaptureBundle(bundle, { verifyAssets: true, maxDocumentBytes: 16 * 1024 ** 2, maxVerifiedAssets: 200, maxAssetBytes: 20 * 1024 ** 2, maxTotalAssetBytes: 500 * 1024 ** 2, maxAssetVerificationMs: 30000 });
6
+ const result = await verifyCaptureBundle(bundle, {
7
+ verifyAssets: true,
8
+ maxDocumentBytes: 16 * 1024 ** 2,
9
+ maxVerifiedAssets: 200,
10
+ maxAssetBytes: 20 * 1024 ** 2,
11
+ maxTotalAssetBytes: 500 * 1024 ** 2,
12
+ maxAssetVerificationMs: 30000,
13
+ });
6
14
  // The document travels via the verified file, never a giant protocol frame.
7
15
  const { markdown: _, ...document } = result.inspection.document;
8
16
  // Metadata is validated upstream and read from the confined manifest by the parent.
9
17
  const { metadata: _metadata, ...inspection } = result.inspection;
10
18
  const output = JSON.stringify({ ...result, inspection: { ...inspection, document } });
11
19
  if (Buffer.byteLength(output) > 65536)
12
- throw Error('frame-limit');
20
+ throw new SpongeError('limit-exceeded');
13
21
  process.stdout.write(output);
14
22
  }
15
23
  catch {
16
24
  process.stdout.write(JSON.stringify({ ok: false }));
17
25
  process.exitCode = 1;
18
26
  }
19
- export {};
@@ -10,10 +10,13 @@ const escapable = /^[\\`*_{}[\]()#+\-.!|>~<"']$/;
10
10
  function fold(raw) {
11
11
  let text = '', space = false;
12
12
  const map = [], ends = [], clusters = [...graphemes.segment(raw)];
13
- const emit = (out, start, end) => { text += out; for (let k = 0; k < out.length; k++) {
14
- map.push(start);
15
- ends.push(end);
16
- } };
13
+ const emit = (out, start, end) => {
14
+ text += out;
15
+ for (let k = 0; k < out.length; k++) {
16
+ map.push(start);
17
+ ends.push(end);
18
+ }
19
+ };
17
20
  for (let c = 0; c < clusters.length; c++) {
18
21
  const { segment, index } = clusters[c];
19
22
  // An escaped character maps to its backslash, so a match starting there keeps the escape.
@@ -29,7 +32,11 @@ function fold(raw) {
29
32
  continue;
30
33
  }
31
34
  space = false;
32
- emit(ch.replace(/[‘’‚′]/g, "'").replace(/[“”„″]/g, '"').replace(/[\u2010-\u2015\u2212]/g, '-').normalize('NFC'), index, end);
35
+ emit(ch
36
+ .replace(/[‘’‚′]/g, "'")
37
+ .replace(/[“”„″]/g, '"')
38
+ .replace(/[\u2010-\u2015\u2212]/g, '-')
39
+ .normalize('NFC'), index, end);
33
40
  }
34
41
  if (text.endsWith(' ')) {
35
42
  text = text.slice(0, -1);
@@ -38,7 +45,9 @@ function fold(raw) {
38
45
  }
39
46
  return { text, map, ends };
40
47
  }
41
- export function foldText(value) { return fold(value).text; }
48
+ export function foldText(value) {
49
+ return fold(value).text;
50
+ }
42
51
  /** Every occurrence of `quote` in `source`, compared after folding. */
43
52
  export function locateAll(source, quote) {
44
53
  const s = fold(source), q = fold(quote).text, out = [];
@@ -47,14 +56,21 @@ export function locateAll(source, quote) {
47
56
  for (let at = s.text.indexOf(q); at >= 0; at = s.text.indexOf(q, at + 1)) {
48
57
  // A match must start and end on cluster boundaries; half a cluster has no raw offsets.
49
58
  const last = at + q.length - 1;
50
- if (at > 0 && s.map[at - 1] === s.map[at] || last + 1 < s.text.length && s.map[last + 1] === s.map[last])
59
+ if ((at > 0 && s.map[at - 1] === s.map[at]) || (last + 1 < s.text.length && s.map[last + 1] === s.map[last]))
51
60
  continue;
52
61
  out.push({ start: s.map[at], end: s.ends[last] });
53
62
  }
54
63
  return out;
55
64
  }
56
65
  export function textQuote(source, start, end) {
57
- return { type: 'text-quote', exact: source.slice(start, end), prefix: source.slice(Math.max(0, start - context), start), suffix: source.slice(end, end + context), start, end };
66
+ return {
67
+ type: 'text-quote',
68
+ exact: source.slice(start, end),
69
+ prefix: source.slice(Math.max(0, start - context), start),
70
+ suffix: source.slice(end, end + context),
71
+ start,
72
+ end,
73
+ };
58
74
  }
59
75
  /** The single match for `quote`, or undefined when absent or ambiguous. */
60
76
  export function locateQuote(source, quote) {
@@ -63,9 +79,16 @@ export function locateQuote(source, quote) {
63
79
  }
64
80
  /** A stored selector still names its exact text in `source`. */
65
81
  export function anchorHolds(source, anchor) {
66
- return Number.isSafeInteger(anchor.start) && Number.isSafeInteger(anchor.end) && anchor.start >= 0 && anchor.end > anchor.start && source.slice(anchor.start, anchor.end) === anchor.exact;
82
+ return (Number.isSafeInteger(anchor.start) &&
83
+ Number.isSafeInteger(anchor.end) &&
84
+ anchor.start >= 0 &&
85
+ anchor.end > anchor.start &&
86
+ source.slice(anchor.start, anchor.end) === anchor.exact);
87
+ }
88
+ export function words(value) {
89
+ const t = foldText(value);
90
+ return t ? t.split(' ').length : 0;
67
91
  }
68
- export function words(value) { const t = foldText(value); return t ? t.split(' ').length : 0; }
69
92
  /**
70
93
  * Rendered-text projection for snapping a near-verbatim quote: drops inline Markdown syntax (link targets, brackets,
71
94
  * emphasis, code ticks, line-start block markers), folds like `fold`, and lowercases. Offsets map back to raw text.
@@ -90,7 +113,7 @@ function rendered(raw) {
90
113
  continue;
91
114
  }
92
115
  }
93
- if (ch === '!' && raw[i + 1] === '[' || /[[\]*_`]/.test(ch))
116
+ if ((ch === '!' && raw[i + 1] === '[') || /[[\]*_`]/.test(ch))
94
117
  continue;
95
118
  if (ch === '\\' && i + 1 < raw.length && /[\\`*_{}[\]()#+\-.!|>~<"']/.test(raw[i + 1])) {
96
119
  ch = raw[++i];
@@ -105,7 +128,12 @@ function rendered(raw) {
105
128
  continue;
106
129
  }
107
130
  space = false;
108
- ch = ch.replace(/[‘’‚′]/g, "'").replace(/[“”„″]/g, '"').replace(/[‐-―−]/g, '-').normalize('NFC').toLowerCase();
131
+ ch = ch
132
+ .replace(/[‘’‚′]/g, "'")
133
+ .replace(/[“”„″]/g, '"')
134
+ .replace(/[‐-―−]/g, '-')
135
+ .normalize('NFC')
136
+ .toLowerCase();
109
137
  // One entry per output code unit: normalization and case mapping may change the length.
110
138
  text += ch;
111
139
  for (let k = 0; k < ch.length; k++) {
@@ -43,7 +43,6 @@ export declare function updateAnnotation(db: DatabaseSync, id: string, body: str
43
43
  export declare function deleteAnnotation(db: DatabaseSync, id: string): void;
44
44
  /** Reading state is one field per item; `updatedAt` travels with it. */
45
45
  export declare function setReadingState(db: DatabaseSync, itemId: string, state: unknown): void;
46
- export declare function readAnnotation(db: DatabaseSync, id: string): Annotation | undefined;
47
46
  /** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
48
47
  export declare function listAnnotations(db: DatabaseSync, itemId: string): (Annotation & {
49
48
  anchored?: boolean;
@@ -1,16 +1,23 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { anchorHolds, locateAll, textQuote } from './quote.js';
3
3
  import { mutate } from '../store/changes.js';
4
- import { StoreError } from '../store/types.js';
5
- import { digest, text } from '../store/files.js';
4
+ import { text } from '../store/files.js';
5
+ import { sha256 } from '../util/hash.js';
6
+ import { SpongeError } from '../errors.js';
6
7
  const now = () => new Date().toISOString();
7
- const fail = (code = 'invalid-input') => { throw new StoreError(code); };
8
+ const fail = (code = 'invalid-input') => {
9
+ throw new SpongeError(code);
10
+ };
8
11
  export function markdownFor(db, revisionId) {
9
- const row = db.prepare("SELECT markdown FROM representations WHERE revision_id=? AND kind='markdown'").get(revisionId);
12
+ const row = db
13
+ .prepare("SELECT markdown FROM representations WHERE revision_id=? AND kind='markdown'")
14
+ .get(revisionId);
10
15
  return typeof row?.markdown === 'string' ? row.markdown : fail('unavailable');
11
16
  }
12
17
  /** Create through the log with the full row image, so a replay rebuilds the same row. */
13
- function create(db, id, row) { mutate(db, 'annotation', id, 'created', row); }
18
+ function create(db, id, row) {
19
+ mutate(db, 'annotation', id, 'created', row);
20
+ }
14
21
  /**
15
22
  * Highlight `quote` in the item's current revision. An ambiguous quote needs `occurrence` (0-based),
16
23
  * so a highlight never silently lands on the wrong passage.
@@ -27,7 +34,15 @@ export function addHighlight(db, input) {
27
34
  if (!Number.isSafeInteger(index) || index < 0 || index >= matches.length)
28
35
  fail();
29
36
  const anchor = textQuote(source, matches[index].start, matches[index].end), id = randomUUID();
30
- create(db, id, { itemId: input.itemId, kind: 'highlight', revisionId, parentId: null, anchor, body: input.note ?? '', createdAt: now() });
37
+ create(db, id, {
38
+ itemId: input.itemId,
39
+ kind: 'highlight',
40
+ revisionId,
41
+ parentId: null,
42
+ anchor,
43
+ body: input.note ?? '',
44
+ createdAt: now(),
45
+ });
31
46
  return readAnnotation(db, id);
32
47
  }
33
48
  export function addComment(db, input) {
@@ -40,7 +55,15 @@ export function addComment(db, input) {
40
55
  fail('unavailable');
41
56
  }
42
57
  const id = randomUUID();
43
- create(db, id, { itemId: input.itemId, kind: 'comment', revisionId: null, parentId: input.parentId ?? null, anchor: null, body: input.body, createdAt: now() });
58
+ create(db, id, {
59
+ itemId: input.itemId,
60
+ kind: 'comment',
61
+ revisionId: null,
62
+ parentId: input.parentId ?? null,
63
+ anchor: null,
64
+ body: input.body,
65
+ createdAt: now(),
66
+ });
44
67
  return readAnnotation(db, id);
45
68
  }
46
69
  /**
@@ -48,14 +71,23 @@ export function addComment(db, input) {
48
71
  * once the user edits or deletes it, the user's choice wins and the import skips it.
49
72
  */
50
73
  export function importNote(db, input) {
51
- const id = digest(JSON.stringify(['import-note-v1', input.namespace, input.sourceKey, input.noteId]));
74
+ const id = sha256(JSON.stringify(['import-note-v1', input.namespace, input.sourceKey, input.noteId]));
52
75
  const prior = db.prepare('SELECT item_id,body,updated_at,deleted_at FROM annotations WHERE id=?').get(id);
53
76
  if (prior) {
54
- if (prior.item_id !== input.itemId || prior.body !== input.body && prior.updated_at === null && prior.deleted_at === null)
77
+ if (prior.item_id !== input.itemId ||
78
+ (prior.body !== input.body && prior.updated_at === null && prior.deleted_at === null))
55
79
  fail('integrity');
56
80
  return;
57
81
  }
58
- create(db, id, { itemId: input.itemId, kind: 'note', revisionId: input.revisionId, parentId: null, anchor: { kind: 'imported-note', sourceId: input.noteId }, body: input.body, createdAt: now() });
82
+ create(db, id, {
83
+ itemId: input.itemId,
84
+ kind: 'note',
85
+ revisionId: input.revisionId,
86
+ parentId: null,
87
+ anchor: { kind: 'imported-note', sourceId: input.noteId },
88
+ body: input.body,
89
+ createdAt: now(),
90
+ });
59
91
  }
60
92
  export function updateAnnotation(db, id, body) {
61
93
  text(body, 16384);
@@ -75,12 +107,32 @@ export function deleteAnnotation(db, id) {
75
107
  mutate(db, 'annotation', id, 'deleted', now());
76
108
  }
77
109
  /** Reading state is one field per item; `updatedAt` travels with it. */
78
- export function setReadingState(db, itemId, state) { mutate(db, 'reading', itemId, 'state', { state, updatedAt: now() }); }
79
- function annotationRow(r) { return { id: String(r.id), itemId: String(r.item_id), kind: r.kind, revisionId: r.revision_id === null ? null : String(r.revision_id), parentId: r.parent_id === null ? null : String(r.parent_id), anchor: JSON.parse(String(r.anchor_json)), body: String(r.body), createdAt: String(r.created_at), updatedAt: r.updated_at === null ? null : String(r.updated_at) }; }
80
- export function readAnnotation(db, id) { const r = db.prepare('SELECT * FROM annotations WHERE id=? AND deleted_at IS NULL').get(id); return r ? annotationRow(r) : undefined; }
110
+ export function setReadingState(db, itemId, state) {
111
+ mutate(db, 'reading', itemId, 'state', { state, updatedAt: now() });
112
+ }
113
+ function annotationRow(r) {
114
+ return {
115
+ id: String(r.id),
116
+ itemId: String(r.item_id),
117
+ kind: r.kind,
118
+ revisionId: r.revision_id === null ? null : String(r.revision_id),
119
+ parentId: r.parent_id === null ? null : String(r.parent_id),
120
+ anchor: JSON.parse(String(r.anchor_json)),
121
+ body: String(r.body),
122
+ createdAt: String(r.created_at),
123
+ updatedAt: r.updated_at === null ? null : String(r.updated_at),
124
+ };
125
+ }
126
+ function readAnnotation(db, id) {
127
+ const r = db.prepare('SELECT * FROM annotations WHERE id=? AND deleted_at IS NULL').get(id);
128
+ return r ? annotationRow(r) : undefined;
129
+ }
81
130
  /** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
82
131
  export function listAnnotations(db, itemId) {
83
- return db.prepare('SELECT * FROM annotations WHERE item_id=? AND deleted_at IS NULL ORDER BY created_at,rowid').all(itemId).map(r => {
132
+ return db
133
+ .prepare('SELECT * FROM annotations WHERE item_id=? AND deleted_at IS NULL ORDER BY created_at,rowid')
134
+ .all(itemId)
135
+ .map((r) => {
84
136
  const a = annotationRow(r);
85
137
  if (a.kind !== 'highlight' || !a.revisionId)
86
138
  return a;
@@ -1,7 +1,7 @@
1
1
  import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
2
2
  import { type CaptureRequest, type CaptureOutcome, type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
3
3
  import type { LibraryStore, AttemptToken } from '../store/index.js';
4
- export interface ArticleResult {
4
+ interface ArticleResult {
5
5
  ok: true;
6
6
  itemId: string;
7
7
  revisionId: string;
@@ -21,3 +21,4 @@ export interface ArticleOptions {
21
21
  }
22
22
  /** One attempt. Durable queue policy and process recovery live above this function. */
23
23
  export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;
24
+ export {};
@@ -1,82 +1,78 @@
1
- import { createHash } from 'node:crypto';
2
1
  import { join } from 'node:path';
3
2
  import { validateCapture } from '../../contracts/validate.js';
4
- import { captureArticle } from '../acquisition/index.js';
5
- import { failure } from '../acquisition/errors.js';
3
+ import { captureArticle, } from '../acquisition/index.js';
4
+ import { runAttempt, classifyErrors } from '../ingest/attempt.js';
5
+ import { sha256 } from '../util/hash.js';
6
6
  import { bundleBytes, bundleText } from './bundle-files.js';
7
7
  import { selectedMetadata } from '../metadata/article.js';
8
8
  import { localizeMarkdown } from './markdown.js';
9
- const known = new Set(['integrity', 'limit-exceeded', 'cancelled', 'invalid-input', 'owner-busy', 'unavailable']);
9
+ import { SpongeError } from '../errors.js';
10
+ const classify = classifyErrors([
11
+ 'integrity',
12
+ 'limit-exceeded',
13
+ 'cancelled',
14
+ 'invalid-input',
15
+ 'owner-busy',
16
+ 'unavailable',
17
+ ]);
10
18
  /** One attempt. Durable queue policy and process recovery live above this function. */
11
- export async function ingestArticle(store, token, options) {
12
- let captured, committed = false;
13
- let outcome = failure(token.attemptId, 'unclassified', 'storage');
14
- let cleanup = 'settled';
15
- try {
16
- const source = store.sourceForJob(token.jobId);
19
+ export function ingestArticle(store, token, options) {
20
+ return runAttempt(store, token, classify, async (attempt) => {
21
+ const source = store.queue.source(token.jobId);
17
22
  if (!source || source.kind !== 'article')
18
- throw Object.assign(Error('unsupported'), { code: 'invalid-input' });
19
- const result = await (options.capture ?? captureArticle)({ runProcess: options.runProcess, runtime: options.runtime, stagingRoot: join(store.root, 'staging'), sourceUrl: source.locator, requestId: token.attemptId, connection: options.connection, signal: options.signal, allowPrivateNetwork: options.allowPrivateNetwork });
23
+ throw new SpongeError('invalid-input');
24
+ const result = await (options.capture ?? captureArticle)({
25
+ runProcess: options.runProcess,
26
+ runtime: options.runtime,
27
+ stagingRoot: join(store.root, 'staging'),
28
+ sourceUrl: source.locator,
29
+ requestId: token.attemptId,
30
+ connection: options.connection,
31
+ signal: options.signal,
32
+ allowPrivateNetwork: options.allowPrivateNetwork,
33
+ });
20
34
  if (!result.ok) {
21
- outcome = result;
22
- cleanup = result.code === 'cleanup-required' ? 'pending' : 'settled';
35
+ if (result.code === 'cleanup-required')
36
+ attempt.cleanup = 'pending';
37
+ return result;
23
38
  }
24
- else {
25
- captured = result;
26
- await options.checkpoint?.('extracting');
27
- const capture = result.capture;
28
- validateCapture(capture);
29
- if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
30
- throw Object.assign(Error('integrity'), { code: 'integrity' });
31
- const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
32
- const assets = [];
33
- for (const asset of capture.assets) {
34
- if (options.signal?.aborted)
35
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
36
- const blob = await store.putBlob(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
37
- assets.push({ ...blob, mediaType: asset.mediaType });
38
- }
39
- if (options.signal?.aborted) {
40
- store.requestCancellation(token.jobId);
41
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
42
- }
43
- const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
44
- const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
45
- const status = localized.missingImages ? 'partial' : capture.status;
46
- // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
47
- const bundleHash = createHash('sha256').update(JSON.stringify(['article-local-markdown-v1', capture.manifestSha256])).digest('hex');
48
- await options.checkpoint?.('promoting');
49
- const revision = store.commitRevision({ ...token, itemId: source.itemId, bundleHash, markdown: localized.markdown, title, extractor: capture.extractor, status, metadata: capture.metadata, metadataWarnings: capture.metadataWarnings, assets, warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])] });
50
- committed = true;
51
- outcome = { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
39
+ attempt.hold(result);
40
+ await options.checkpoint?.('extracting');
41
+ const capture = result.capture;
42
+ validateCapture(capture);
43
+ if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
44
+ throw new SpongeError('integrity');
45
+ const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
46
+ const assets = [];
47
+ for (const asset of capture.assets) {
48
+ if (options.signal?.aborted)
49
+ throw new SpongeError('cancelled');
50
+ const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
51
+ assets.push({ ...blob, mediaType: asset.mediaType });
52
52
  }
53
- }
54
- catch (error) {
55
- const code = error?.code;
56
- outcome = failure(token.attemptId, code && known.has(code) ? code : 'unclassified', 'storage');
57
- }
58
- if (captured)
59
- try {
60
- await captured.dispose();
53
+ if (options.signal?.aborted) {
54
+ store.queue.cancel(token.jobId);
55
+ throw new SpongeError('cancelled');
61
56
  }
62
- catch {
63
- cleanup = 'pending';
64
- }
65
- if (!committed)
66
- try {
67
- store.failAttempt(token, outcome.ok ? 'unclassified' : outcome.code, outcome.ok ? {} : { retryable: outcome.retryable, retryAfterMs: outcome.retryAfterMs });
68
- }
69
- catch {
70
- cleanup = 'pending';
71
- }
72
- if (cleanup === 'settled')
73
- try {
74
- store.settleAttempt(token);
75
- }
76
- catch {
77
- cleanup = 'pending';
78
- }
79
- if (cleanup === 'pending')
80
- return { ...failure(token.attemptId, 'cleanup-required', 'cleanup'), cleanup };
81
- return { ...outcome, cleanup };
57
+ const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
58
+ const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
59
+ const status = localized.missingImages ? 'partial' : capture.status;
60
+ // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
61
+ const bundleHash = sha256(JSON.stringify(['article-local-markdown-v1', capture.manifestSha256]));
62
+ await options.checkpoint?.('promoting');
63
+ const revision = store.revisions.commit({
64
+ ...token,
65
+ itemId: source.itemId,
66
+ bundleHash,
67
+ markdown: localized.markdown,
68
+ title,
69
+ extractor: capture.extractor,
70
+ status,
71
+ metadata: capture.metadata,
72
+ metadataWarnings: capture.metadataWarnings,
73
+ assets,
74
+ warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
75
+ });
76
+ return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
77
+ });
82
78
  }
@@ -1,8 +1,4 @@
1
1
  import { type Asset } from '../../contracts/index.js';
2
- export declare class BundleIntegrityError extends Error {
3
- readonly code = "integrity";
4
- constructor();
5
- }
6
2
  /** Stream only the verified regular inode; parent path checks never follow symlinks. */
7
3
  export declare function bundleBytes(root: string, asset: Asset): AsyncGenerator<Uint8Array>;
8
4
  export declare function bundleText(root: string, asset: Asset): Promise<string>;
@@ -4,32 +4,29 @@ import { createHash } from 'node:crypto';
4
4
  import { dirname, join, isAbsolute, relative, sep } from 'node:path';
5
5
  import { limits } from '../../contracts/index.js';
6
6
  import { validateAsset } from '../../contracts/validate.js';
7
- export class BundleIntegrityError extends Error {
8
- code = 'integrity';
9
- constructor() { super('integrity'); }
10
- }
7
+ import { SpongeError } from '../errors.js';
11
8
  /** Stream only the verified regular inode; parent path checks never follow symlinks. */
12
9
  export async function* bundleBytes(root, asset) {
13
10
  validateAsset(asset);
14
11
  if (asset.bytes > limits.pdfBytes)
15
- throw new BundleIntegrityError();
16
- if (!isAbsolute(root) || await realpath(root) !== root)
17
- throw new BundleIntegrityError();
12
+ throw new SpongeError('integrity');
13
+ if (!isAbsolute(root) || (await realpath(root)) !== root)
14
+ throw new SpongeError('integrity');
18
15
  const path = join(root, asset.relativePath), rel = relative(root, path);
19
16
  if (!rel || rel.startsWith('..' + sep) || isAbsolute(rel))
20
- throw new BundleIntegrityError();
17
+ throw new SpongeError('integrity');
21
18
  let parent = root;
22
19
  for (const part of relative(root, dirname(path)).split(sep).filter(Boolean)) {
23
20
  parent = join(parent, part);
24
21
  const stat = await lstat(parent);
25
22
  if (!stat.isDirectory() || stat.isSymbolicLink())
26
- throw new BundleIntegrityError();
23
+ throw new SpongeError('integrity');
27
24
  }
28
25
  const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
29
26
  try {
30
27
  const stat = await file.stat();
31
28
  if (!stat.isFile() || stat.nlink !== 1 || stat.size !== asset.bytes)
32
- throw new BundleIntegrityError();
29
+ throw new SpongeError('integrity');
33
30
  const digest = createHash('sha256');
34
31
  let count = 0;
35
32
  for (;;) {
@@ -39,13 +36,13 @@ export async function* bundleBytes(root, asset) {
39
36
  break;
40
37
  count += bytesRead;
41
38
  if (count > asset.bytes)
42
- throw new BundleIntegrityError();
39
+ throw new SpongeError('integrity');
43
40
  const bytes = buffer.subarray(0, bytesRead);
44
41
  digest.update(bytes);
45
42
  yield bytes;
46
43
  }
47
44
  if (count !== asset.bytes || digest.digest('hex') !== asset.sha256)
48
- throw new BundleIntegrityError();
45
+ throw new SpongeError('integrity');
49
46
  }
50
47
  finally {
51
48
  await file.close();
@@ -53,7 +50,7 @@ export async function* bundleBytes(root, asset) {
53
50
  }
54
51
  export async function bundleText(root, asset) {
55
52
  if (asset.mediaType !== 'text/markdown' || asset.bytes > limits.markdownBytes)
56
- throw new BundleIntegrityError();
53
+ throw new SpongeError('integrity');
57
54
  const chunks = [];
58
55
  for await (const chunk of bundleBytes(root, asset))
59
56
  chunks.push(chunk);
@@ -61,6 +58,6 @@ export async function bundleText(root, asset) {
61
58
  return new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks));
62
59
  }
63
60
  catch {
64
- throw new BundleIntegrityError();
61
+ throw new SpongeError('integrity');
65
62
  }
66
63
  }
@@ -1,16 +1,19 @@
1
1
  import { fromMarkdown } from 'mdast-util-from-markdown';
2
2
  import { toMarkdown } from 'mdast-util-to-markdown';
3
+ import { SpongeError } from '../errors.js';
3
4
  const assetPrefix = 'sponge-asset:';
4
5
  function localKey(url) {
5
6
  try {
6
7
  const key = decodeURIComponent(url).replace(/^\.\//u, '');
7
- return key.split('/').some(p => p === '..' || p === '') || key.includes('\\') || key.includes(':') ? null : key;
8
+ return key.split('/').some((p) => p === '..' || p === '') || key.includes('\\') || key.includes(':') ? null : key;
8
9
  }
9
10
  catch {
10
11
  return null;
11
12
  }
12
13
  }
13
- function text(value) { return toMarkdown({ type: 'text', value }).trimEnd(); }
14
+ function text(value) {
15
+ return toMarkdown({ type: 'text', value }).trimEnd();
16
+ }
14
17
  /** Preserve original formatting/frontmatter; edit parsed destination spans, never regex Markdown. */
15
18
  export function localizeMarkdown(markdown, assets) {
16
19
  const frontmatter = /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0];
@@ -18,7 +21,7 @@ export function localizeMarkdown(markdown, assets) {
18
21
  const body = localizeMarkdown(markdown.slice(frontmatter.length), assets);
19
22
  return { ...body, markdown: frontmatter + body.markdown };
20
23
  }
21
- const tree = fromMarkdown(markdown), byPath = new Map(assets.map(a => [a.relativePath, a.sha256]));
24
+ const tree = fromMarkdown(markdown), byPath = new Map(assets.map((a) => [a.relativePath, a.sha256]));
22
25
  const definitions = new Map();
23
26
  const patches = [];
24
27
  let missingImages = false;
@@ -33,7 +36,7 @@ export function localizeMarkdown(markdown, assets) {
33
36
  function replace(node, value) {
34
37
  const start = node.position?.start.offset, end = node.position?.end.offset;
35
38
  if (start === undefined || end === undefined)
36
- throw Error('integrity');
39
+ throw new SpongeError('integrity');
37
40
  patches.push({ start, end, value });
38
41
  }
39
42
  function walk(node) {
@@ -41,7 +44,12 @@ export function localizeMarkdown(markdown, assets) {
41
44
  const url = node.type === 'image' ? node.url : definitions.get(node.identifier)?.url;
42
45
  const key = url ? localKey(url) : null, sha = key ? byPath.get(key) : undefined;
43
46
  if (sha) {
44
- const image = { type: 'image', url: assetPrefix + sha, alt: node.alt ?? '', title: node.type === 'image' ? node.title : definitions.get(node.identifier)?.title };
47
+ const image = {
48
+ type: 'image',
49
+ url: assetPrefix + sha,
50
+ alt: node.alt ?? '',
51
+ title: node.type === 'image' ? node.title : definitions.get(node.identifier)?.title,
52
+ };
45
53
  replace(node, toMarkdown(image).trimEnd());
46
54
  Object.assign(node, image);
47
55
  }
@@ -73,7 +81,11 @@ export function localizeMarkdown(markdown, assets) {
73
81
  if (node.type === 'definition')
74
82
  replace(node, '');
75
83
  else {
76
- const plain = (n) => 'value' in n ? String(n.value) : 'children' in n ? n.children.map(c => plain(c)).join('') : '';
84
+ const plain = (n) => 'value' in n
85
+ ? String(n.value)
86
+ : 'children' in n
87
+ ? n.children.map((c) => plain(c)).join('')
88
+ : '';
77
89
  replace(node, text(plain(node)));
78
90
  }
79
91
  return;
@@ -1,7 +1,3 @@
1
- export declare class InputError extends Error {
2
- readonly code = "invalid-input";
3
- constructor();
4
- }
5
1
  /** The identity form of an http(s) locator; source keys and shareable URIs both use it. Its output pins every existing source key. */
6
2
  export declare function normalizeWebLocator(locator: string): string;
7
3
  export declare function articleSource(locator: string, realm?: string): {