@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -6,21 +6,27 @@ export function shareableWebUri(locator) {
6
6
  const url = new URL(normalizeWebLocator(locator));
7
7
  url.hash = '';
8
8
  if (url.search)
9
- url.search = filterQuery(url.search, key => clickTrackers.has(key) || isSecretField(key));
9
+ url.search = filterQuery(url.search, (key) => clickTrackers.has(key) || isSecretField(key));
10
10
  return url.href;
11
11
  }
12
12
  const youtubeId = /^[A-Za-z0-9_-]{11}$/;
13
- const decoded = (value) => { try {
14
- return decodeURIComponent(value);
15
- }
16
- catch {
17
- return undefined;
18
- } };
13
+ const decoded = (value) => {
14
+ try {
15
+ return decodeURIComponent(value);
16
+ }
17
+ catch {
18
+ return undefined;
19
+ }
20
+ };
19
21
  /** Provider identifiers name the same work across URL spellings, hosts and mirrors. */
20
- export function providerUris(href) {
22
+ function providerUris(href) {
21
23
  const url = new URL(href), host = url.hostname.replace(/^(www|m|mobile)\./, ''), parts = url.pathname.split('/').filter(Boolean), out = [];
22
24
  if (host === 'youtube.com' || host === 'music.youtube.com') {
23
- const id = url.pathname === '/watch' ? url.searchParams.get('v') : ['shorts', 'live', 'embed'].includes(parts[0]) ? parts[1] : undefined;
25
+ const id = url.pathname === '/watch'
26
+ ? url.searchParams.get('v')
27
+ : ['shorts', 'live', 'embed'].includes(parts[0])
28
+ ? parts[1]
29
+ : undefined;
24
30
  if (id && youtubeId.test(id))
25
31
  out.push('youtube:' + id);
26
32
  }
@@ -29,7 +35,11 @@ export function providerUris(href) {
29
35
  if ((host === 'x.com' || host === 'twitter.com') && parts[1] === 'status' && /^\d{1,25}$/.test(parts[2] ?? ''))
30
36
  out.push('x:status/' + parts[2]);
31
37
  if (host === 'arxiv.org' && ['abs', 'pdf'].includes(parts[0]) && parts[1]) {
32
- const id = parts.slice(1).join('/').replace(/\.pdf$/, '').replace(/v\d+$/, '');
38
+ const id = parts
39
+ .slice(1)
40
+ .join('/')
41
+ .replace(/\.pdf$/, '')
42
+ .replace(/v\d+$/, '');
33
43
  if (/^(\d{4}\.\d{4,5}|[a-z-]+(\.[A-Z]{2})?\/\d{7})$/.test(id))
34
44
  out.push('arxiv:' + id);
35
45
  }
@@ -40,7 +50,9 @@ export function providerUris(href) {
40
50
  }
41
51
  return out;
42
52
  }
43
- export function isResolvable(uri) { return /^(https?:|doi:|arxiv:|isbn:|youtube:|x:)/.test(uri); }
53
+ export function isResolvable(uri) {
54
+ return /^(https?:|doi:|arxiv:|isbn:|youtube:|x:)/.test(uri);
55
+ }
44
56
  /**
45
57
  * Derive an item's URIs from its first source and, once captured, its source PDF. Total: a
46
58
  * locator that cannot be parsed, carries userinfo or is too long yields only `sponge:item/<id>`.
@@ -48,8 +60,10 @@ export function isResolvable(uri) { return /^(https?:|doi:|arxiv:|isbn:|youtube:
48
60
  * `urn:sha256:<file hash>`, which matches the same file anywhere but is not resolvable.
49
61
  */
50
62
  export function deriveUris(input) {
51
- const out = [], add = (uri, origin) => { if (uri.length >= 3 && Buffer.byteLength(uri) <= maxUriBytes && !out.some(u => u.uri === uri))
52
- out.push({ uri, role: out.length ? 'alias' : 'primary', resolvable: isResolvable(uri), origin }); };
63
+ const out = [], add = (uri, origin) => {
64
+ if (uri.length >= 3 && Buffer.byteLength(uri) <= maxUriBytes && !out.some((u) => u.uri === uri))
65
+ out.push({ uri, role: out.length ? 'alias' : 'primary', resolvable: isResolvable(uri), origin });
66
+ };
53
67
  if (/^https?:/i.test(input.locator)) {
54
68
  let web;
55
69
  try {
@@ -64,7 +78,9 @@ export function deriveUris(input) {
64
78
  for (const uri of providerUris(web))
65
79
  add(uri, 'locator');
66
80
  }
67
- catch { /* a provider alias is optional */ }
81
+ catch {
82
+ /* a provider alias is optional */
83
+ }
68
84
  }
69
85
  }
70
86
  if (input.sourcePdfSha256)
@@ -73,10 +89,13 @@ export function deriveUris(input) {
73
89
  add('sponge:item/' + input.itemId, 'locator');
74
90
  return out;
75
91
  }
76
- /** How the first source was reached decides the default visibility; see docs/decisions/sync-and-publishing.md. */
77
- export function accessClass(realm, locator) {
78
- if (locator.startsWith('file:'))
79
- return 'local';
92
+ /**
93
+ * How the first source was reached decides the default visibility; see docs/decisions/sync-and-publishing.md.
94
+ * An imported local file with a recorded web address (`webUri`, origin 'metadata') is classed by that address instead.
95
+ */
96
+ export function accessClass(realm, locator, webUri) {
97
+ if (/^(file|local):/i.test(locator))
98
+ return realm.startsWith('import:') && webUri && /^https?:/i.test(webUri) ? accessClass(realm, webUri) : 'local';
80
99
  if (realm.startsWith('connection:'))
81
100
  return 'account';
82
101
  // A guard on top of the source rule: a link that is itself a credential is never public by default.
@@ -1,6 +1,15 @@
1
1
  import type { DatabaseSync } from 'node:sqlite';
2
2
  import { type AccessClass, type ItemUri } from './uri.js';
3
3
  export type Visibility = 'public' | 'private';
4
+ /** The recorded web address of a capture that has none of its own (origin 'metadata', primary), if any. */
5
+ export declare function metadataUri(db: DatabaseSync, itemId: string): string | undefined;
6
+ /** Whether the capture itself (its locator or revision) yields a web URI. */
7
+ export declare function hasWebUri(db: DatabaseSync, itemId: string): boolean;
8
+ /**
9
+ * Record `uri` (already a shareable web URI) as the primary of a capture without a web URI of its own; derived URIs become
10
+ * aliases and a previously recorded primary is replaced. Idempotent. Derived state like every item URI, not a logged field.
11
+ */
12
+ export declare function setMetadataUri(db: DatabaseSync, itemId: string, uri: string): void;
4
13
  /** Recompute derived URIs; metadata and user URIs are left alone. Idempotent. */
5
14
  export declare function syncUris(db: DatabaseSync, itemId: string): void;
6
15
  /**
@@ -20,5 +29,6 @@ export interface VisibilityState {
20
29
  override: Visibility | null;
21
30
  effective: Visibility;
22
31
  }
23
- export declare function visibility(db: DatabaseSync, itemId: string): VisibilityState;
32
+ /** `webUri` previews a metadata URI not yet recorded; by default the recorded one counts. */
33
+ export declare function visibility(db: DatabaseSync, itemId: string, webUri?: string | undefined): VisibilityState;
24
34
  export declare function setVisibility(db: DatabaseSync, itemId: string, value: Visibility | null): VisibilityState;
@@ -1,24 +1,71 @@
1
- import { accessClass, defaultVisibility, deriveUris } from './uri.js';
1
+ import { accessClass, defaultVisibility, deriveUris, maxUriBytes } from './uri.js';
2
2
  import { mutate } from '../store/changes.js';
3
- import { StoreError } from '../store/types.js';
4
3
  import { text } from '../store/files.js';
5
- const fail = (code = 'invalid-input') => { throw new StoreError(code); };
6
- function firstSource(db, itemId) { return db.prepare('SELECT realm_key,locator FROM source_refs WHERE item_id=? ORDER BY rowid LIMIT 1').get(itemId); }
4
+ import { SpongeError } from '../errors.js';
5
+ const fail = (code = 'invalid-input') => {
6
+ throw new SpongeError(code);
7
+ };
8
+ function firstSource(db, itemId) {
9
+ return db.prepare('SELECT realm_key,locator FROM source_refs WHERE item_id=? ORDER BY rowid LIMIT 1').get(itemId);
10
+ }
11
+ /** The recorded web address of a capture that has none of its own (origin 'metadata', primary), if any. */
12
+ export function metadataUri(db, itemId) {
13
+ const r = db
14
+ .prepare("SELECT uri FROM item_uris WHERE item_id=? AND origin='metadata' AND role='primary'")
15
+ .get(itemId);
16
+ return r && /^https?:/.test(String(r.uri)) ? String(r.uri) : undefined;
17
+ }
18
+ /** Whether the capture itself (its locator or revision) yields a web URI. */
19
+ export function hasWebUri(db, itemId) {
20
+ return !!db
21
+ .prepare("SELECT 1 FROM item_uris WHERE item_id=? AND origin IN ('locator','revision') AND (uri LIKE 'http:%' OR uri LIKE 'https:%')")
22
+ .get(itemId);
23
+ }
24
+ /**
25
+ * Record `uri` (already a shareable web URI) as the primary of a capture without a web URI of its own; derived URIs become
26
+ * aliases and a previously recorded primary is replaced. Idempotent. Derived state like every item URI, not a logged field.
27
+ */
28
+ export function setMetadataUri(db, itemId, uri) {
29
+ text(uri, maxUriBytes);
30
+ if (!/^https?:/.test(uri) || hasWebUri(db, itemId))
31
+ fail();
32
+ if (metadataUri(db, itemId) === uri)
33
+ return;
34
+ db.prepare("DELETE FROM item_uris WHERE item_id=? AND origin='metadata' AND role='primary'").run(itemId);
35
+ db.prepare("UPDATE item_uris SET role='alias' WHERE item_id=?").run(itemId);
36
+ db.prepare("INSERT INTO item_uris VALUES(?,?,'primary',1,'metadata') ON CONFLICT(item_id,uri) DO UPDATE SET role='primary',resolvable=1,origin='metadata'").run(itemId, uri);
37
+ }
7
38
  /** Recompute derived URIs; metadata and user URIs are left alone. Idempotent. */
8
39
  export function syncUris(db, itemId) {
9
40
  const source = firstSource(db, itemId);
10
41
  if (!source)
11
42
  return;
12
- const pdf = db.prepare("SELECT ra.sha256 FROM items i JOIN revision_assets ra ON ra.revision_id=i.current_revision WHERE i.id=? AND ra.relationship='source-pdf' ORDER BY ra.ordinal LIMIT 1").get(itemId);
13
- const uris = deriveUris({ itemId, locator: String(source.locator), ...(pdf ? { sourcePdfSha256: String(pdf.sha256) } : {}) });
14
- const current = db.prepare("SELECT uri,role,resolvable,origin FROM item_uris WHERE item_id=? AND origin IN ('locator','revision') ORDER BY uri").all(itemId).map(r => `${r.uri}|${r.role}|${r.resolvable}|${r.origin}`).join('\n');
15
- const next = uris.map(u => `${u.uri}|${u.role}|${Number(u.resolvable)}|${u.origin}`).sort().join('\n');
43
+ const pdf = db
44
+ .prepare("SELECT ra.sha256 FROM items i JOIN revision_assets ra ON ra.revision_id=i.current_revision WHERE i.id=? AND ra.relationship='source-pdf' ORDER BY ra.ordinal LIMIT 1")
45
+ .get(itemId);
46
+ const uris = deriveUris({
47
+ itemId,
48
+ locator: String(source.locator),
49
+ ...(pdf ? { sourcePdfSha256: String(pdf.sha256) } : {}),
50
+ });
51
+ // A capture without a web URI keeps a recorded web address as its primary; its derived URIs are then aliases.
52
+ const kept = !uris.some((u) => /^https?:/.test(u.uri)) && metadataUri(db, itemId), rows = kept ? uris.map((u) => ({ ...u, role: 'alias' })) : uris;
53
+ const current = db
54
+ .prepare("SELECT uri,role,resolvable,origin FROM item_uris WHERE item_id=? AND origin IN ('locator','revision') ORDER BY uri")
55
+ .all(itemId)
56
+ .map((r) => `${r.uri}|${r.role}|${r.resolvable}|${r.origin}`)
57
+ .join('\n');
58
+ const next = rows
59
+ .map((u) => `${u.uri}|${u.role}|${Number(u.resolvable)}|${u.origin}`)
60
+ .sort()
61
+ .join('\n');
16
62
  if (current === next)
17
63
  return;
18
64
  db.prepare("DELETE FROM item_uris WHERE item_id=? AND origin IN ('locator','revision')").run(itemId);
19
65
  // A derived primary replaces any other primary; keep later metadata/user URIs as aliases.
20
- db.prepare("UPDATE item_uris SET role='alias' WHERE item_id=?").run(itemId);
21
- for (const u of uris)
66
+ if (!kept)
67
+ db.prepare("UPDATE item_uris SET role='alias' WHERE item_id=?").run(itemId);
68
+ for (const u of rows)
22
69
  db.prepare('INSERT INTO item_uris VALUES(?,?,?,?,?) ON CONFLICT(item_id,uri) DO UPDATE SET role=excluded.role,resolvable=excluded.resolvable,origin=excluded.origin').run(itemId, u.uri, u.role, Number(u.resolvable), u.origin);
23
70
  }
24
71
  /**
@@ -26,7 +73,10 @@ export function syncUris(db, itemId) {
26
73
  * one hostile locator leaves that item without URIs instead of making the library unopenable.
27
74
  */
28
75
  export function backfillUris(db) {
29
- const missing = db.prepare("SELECT i.id FROM items i WHERE NOT EXISTS (SELECT 1 FROM item_uris u WHERE u.item_id=i.id AND u.role='primary') AND EXISTS (SELECT 1 FROM source_refs r WHERE r.item_id=i.id)").all().map(r => String(r.id));
76
+ const missing = db
77
+ .prepare("SELECT i.id FROM items i WHERE NOT EXISTS (SELECT 1 FROM item_uris u WHERE u.item_id=i.id AND u.role='primary') AND EXISTS (SELECT 1 FROM source_refs r WHERE r.item_id=i.id)")
78
+ .all()
79
+ .map((r) => String(r.id));
30
80
  let skipped = 0;
31
81
  for (const id of missing) {
32
82
  db.exec('SAVEPOINT backfill_uri');
@@ -42,26 +92,40 @@ export function backfillUris(db) {
42
92
  return { filled: missing.length - skipped, skipped };
43
93
  }
44
94
  export function itemUris(db, itemId) {
45
- return db.prepare("SELECT uri,role,resolvable,origin FROM item_uris WHERE item_id=? ORDER BY role='alias',uri").all(itemId).map(r => ({ uri: String(r.uri), role: r.role, resolvable: !!r.resolvable, origin: r.origin }));
95
+ return db
96
+ .prepare("SELECT uri,role,resolvable,origin FROM item_uris WHERE item_id=? ORDER BY role='alias',uri")
97
+ .all(itemId)
98
+ .map((r) => ({
99
+ uri: String(r.uri),
100
+ role: r.role,
101
+ resolvable: !!r.resolvable,
102
+ origin: r.origin,
103
+ }));
46
104
  }
47
105
  /** Items sharing any URI with `uri`: the local half of "who else is reading this". */
48
106
  export function itemsForUri(db, uri) {
49
107
  text(uri, 8192);
50
- return db.prepare('SELECT DISTINCT u.item_id FROM item_uris u JOIN items i ON i.id=u.item_id WHERE u.uri=? AND i.deleted_at IS NULL ORDER BY u.item_id').all(uri).map(r => String(r.item_id));
108
+ return db
109
+ .prepare('SELECT DISTINCT u.item_id FROM item_uris u JOIN items i ON i.id=u.item_id WHERE u.uri=? AND i.deleted_at IS NULL ORDER BY u.item_id')
110
+ .all(uri)
111
+ .map((r) => String(r.item_id));
51
112
  }
52
- export function visibility(db, itemId) {
113
+ /** `webUri` previews a metadata URI not yet recorded; by default the recorded one counts. */
114
+ export function visibility(db, itemId, webUri = metadataUri(db, itemId)) {
53
115
  const item = db.prepare('SELECT visibility,deleted_at FROM items WHERE id=?').get(itemId) ?? fail('unavailable'), source = firstSource(db, itemId);
54
- const access = source ? accessClass(String(source.realm_key), String(source.locator)) : 'local';
116
+ const access = source ? accessClass(String(source.realm_key), String(source.locator), webUri) : 'local';
55
117
  const reading = db.prepare('SELECT state_json FROM reading_state WHERE item_id=?').get(itemId);
56
118
  let importedPrivacy;
57
119
  try {
58
- importedPrivacy = reading ? JSON.parse(String(reading.state_json))?.sourcePrivacy : undefined;
120
+ importedPrivacy = reading
121
+ ? JSON.parse(String(reading.state_json))?.sourcePrivacy
122
+ : undefined;
59
123
  }
60
124
  catch {
61
125
  importedPrivacy = undefined;
62
126
  }
63
127
  const base = defaultVisibility(access, importedPrivacy), override = (item.visibility ?? null);
64
- return { access, default: base, override, effective: item.deleted_at ? 'private' : override ?? base };
128
+ return { access, default: base, override, effective: item.deleted_at ? 'private' : (override ?? base) };
65
129
  }
66
130
  export function setVisibility(db, itemId, value) {
67
131
  if (value !== null && value !== 'public' && value !== 'private')
@@ -1,62 +1,108 @@
1
- import { hash } from '../acquisition/files.js';
2
- import { failure } from '../acquisition/errors.js';
1
+ import { sha256 as hash } from '../util/hash.js';
2
+ import { runAttempt } from '../ingest/attempt.js';
3
3
  import { bundleBytes, bundleText } from '../app/bundle-files.js';
4
4
  import { localizeMarkdown } from '../app/markdown.js';
5
5
  import { selectedMetadata } from '../metadata/article.js';
6
6
  import { drainQueue } from '../worker/engine.js';
7
7
  import { readImportManifest, importBundleRoot } from './manifest.js';
8
8
  import { inspectImportBundle, releasedBundleVerifier } from './inspect.js';
9
- const errorCode = (error) => { const code = error?.code ?? (error instanceof Error ? error.message : ''); return ['unsupported', 'invalid-input', 'integrity', 'limit-exceeded', 'cancelled', 'cleanup-required', 'runtime-missing', 'owner-busy', 'unavailable'].includes(code) ? code : 'unclassified'; };
10
- async function promote(context, bundle, namespace, entry, sourceHash) {
9
+ import { SpongeError, publicCode } from '../errors.js';
10
+ const errorCode = (error) => {
11
+ const code = publicCode(error);
12
+ return [
13
+ 'unsupported',
14
+ 'invalid-input',
15
+ 'integrity',
16
+ 'limit-exceeded',
17
+ 'cancelled',
18
+ 'cleanup-required',
19
+ 'runtime-missing',
20
+ 'owner-busy',
21
+ 'unavailable',
22
+ ].includes(code)
23
+ ? code
24
+ : 'unclassified';
25
+ };
26
+ function promote(context, bundle, namespace, entry, sourceHash) {
11
27
  const { store, attempt, signal, checkpoint } = context;
12
- let committed = false;
13
- let result = failure(attempt.attemptId, 'unclassified');
14
- try {
15
- const source = store.sourceForJob(attempt.jobId);
28
+ return runAttempt(store, attempt, errorCode, async () => {
29
+ const source = store.queue.source(attempt.jobId);
16
30
  if (!source)
17
- throw Error('integrity');
31
+ throw new SpongeError('integrity');
18
32
  await checkpoint('extracting');
19
33
  const markdown = await bundleText(bundle.root, bundle.markdown), localized = localizeMarkdown(bundle.kind === 'pdf' ? markdown.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1') : markdown, bundle.assets), assets = [];
20
34
  for (const asset of bundle.assets) {
21
35
  if (signal.aborted)
22
- throw Error('cancelled');
23
- const stored = await store.putBlob(attempt, asset.sha256, asset.bytes, bundleBytes(bundle.root, asset));
36
+ throw new SpongeError('cancelled');
37
+ const stored = await store.blobs.put(attempt, asset.sha256, asset.bytes, bundleBytes(bundle.root, asset));
24
38
  assets.push({ ...stored, mediaType: asset.mediaType, relationship: asset.relationship });
25
39
  }
26
40
  if (signal.aborted)
27
- throw Error('cancelled');
41
+ throw new SpongeError('cancelled');
28
42
  await checkpoint('promoting');
29
43
  const title = selectedMetadata(bundle.metadata, 'title')?.value;
30
- store.commitRevision({ ...attempt, itemId: source.itemId, bundleHash: importBundleHash(sourceHash), markdown: localized.markdown, title: typeof title === 'string' ? title : undefined, extractor: { name: 'sponge-wordcell-import', version: '1' }, status: bundle.status === 'partial' || localized.missingImages ? 'partial' : 'complete', metadata: [...bundle.metadata, ...Object.entries({ importNamespace: namespace, externalSourceId: entry.id, sourcePrivacy: entry.privacy }).map(([field, value]) => ({ field, value, source: bundle.kind === 'pdf' ? 'pdf' : 'html', observedAt: new Date().toISOString(), rule: 'sponge-import-v1' }))], assets, warnings: bundle.status === 'partial' || localized.missingImages ? ['partial'] : [], importedState: { namespace, sourceKey: entry.id, sourceHash, importerVersion: '1', notes: entry.notes, readingState: entry.readingState, privacy: entry.privacy } });
31
- committed = true;
32
- result = { ok: true, itemId: source.itemId };
33
- }
34
- catch (error) {
35
- result = failure(attempt.attemptId, errorCode(error), 'storage');
36
- }
37
- if (!committed)
38
- store.failAttempt(attempt, result.ok ? 'unclassified' : result.code);
39
- store.settleAttempt(attempt);
40
- return { ...result, cleanup: 'settled' };
44
+ store.revisions.commit({
45
+ ...attempt,
46
+ itemId: source.itemId,
47
+ bundleHash: importBundleHash(sourceHash),
48
+ markdown: localized.markdown,
49
+ title: typeof title === 'string' ? title : undefined,
50
+ extractor: { name: 'sponge-wordcell-import', version: '1' },
51
+ status: bundle.status === 'partial' || localized.missingImages ? 'partial' : 'complete',
52
+ metadata: [
53
+ ...bundle.metadata,
54
+ ...Object.entries({ importNamespace: namespace, externalSourceId: entry.id, sourcePrivacy: entry.privacy }).map(([field, value]) => ({
55
+ field,
56
+ value,
57
+ source: bundle.kind === 'pdf' ? 'pdf' : 'html',
58
+ observedAt: new Date().toISOString(),
59
+ rule: 'sponge-import-v1',
60
+ })),
61
+ ],
62
+ assets,
63
+ warnings: bundle.status === 'partial' || localized.missingImages ? ['partial'] : [],
64
+ importedState: {
65
+ namespace,
66
+ sourceKey: entry.id,
67
+ sourceHash,
68
+ importerVersion: '1',
69
+ notes: entry.notes,
70
+ readingState: entry.readingState,
71
+ privacy: entry.privacy,
72
+ },
73
+ });
74
+ return { ok: true, itemId: source.itemId };
75
+ });
41
76
  }
42
77
  /** One verified bundle: a receipt short-circuits, a dry run stops after verification, otherwise the capture is promoted through the queue. */
43
78
  export async function importBundleEntry(store, root, namespace, entry, verify, dryRun, signal) {
44
- const bundle = await inspectImportBundle(await importBundleRoot(root, entry), verify), sourceHash = hash(JSON.stringify([bundle.importHash, entry])), receipt = store.lookupImportReceipt(namespace, entry.id, sourceHash);
79
+ const bundle = await inspectImportBundle(await importBundleRoot(root, entry), verify), sourceHash = hash(JSON.stringify([bundle.importHash, entry])), receipt = store.imports.receipt(namespace, entry.id, sourceHash);
45
80
  if (signal?.aborted)
46
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
81
+ throw new SpongeError('cancelled');
47
82
  const base = { sourceHash, historicalIntegrity: bundle.historicalIntegrity };
48
83
  if (receipt)
49
84
  return { ...base, status: 'duplicate', itemId: receipt.itemId };
50
85
  if (dryRun)
51
86
  return { ...base, status: 'ready' };
52
- const submission = store.submitSource({ kind: bundle.kind, sourceKey: hash(JSON.stringify([namespace, entry.id])), locator: bundle.sourceUrl, realm: 'import:' + namespace, idempotencyKey: 'import:' + hash(JSON.stringify([namespace, entry.id, sourceHash])), intent: 'refresh' });
53
- const job = store.listJobs().find(row => row.id === submission.jobId);
87
+ const submission = store.queue.submit({
88
+ kind: bundle.kind,
89
+ sourceKey: hash(JSON.stringify([namespace, entry.id])),
90
+ locator: bundle.sourceUrl,
91
+ realm: 'import:' + namespace,
92
+ idempotencyKey: 'import:' + hash(JSON.stringify([namespace, entry.id, sourceHash])),
93
+ intent: 'refresh',
94
+ });
95
+ const job = store.queue.list().find((row) => row.id === submission.jobId);
54
96
  if (job && ['failed', 'needs-auth', 'needs-permission', 'unsupported'].includes(String(job.state)))
55
- store.retryJob(submission.jobId, { automatic: false });
56
- const results = await drainQueue(store, { jobId: submission.jobId, signal, handlers: { [bundle.kind]: context => promote(context, bundle, namespace, entry, sourceHash) } });
97
+ store.queue.retry(submission.jobId, { automatic: false });
98
+ const results = await drainQueue(store, {
99
+ jobId: submission.jobId,
100
+ signal,
101
+ handlers: { [bundle.kind]: (context) => promote(context, bundle, namespace, entry, sourceHash) },
102
+ });
57
103
  const result = results[0];
58
104
  if (!result || !result.ok)
59
- throw Object.assign(Error(result && !result.ok ? result.code : 'integrity'), { code: result && !result.ok ? result.code : 'integrity' });
105
+ throw new SpongeError(result && !result.ok ? result.code : 'integrity');
60
106
  return { ...base, status: 'imported', itemId: submission.itemId };
61
107
  }
62
108
  /** The revision a bundle import committed: content-addressed by its source hash. */
@@ -64,20 +110,42 @@ export const importBundleHash = (sourceHash) => hash(JSON.stringify(['wordcell-i
64
110
  /** Additive bounded batches. Every applied entry is reverified; source bundles are read-only. */
65
111
  export async function importManifest(store, path, options) {
66
112
  const { root, manifest, sha256 } = await readImportManifest(path), limit = options.limit ?? 20, offset = options.offset ?? 0;
67
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(offset) || offset < 0 || offset > manifest.entries.length)
68
- throw Error('invalid-input');
69
- const dryRun = options.dryRun !== false, report = { schemaVersion: 1, manifestSha256: sha256, namespace: manifest.namespace, dryRun, entries: [], nextOffset: offset + limit < manifest.entries.length ? offset + limit : null };
113
+ if (!Number.isSafeInteger(limit) ||
114
+ limit < 1 ||
115
+ limit > 100 ||
116
+ !Number.isSafeInteger(offset) ||
117
+ offset < 0 ||
118
+ offset > manifest.entries.length)
119
+ throw new SpongeError('invalid-input');
120
+ const dryRun = options.dryRun !== false, report = {
121
+ schemaVersion: 1,
122
+ manifestSha256: sha256,
123
+ namespace: manifest.namespace,
124
+ dryRun,
125
+ entries: [],
126
+ nextOffset: offset + limit < manifest.entries.length ? offset + limit : null,
127
+ };
70
128
  const verify = options.verify ?? releasedBundleVerifier(options.runtime, store.root, options.signal);
71
129
  for (const entry of manifest.entries.slice(offset, offset + limit)) {
72
130
  if (options.signal?.aborted)
73
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
131
+ throw new SpongeError('cancelled');
74
132
  try {
75
- const result = await importBundleEntry(store, root, manifest.namespace, entry, verify, dryRun, options.signal), base = { id: entry.id, sourceHash: result.sourceHash, historicalIntegrity: result.historicalIntegrity, privacy: entry.privacy };
133
+ const result = await importBundleEntry(store, root, manifest.namespace, entry, verify, dryRun, options.signal), base = {
134
+ id: entry.id,
135
+ sourceHash: result.sourceHash,
136
+ historicalIntegrity: result.historicalIntegrity,
137
+ privacy: entry.privacy,
138
+ };
76
139
  report.entries.push({ ...base, status: result.status, ...(result.itemId ? { itemId: result.itemId } : {}) });
77
140
  }
78
141
  catch (error) {
79
142
  const code = errorCode(error);
80
- report.entries.push({ id: entry.id, status: code === 'unsupported' ? 'unsupported' : 'failed', code, privacy: entry.privacy });
143
+ report.entries.push({
144
+ id: entry.id,
145
+ status: code === 'unsupported' ? 'unsupported' : 'failed',
146
+ code,
147
+ privacy: entry.privacy,
148
+ });
81
149
  if (code === 'cleanup-required' || code === 'owner-busy' || code === 'cancelled') {
82
150
  report.nextOffset = offset + report.entries.length - 1;
83
151
  break;
@@ -2,33 +2,51 @@ import { randomUUID } from 'node:crypto';
2
2
  import { join } from 'node:path';
3
3
  import { rm } from 'node:fs/promises';
4
4
  import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
5
- import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
5
+ import { directory, writeJournal, readJournal } from '../worker/journal.js';
6
+ import { syncDirectoryAsync } from '../util/fsync.js';
6
7
  import { supervisedRunner } from '../worker/supervisor.js';
7
- const pending = () => Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
8
+ import { SpongeError, codeOf } from '../errors.js';
9
+ const pending = () => new SpongeError('cleanup-required');
8
10
  /** Read-only verification still owns subprocesses, including during an import preview. */
9
11
  export async function withImportVerification(libraryRoot, verify) {
10
12
  const custody = await acquireWorkerCustody(libraryRoot), attemptId = randomUUID();
11
13
  let scratchRoot, unknown = false;
12
14
  try {
13
15
  const attempt = await directory(join(custody.directory, 'attempts', attemptId));
14
- await writeJournal(join(attempt, 'import-verification.json'), { schemaVersion: 1, kind: 'import-verification', attemptId, ownerIncarnation: custody.incarnation });
16
+ await writeJournal(join(attempt, 'import-verification.json'), {
17
+ schemaVersion: 1,
18
+ kind: 'import-verification',
19
+ attemptId,
20
+ ownerIncarnation: custody.incarnation,
21
+ });
15
22
  scratchRoot = await directory(join(libraryRoot, 'staging', 'import-verification-' + attemptId));
16
- await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'import-verification', requestId: attemptId, ownerIncarnation: custody.incarnation });
17
- return await verify({ scratchRoot, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
23
+ await writeJournal(join(scratchRoot, 'ownership.json'), {
24
+ schemaVersion: 1,
25
+ kind: 'import-verification',
26
+ requestId: attemptId,
27
+ ownerIncarnation: custody.incarnation,
28
+ });
29
+ return await verify({
30
+ scratchRoot,
31
+ runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId),
32
+ });
18
33
  }
19
34
  catch (error) {
20
- unknown = error?.code === 'cleanup-required' || (error instanceof Error && error.message === 'cleanup-required');
35
+ unknown = codeOf(error) === 'cleanup-required';
21
36
  throw error;
22
37
  }
23
38
  finally {
24
- if (unknown || !await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId))
39
+ if (unknown || !(await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId)))
25
40
  throw pending();
26
41
  if (scratchRoot) {
27
42
  const marker = await readJournal(join(scratchRoot, 'ownership.json'));
28
- if (marker.schemaVersion !== 1 || marker.kind !== 'import-verification' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
43
+ if (marker.schemaVersion !== 1 ||
44
+ marker.kind !== 'import-verification' ||
45
+ marker.requestId !== attemptId ||
46
+ marker.ownerIncarnation !== custody.incarnation)
29
47
  throw pending();
30
48
  await rm(scratchRoot, { recursive: true });
31
- await syncDir(join(libraryRoot, 'staging'));
49
+ await syncDirectoryAsync(join(libraryRoot, 'staging'));
32
50
  }
33
51
  await custody.release();
34
52
  }