@hraness/sponge 0.2.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +23 -10
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +39 -17
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +10 -46
  64. package/dist/src/digest/editorial.js +73 -59
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +53 -32
  67. package/dist/src/digest/pipeline.js +357 -125
  68. package/dist/src/digest/schema.d.ts +44 -17
  69. package/dist/src/digest/schema.js +132 -44
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +31 -24
  82. package/dist/src/importers/jungle-digest.js +217 -115
  83. package/dist/src/importers/jungle-store.d.ts +20 -11
  84. package/dist/src/importers/jungle-store.js +136 -41
  85. package/dist/src/importers/jungle.d.ts +32 -16
  86. package/dist/src/importers/jungle.js +145 -32
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +108 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -77
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -1,18 +1,62 @@
1
- import { canonicalDigest, derivationId, putDigest, reviewDigest } from '../digest/store.js';
2
- import { canonicalEditorial, putEditorial, reviewEditorial } from '../digest/editorial.js';
1
+ import { canonicalDigest } from '../digest/derivation.js';
2
+ import { canonicalEditorial } from '../digest/editorial.js';
3
+ import { derivationId, putDerivation, reviewDerivation } from '../store/derivations.js';
3
4
  import { parseDigest } from '../digest/schema.js';
4
- import { visibility as visibilityState, setVisibility } from '../identity/visibility.js';
5
- import { digest } from '../store/files.js';
5
+ import { visibility as visibilityState, setVisibility, hasWebUri, metadataUri, setMetadataUri, } from '../identity/visibility.js';
6
+ import { isCapabilityUrl } from '../identity/query-policy.js';
7
+ import { isResolvable, maxUriBytes, shareableWebUri } from '../identity/uri.js';
8
+ import { mutate } from '../store/changes.js';
9
+ import { sha256 } from '../util/hash.js';
6
10
  import { jungleTags } from './jungle-digest.js';
7
11
  /**
8
12
  * The library side of `sponge import jungle`: one Jungle Reading note onto the item holding its capture.
9
13
  * `planJungleNote` only reads; `applyJungleNote` runs the same plan inside the caller's transaction, through the logged paths.
10
14
  */
11
- export const jungleNamespace = 'jungle', jungleRealm = 'import:jungle', jungleProducer = { importer: 'sponge-jungle-reading', version: '1' };
15
+ export const jungleNamespace = 'jungle', jungleRealm = 'import:jungle';
16
+ const jungleProducer = { importer: 'sponge-jungle-reading', version: '2' };
12
17
  /** Receipts of the importer's own visibility writes (local, like every import receipt), so a later user override is recognisable. */
13
18
  const visibilityReceipts = 'jungle-visibility';
19
+ /**
20
+ * The note's `reading.source_url` as a shareable URI: http(s) only, through the same derivation as a captured locator.
21
+ * A capability URL (query-policy.ts) is refused outright, so its stripped form never becomes a public address either.
22
+ */
23
+ function noteUri(sourceUrl) {
24
+ if (!/^https?:\/\//i.test(sourceUrl))
25
+ return { refused: 'no-source-url' };
26
+ if (isCapabilityUrl(sourceUrl))
27
+ return { refused: 'capability' };
28
+ let uri;
29
+ try {
30
+ uri = shareableWebUri(sourceUrl);
31
+ }
32
+ catch {
33
+ return { refused: 'no-source-url' };
34
+ }
35
+ return Buffer.byteLength(uri) <= maxUriBytes && isResolvable(uri) && !isCapabilityUrl(uri)
36
+ ? { uri }
37
+ : { refused: 'no-source-url' };
38
+ }
39
+ /** What a local capture's URI action would be, before or after its item exists. */
40
+ export function uriAction(local, firstNote, sourceUrl, recorded) {
41
+ if (!firstNote)
42
+ return { action: 'other-note' };
43
+ if (!local)
44
+ return { action: 'web-capture' };
45
+ const { uri, refused } = noteUri(sourceUrl);
46
+ return uri ? { action: uri === recorded ? 'unchanged' : 'new', uri } : { action: refused };
47
+ }
48
+ function titleAction(db, input) {
49
+ if (input.firstNote === false)
50
+ return 'other-note';
51
+ const title = input.editorial.title.trim(), item = db.prepare('SELECT title_note,title_override FROM items WHERE id=?').get(input.itemId);
52
+ if (item?.title_override !== null && item?.title_override !== undefined)
53
+ return 'kept-user-edit';
54
+ return !title || Buffer.byteLength(title) > 4096 || title === item?.title_note ? 'unchanged' : 'set';
55
+ }
14
56
  /** What Jungle's `reading.status` asks for: `published` is Jungle's explicit public approval, `draft` stays private. */
15
- export function desiredVisibility(status) { return status === 'published' ? 'public' : status === 'draft' ? 'private' : undefined; }
57
+ function desiredVisibility(status) {
58
+ return status === 'published' ? 'public' : status === 'draft' ? 'private' : undefined;
59
+ }
16
60
  /**
17
61
  * Jungle's decision becomes an explicit override only while nobody else has set one: once the latest override was not
18
62
  * written by this importer, the user's choice wins. A public decision never lifts the capability-URL or local-file guards.
@@ -30,66 +74,117 @@ export function visibilityAction(state, status, userOverride) {
30
74
  return { action: 'unchanged' };
31
75
  return { action: target === null ? 'restore-default' : target === 'public' ? 'set-public' : 'set-private', target };
32
76
  }
33
- const allAccepted = (o) => ({ gist: 'accepted', ideas: o.ideas.map(() => 'accepted'), quotes: o.ideas.map(i => i.quotes.map(() => 'accepted')) });
34
- const imported = (r) => { try {
35
- return JSON.parse(String(r.producer_json))?.importer === jungleProducer.importer;
36
- }
37
- catch {
38
- return false;
39
- } };
40
- const latestVisibility = (db, itemId) => db.prepare("SELECT id FROM changes WHERE entity='item' AND entity_id=? AND field='visibility' ORDER BY seq DESC LIMIT 1").get(itemId);
77
+ const allAccepted = (o) => ({
78
+ gist: 'accepted',
79
+ ideas: o.ideas.map(() => 'accepted'),
80
+ quotes: o.ideas.map((i) => i.quotes.map(() => 'accepted')),
81
+ });
82
+ const imported = (r) => {
83
+ try {
84
+ return JSON.parse(String(r.producer_json))?.importer === jungleProducer.importer;
85
+ }
86
+ catch {
87
+ return false;
88
+ }
89
+ };
90
+ const latestVisibility = (db, itemId) => db
91
+ .prepare("SELECT id FROM changes WHERE entity='item' AND entity_id=? AND field='visibility' ORDER BY seq DESC LIMIT 1")
92
+ .get(itemId);
41
93
  /** The latest logged visibility write was not this importer's: the user chose, even when the choice was `default`. */
42
94
  function userOverride(db, itemId) {
43
95
  const latest = latestVisibility(db, itemId);
44
96
  if (!latest)
45
97
  return (db.prepare('SELECT visibility FROM items WHERE id=?').get(itemId)?.visibility ?? null) !== null;
46
- return !db.prepare('SELECT 1 FROM import_receipts WHERE namespace=? AND source_key=? AND source_hash=?').get(visibilityReceipts, itemId, digest(String(latest.id)));
98
+ return !db
99
+ .prepare('SELECT 1 FROM import_receipts WHERE namespace=? AND source_key=? AND source_hash=?')
100
+ .get(visibilityReceipts, itemId, sha256(String(latest.id)));
47
101
  }
48
102
  export function planJungleNote(db, input) {
49
103
  const { itemId, revisionId, editorial } = input, editorialId = derivationId(revisionId, 'editorial', editorial.markdownSha256, canonicalEditorial(editorial));
50
- const editorials = db.prepare("SELECT id,review_json FROM derivations WHERE item_id=? AND kind='editorial'").all(itemId);
51
- const editorialAction = editorials.some(r => r.id === editorialId) ? 'unchanged' : editorials.some(r => JSON.parse(String(r.review_json)).note !== 'accepted') ? 'user-review-wins' : 'new';
52
- let v1 = 'not-digest', v1Id;
53
- if (editorial.form === 'digest' && !input.v1)
54
- v1 = 'not-convertible';
55
- else if (input.v1) {
56
- const digests = db.prepare("SELECT id,producer_json,output_json,review_json FROM derivations WHERE item_id=? AND kind='digest'").all(itemId), parsed = parseDigest(input.v1).digest;
57
- v1Id = parsed && derivationId(revisionId, 'digest', editorialId, canonicalDigest(parsed));
58
- if (!parsed)
59
- v1 = 'not-convertible';
60
- else if (digests.some(r => r.id === v1Id))
61
- v1 = 'unchanged';
62
- else if (digests.some(r => !imported(r)))
63
- v1 = 'user-digest-wins';
64
- else if (digests.some(r => JSON.stringify(JSON.parse(String(r.review_json))) !== JSON.stringify(allAccepted(JSON.parse(String(r.output_json))))))
65
- v1 = 'user-review-wins';
104
+ const editorials = db
105
+ .prepare("SELECT id,review_json FROM derivations WHERE item_id=? AND kind='editorial'")
106
+ .all(itemId);
107
+ const editorialAction = editorials.some((r) => r.id === editorialId)
108
+ ? 'unchanged'
109
+ : editorials.some((r) => JSON.parse(String(r.review_json)).note !== 'accepted')
110
+ ? 'user-review-wins'
111
+ : 'new';
112
+ // Any earlier digest of the item, v1 included, takes part: an unchanged imported one is superseded by the v2 import, a user's is never.
113
+ let action = 'not-digest', digestId;
114
+ if (editorial.form === 'digest' && !input.digest)
115
+ action = 'not-convertible';
116
+ else if (input.digest) {
117
+ const digests = db
118
+ .prepare("SELECT id,producer_json,output_json,review_json FROM derivations WHERE item_id=? AND kind='digest'")
119
+ .all(itemId), parsed = parseDigest(input.digest).digest;
120
+ digestId = parsed && derivationId(revisionId, 'digest', editorialId, canonicalDigest(parsed));
121
+ if (!parsed || parsed.schemaVersion !== 2)
122
+ action = 'not-convertible';
123
+ else if (digests.some((r) => r.id === digestId))
124
+ action = 'unchanged';
125
+ else if (digests.some((r) => !imported(r)))
126
+ action = 'user-digest-wins';
127
+ else if (digests.some((r) => JSON.stringify(JSON.parse(String(r.review_json))) !==
128
+ JSON.stringify(allAccepted(JSON.parse(String(r.output_json))))))
129
+ action = 'user-review-wins';
66
130
  else
67
- v1 = editorialAction === 'user-review-wins' ? 'user-review-wins' : 'new';
131
+ action = editorialAction === 'user-review-wins' ? 'user-review-wins' : 'new';
68
132
  }
69
- const visibility = visibilityAction(visibilityState(db, itemId), editorial.status, userOverride(db, itemId)).action;
70
- return { editorialId, editorial: editorialAction, ...(v1Id ? { v1Id } : {}), v1, visibility };
133
+ const local = !hasWebUri(db, itemId), uri = uriAction(local, input.firstNote !== false, editorial.sourceUrl, metadataUri(db, itemId));
134
+ const visibility = visibilityAction(visibilityState(db, itemId, uri.uri ?? (input.firstNote === false ? input.firstNoteUri : undefined) ?? metadataUri(db, itemId)), editorial.status, userOverride(db, itemId)).action;
135
+ return {
136
+ editorialId,
137
+ editorial: editorialAction,
138
+ ...(digestId ? { digestId } : {}),
139
+ digest: action,
140
+ uri: uri.action,
141
+ title: titleAction(db, input),
142
+ visibility,
143
+ };
71
144
  }
72
145
  /**
73
146
  * Apply one note; the caller holds the write transaction. A new editorial object is accepted through `review.note` unless
74
- * the user already rejected or cleared an earlier one; a clean Digest v1 is stored and accepted per field, linked to the
147
+ * the user already rejected or cleared an earlier one; a converted Digest v2 is stored and accepted per field, linked to the
75
148
  * editorial object by its input hash, unless the user owns a digest of this item or changed an imported digest's review.
76
149
  */
77
150
  export function applyJungleNote(db, input) {
78
151
  const plan = planJungleNote(db, input), producer = { ...jungleProducer, note: input.editorial.slug };
79
152
  if (plan.editorial !== 'unchanged') {
80
- putEditorial(db, { itemId: input.itemId, revisionId: input.revisionId, producer, output: input.editorial });
153
+ putDerivation(db, 'editorial', {
154
+ itemId: input.itemId,
155
+ revisionId: input.revisionId,
156
+ producer,
157
+ output: input.editorial,
158
+ });
81
159
  if (plan.editorial === 'new')
82
- reviewEditorial(db, plan.editorialId, 'accepted');
160
+ reviewDerivation(db, 'editorial', plan.editorialId, 'accepted');
83
161
  }
84
- if (plan.v1 === 'new') {
85
- const stored = putDigest(db, { itemId: input.itemId, revisionId: input.revisionId, producer: { ...producer, editorial: plan.editorialId }, inputHash: plan.editorialId, output: input.v1, options: { tags: jungleTags } });
86
- reviewDigest(db, stored.id, allAccepted(input.v1));
162
+ // A local capture takes the note's web address and title before visibility is decided, so the published state applies to it.
163
+ if (plan.uri === 'new')
164
+ setMetadataUri(db, input.itemId, uriAction(true, true, input.editorial.sourceUrl).uri);
165
+ // The note's title is the item's default; a refresh changes only the captured title, and the user's override wins over both.
166
+ const title = input.editorial.title.trim();
167
+ if (input.firstNote !== false &&
168
+ title &&
169
+ Buffer.byteLength(title) <= 4096 &&
170
+ title !== db.prepare('SELECT title_note FROM items WHERE id=?').get(input.itemId)?.title_note)
171
+ mutate(db, 'item', input.itemId, 'noteTitle', title);
172
+ if (plan.digest === 'new') {
173
+ const stored = putDerivation(db, 'digest', {
174
+ itemId: input.itemId,
175
+ revisionId: input.revisionId,
176
+ producer: { ...producer, editorial: plan.editorialId },
177
+ inputHash: plan.editorialId,
178
+ output: input.digest,
179
+ options: { tags: jungleTags },
180
+ });
181
+ reviewDerivation(db, 'digest', stored.id, allAccepted(input.digest));
87
182
  }
88
183
  const action = visibilityAction(visibilityState(db, input.itemId), input.editorial.status, userOverride(db, input.itemId));
89
184
  if (action.target !== undefined) {
90
185
  setVisibility(db, input.itemId, action.target);
91
186
  const latest = latestVisibility(db, input.itemId);
92
- db.prepare('INSERT OR IGNORE INTO import_receipts VALUES(?,?,?,?,?,?)').run(visibilityReceipts, input.itemId, digest(String(latest.id)), jungleProducer.version, input.itemId, 'visibility');
187
+ db.prepare('INSERT OR IGNORE INTO import_receipts VALUES(?,?,?,?,?,?)').run(visibilityReceipts, input.itemId, sha256(String(latest.id)), jungleProducer.version, input.itemId, 'visibility');
93
188
  }
94
189
  return plan;
95
190
  }
@@ -1,13 +1,14 @@
1
1
  import type { LibraryStore } from '../store/index.js';
2
- import { type DigestV1 } from '../digest/schema.js';
3
- import { type JungleEditorial } from './jungle-digest.js';
4
- import { type DigestAction, type EditorialAction, type VisibilityAction } from './jungle-store.js';
2
+ import { type QuoteDrop } from './jungle-digest.js';
3
+ import { type DigestAction, type EditorialAction, type TitleAction, type UriAction, type VisibilityAction } from './jungle-store.js';
5
4
  import type { BundleVerifier } from './inspect.js';
6
5
  /**
7
6
  * `sponge import jungle DIR`: every Jungle Reading note (`kb/notes/reading/<slug>.md`) onto the item holding its capture.
8
7
  * The capture goes through the verified bundle import (always as a private association); the note becomes an accepted
9
- * editorial derivation, plus an accepted Digest v1 when it converts with nothing dropped; `reading.status` becomes the
10
- * item's visibility. Jungle is only read. A dry run reads the notes, captures and library and writes nothing.
8
+ * editorial derivation, plus an accepted Digest v2 when it converts without altering owner text (non-verbatim, short and
9
+ * overflowing quotes are dropped and reported); the note's title becomes the item's default title (a user override still wins), and a
10
+ * local capture (no web URI of its own) takes the note's `source_url`;
11
+ * `reading.status` becomes the item's visibility. Jungle is only read. A dry run reads the notes, captures and library and writes nothing.
11
12
  */
12
13
  export interface JungleImportOptions {
13
14
  verify?: BundleVerifier;
@@ -16,22 +17,34 @@ export interface JungleImportOptions {
16
17
  offset?: number;
17
18
  signal?: AbortSignal;
18
19
  }
19
- export type JungleSkip = 'not-reading-note' | 'malformed-note' | 'no-capture' | 'capture-missing' | 'item-deleted' | 'capture-unsupported' | 'capture-failed';
20
- export interface JungleEntry {
20
+ type JungleSkip = 'not-reading-note' | 'malformed-note' | 'no-capture' | 'capture-missing' | 'item-deleted' | 'capture-unsupported' | 'capture-failed';
21
+ /** Per note: quotes assigned to ideas, and each dropped quote by its 0-based position in the note's Quotes section. */
22
+ interface JungleQuoteReport {
23
+ assigned: number;
24
+ dropped: {
25
+ quote: number;
26
+ reason: QuoteDrop;
27
+ }[];
28
+ }
29
+ interface JungleEntry {
21
30
  slug: string;
22
31
  form?: string;
23
32
  status?: string;
24
33
  item?: 'new' | 'existing';
25
34
  itemId?: string;
26
35
  editorial?: EditorialAction;
27
- v1?: DigestAction;
28
- v1Issues?: string[];
36
+ digest?: DigestAction;
37
+ digestIssues?: string[];
38
+ quotes?: JungleQuoteReport;
39
+ uri?: UriAction;
40
+ title?: TitleAction;
29
41
  visibility?: VisibilityAction;
30
42
  skipped?: JungleSkip;
31
43
  code?: string;
32
44
  }
45
+ /** Schema 2: digest actions are Digest v2 (`digest`, `digestIssues`; counts add notes per reason and quotes assigned or dropped); schema 1 reported Digest v1 as `v1`. `uri` and `title` were added within schema 2. */
33
46
  export interface JungleImportReport {
34
- schemaVersion: 1;
47
+ schemaVersion: 2;
35
48
  dryRun: boolean;
36
49
  captureVerification: 'deferred' | 'verified';
37
50
  notes: number;
@@ -39,16 +52,19 @@ export interface JungleImportReport {
39
52
  newItems: number;
40
53
  existingItems: number;
41
54
  editorial: Record<string, number>;
42
- v1: Record<string, number>;
55
+ digest: Record<string, number>;
56
+ digestIssues: Record<string, number>;
57
+ quotes: {
58
+ assigned: number;
59
+ dropped: Record<string, number>;
60
+ };
61
+ uri: Record<string, number>;
62
+ title: Record<string, number>;
43
63
  visibility: Record<string, number>;
44
64
  skipped: Record<string, number>;
45
65
  };
46
66
  entries: JungleEntry[];
47
67
  nextOffset: number | null;
48
68
  }
49
- /** Digest v1 only when the note converts with nothing dropped or trimmed and passes every rule against this revision's Markdown. */
50
- export declare function jungleV1(editorial: JungleEditorial, markdown: string): {
51
- v1?: DigestV1;
52
- issues: string[];
53
- };
54
69
  export declare function importJungle(store: LibraryStore, jungle: string, options?: JungleImportOptions): Promise<JungleImportReport>;
70
+ export {};
@@ -1,47 +1,107 @@
1
1
  import { readdir, realpath } from 'node:fs/promises';
2
2
  import { dirname, isAbsolute, join } from 'node:path';
3
- import { readConfined, hash } from '../acquisition/files.js';
3
+ import { readConfined } from '../acquisition/files.js';
4
+ import { sha256 as hash } from '../util/hash.js';
4
5
  import { object } from '../acquisition/normalize.js';
5
6
  import { localizeMarkdown } from '../app/markdown.js';
6
7
  import { checkDigest } from '../digest/schema.js';
7
8
  import { accessClass, defaultVisibility } from '../identity/uri.js';
8
- import { convertJungleDigest, parseJungleEditorial, jungleTags, JungleDigestError } from './jungle-digest.js';
9
- import { jungleNamespace, jungleRealm, visibilityAction } from './jungle-store.js';
9
+ import { convertJungleDigest, parseJungleEditorial, jungleTags, } from './jungle-digest.js';
10
+ import { jungleNamespace, jungleRealm, uriAction, visibilityAction, } from './jungle-store.js';
10
11
  import { importBundleEntry, importBundleHash } from './apply.js';
12
+ import { SpongeError, codeOf } from '../errors.js';
11
13
  const noteBytes = 4 * 1024 ** 2, captureBytes = 16 * 1024 ** 2;
12
- const errorCode = (error) => String(error?.code ?? (error instanceof Error ? error.message : 'unclassified'));
14
+ const errorCode = (error) => codeOf(error);
13
15
  /** A capture link is `<dir>/<name>` under `kb/`, never escaping it; the bundle is its directory. */
14
- function captureLink(link) { if (isAbsolute(link) || link.includes('\\') || link.split('/').length < 2 || link.split('/').some(p => !p || p === '.' || p === '..'))
15
- throw new JungleDigestError('no-capture'); return link; }
16
- /** Digest v1 only when the note converts with nothing dropped or trimmed and passes every rule against this revision's Markdown. */
17
- export function jungleV1(editorial, markdown) {
16
+ function captureLink(link) {
17
+ if (isAbsolute(link) ||
18
+ link.includes('\\') ||
19
+ link.split('/').length < 2 ||
20
+ link.split('/').some((p) => !p || p === '.' || p === '..'))
21
+ throw new SpongeError('no-capture');
22
+ return link;
23
+ }
24
+ /** Digest v2 when the note converts and passes every rule against this revision's Markdown; otherwise the reasons. */
25
+ function jungleDigest(editorial, markdown) {
18
26
  if (editorial.form !== 'digest')
19
27
  return { issues: [] };
20
- const c = convertJungleDigest(editorial.markdown, markdown), check = checkDigest(c.gold, markdown, { tags: jungleTags }), issues = [...c.lossy, ...c.issues.map(i => `${i.path}:${i.code}`), ...check.issues.map(i => `${i.path}:${i.code}`)];
21
- return c.clean && check.ok ? { v1: c.gold, issues: [] } : { issues: [...new Set(issues)] };
28
+ const c = convertJungleDigest(editorial.markdown, markdown), check = checkDigest(c.gold, markdown, { tags: jungleTags }), issues = [...new Set([...c.issues, ...check.issues].map((i) => `${i.path}:${i.code}`))];
29
+ const quotes = {
30
+ assigned: c.quotes.filter((q) => q.idea !== null).length,
31
+ dropped: c.quotes.flatMap((q, k) => (q.dropped ? [{ quote: k, reason: q.dropped }] : [])),
32
+ };
33
+ return c.digest && check.ok ? { digest: c.digest, issues: [], quotes } : { issues, quotes };
22
34
  }
23
35
  /** What a verified import would store, read from the bundle without the verifier: the same localization, from the manifest's asset list. */
24
36
  async function previewCapture(kb, bundle, link) {
37
+ // An untrusted bundle fails as `integrity`, exactly as apply's `inspectImportBundle` reports it.
38
+ try {
39
+ return await previewBundle(kb, bundle, link);
40
+ }
41
+ catch (error) {
42
+ if (error instanceof SpongeError)
43
+ throw error;
44
+ throw new SpongeError('integrity', { cause: error });
45
+ }
46
+ }
47
+ async function previewBundle(kb, bundle, link) {
25
48
  const manifest = object(JSON.parse((await readConfined(kb, join(bundle, 'capture.json'), captureBytes)).toString('utf8'))), pdf = manifest.kind === 'pdf';
26
- const listed = (pdf ? (Array.isArray(manifest.images) ? manifest.images.map(i => object(i).asset) : []) : Array.isArray(manifest.assets) ? manifest.assets : []).flatMap(raw => { const a = raw && typeof raw === 'object' ? raw : {}; return typeof a.path === 'string' && typeof a.sha256 === 'string' ? [{ relativePath: a.path, sha256: a.sha256, bytes: Number(a.bytes), mediaType: String(a.mimeType) }] : []; });
49
+ const listed = (pdf
50
+ ? Array.isArray(manifest.images)
51
+ ? manifest.images.map((i) => object(i).asset)
52
+ : []
53
+ : Array.isArray(manifest.assets)
54
+ ? manifest.assets
55
+ : []).flatMap((raw) => {
56
+ const a = raw && typeof raw === 'object' ? raw : {};
57
+ return typeof a.path === 'string' && typeof a.sha256 === 'string'
58
+ ? [{ relativePath: a.path, sha256: a.sha256, bytes: Number(a.bytes), mediaType: String(a.mimeType) }]
59
+ : [];
60
+ });
27
61
  const text = (await readConfined(kb, link + '.md', captureBytes)).toString('utf8'), markdown = localizeMarkdown(pdf ? text.replace(/^<!-- pdf-page: ([1-9][0-9]*) -->$/gm, '## Page $1') : text, listed).markdown;
28
62
  const locator = pdf ? String(object(manifest.source ?? {}).requestedUrl ?? '') : String(manifest.sourceUrl ?? '');
29
63
  return { markdown, locator };
30
64
  }
31
65
  export async function importJungle(store, jungle, options = {}) {
32
66
  if (!isAbsolute(jungle))
33
- throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
34
- const kb = await realpath(join(jungle, 'kb')), notesDir = join(kb, 'notes', 'reading'), files = (await readdir(notesDir)).filter(f => /^[a-z0-9][a-z0-9-]*\.md$/.test(f)).sort();
67
+ throw new SpongeError('invalid-input');
68
+ const kb = await realpath(join(jungle, 'kb')), notesDir = join(kb, 'notes', 'reading'), files = (await readdir(notesDir)).filter((f) => /^[a-z0-9][a-z0-9-]*\.md$/.test(f)).sort();
35
69
  const limit = options.limit ?? 1000, offset = options.offset ?? 0, dryRun = options.dryRun !== false;
36
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000 || !Number.isSafeInteger(offset) || offset < 0 || offset > files.length)
37
- throw Object.assign(Error('invalid-input'), { code: 'invalid-input' });
70
+ if (!Number.isSafeInteger(limit) ||
71
+ limit < 1 ||
72
+ limit > 1000 ||
73
+ !Number.isSafeInteger(offset) ||
74
+ offset < 0 ||
75
+ offset > files.length)
76
+ throw new SpongeError('invalid-input');
38
77
  if (!dryRun && !options.verify)
39
- throw Object.assign(Error('runtime-missing'), { code: 'runtime-missing' });
40
- const report = { schemaVersion: 1, dryRun, captureVerification: dryRun ? 'deferred' : 'verified', notes: files.length, counts: { newItems: 0, existingItems: 0, editorial: {}, v1: {}, visibility: {}, skipped: {} }, entries: [], nextOffset: offset + limit < files.length ? offset + limit : null };
41
- const bump = (m, k) => { m[k] = (m[k] ?? 0) + 1; }, seen = new Set();
78
+ throw new SpongeError('runtime-missing');
79
+ const report = {
80
+ schemaVersion: 2,
81
+ dryRun,
82
+ captureVerification: dryRun ? 'deferred' : 'verified',
83
+ notes: files.length,
84
+ counts: {
85
+ newItems: 0,
86
+ existingItems: 0,
87
+ editorial: {},
88
+ digest: {},
89
+ digestIssues: {},
90
+ quotes: { assigned: 0, dropped: {} },
91
+ uri: {},
92
+ title: {},
93
+ visibility: {},
94
+ skipped: {},
95
+ },
96
+ entries: [],
97
+ nextOffset: offset + limit < files.length ? offset + limit : null,
98
+ };
99
+ const bump = (m, k) => {
100
+ m[k] = (m[k] ?? 0) + 1;
101
+ }, seen = new Map();
42
102
  for (const file of files.slice(offset, offset + limit)) {
43
103
  if (options.signal?.aborted)
44
- throw Object.assign(Error('cancelled'), { code: 'cancelled' });
104
+ throw new SpongeError('cancelled');
45
105
  const slug = file.slice(0, -3);
46
106
  let entry = { slug };
47
107
  try {
@@ -50,8 +110,9 @@ export async function importJungle(store, jungle, options = {}) {
50
110
  editorial = parseJungleEditorial((await readConfined(notesDir, file, noteBytes)).toString('utf8'));
51
111
  }
52
112
  catch (error) {
53
- const code = error instanceof JungleDigestError ? error.code : 'malformed';
54
- entry.skipped = code === 'not-digest' ? 'not-reading-note' : code === 'no-capture' ? 'no-capture' : 'malformed-note';
113
+ const raw = codeOf(error), code = raw === 'not-digest' || raw === 'no-capture' ? raw : 'malformed';
114
+ entry.skipped =
115
+ code === 'not-digest' ? 'not-reading-note' : code === 'no-capture' ? 'no-capture' : 'malformed-note';
55
116
  throw entry;
56
117
  }
57
118
  entry = { slug, form: editorial.form, status: editorial.status };
@@ -71,23 +132,51 @@ export async function importJungle(store, jungle, options = {}) {
71
132
  entry.skipped = 'capture-missing';
72
133
  throw entry;
73
134
  }
74
- const prior = store.importedItem(jungleRealm, hash(JSON.stringify([jungleNamespace, link])));
135
+ const prior = store.imports.item(jungleRealm, hash(JSON.stringify([jungleNamespace, link])));
75
136
  if (prior?.deleted) {
76
137
  entry.skipped = 'item-deleted';
77
138
  entry.itemId = prior.itemId;
78
139
  throw entry;
79
140
  }
80
141
  // Two notes on one capture share its item, in a dry run too.
81
- entry.item = prior || seen.has(link) ? 'existing' : 'new';
82
- seen.add(link);
142
+ // The first note on a capture decides its default title and a local capture's URI; later notes see the URI it records.
143
+ const firstNote = !seen.has(link), firstNoteUri = seen.get(link);
144
+ entry.item = prior || !firstNote ? 'existing' : 'new';
145
+ if (firstNote)
146
+ seen.set(link, uriAction(true, true, editorial.sourceUrl).uri);
83
147
  if (dryRun) {
84
148
  if (prior?.revisionId) {
85
- const markdown = store.revisionMarkdown(prior.revisionId), { v1, issues } = jungleV1(editorial, markdown), plan = store.planJungleNote({ itemId: prior.itemId, revisionId: prior.revisionId, editorial, ...(v1 ? { v1 } : {}) });
86
- Object.assign(entry, { itemId: prior.itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
149
+ const markdown = store.revisions.markdown(prior.revisionId), { digest, issues, quotes } = jungleDigest(editorial, markdown), plan = store.imports.planJungleNote({
150
+ itemId: prior.itemId,
151
+ revisionId: prior.revisionId,
152
+ editorial,
153
+ ...(digest ? { digest } : {}),
154
+ firstNote,
155
+ ...(firstNoteUri ? { firstNoteUri } : {}),
156
+ });
157
+ Object.assign(entry, {
158
+ itemId: prior.itemId,
159
+ editorial: plan.editorial,
160
+ digest: plan.digest,
161
+ ...(issues.length ? { digestIssues: issues } : {}),
162
+ ...(quotes ? { quotes } : {}),
163
+ uri: plan.uri,
164
+ title: plan.title,
165
+ visibility: plan.visibility,
166
+ });
87
167
  }
168
+ // A capture without a web locator is stored as a local file; predict the URI, title and access it would get.
88
169
  else {
89
- const preview = await previewCapture(kb, bundle, link), { v1, issues } = jungleV1(editorial, preview.markdown), access = accessClass(jungleRealm, preview.locator);
90
- Object.assign(entry, { editorial: 'new', v1: editorial.form !== 'digest' ? 'not-digest' : v1 ? 'new' : 'not-convertible', ...(issues.length ? { v1Issues: issues } : {}), visibility: visibilityAction({ access, default: defaultVisibility(access, 'private'), override: null }, editorial.status, false).action });
170
+ const preview = await previewCapture(kb, bundle, link), { digest, issues, quotes } = jungleDigest(editorial, preview.markdown), local = !/^https?:/i.test(preview.locator), uri = uriAction(local, firstNote, editorial.sourceUrl), access = accessClass(jungleRealm, local ? 'file:' : preview.locator, uri.uri ?? firstNoteUri), noteTitle = editorial.title.trim();
171
+ Object.assign(entry, {
172
+ editorial: 'new',
173
+ digest: editorial.form !== 'digest' ? 'not-digest' : digest ? 'new' : 'not-convertible',
174
+ ...(issues.length ? { digestIssues: issues } : {}),
175
+ ...(quotes ? { quotes } : {}),
176
+ uri: uri.action,
177
+ title: !firstNote ? 'other-note' : noteTitle && Buffer.byteLength(noteTitle) <= 4096 ? 'set' : 'unchanged',
178
+ visibility: visibilityAction({ access, default: defaultVisibility(access, 'private'), override: null }, editorial.status, false).action,
179
+ });
91
180
  }
92
181
  }
93
182
  else {
@@ -103,12 +192,36 @@ export async function importJungle(store, jungle, options = {}) {
103
192
  entry.code = code;
104
193
  throw entry;
105
194
  }
106
- const itemId = imported.itemId, revisionId = store.importedRevision(itemId, importBundleHash(imported.sourceHash)), markdown = store.revisionMarkdown(revisionId), { v1, issues } = jungleV1(editorial, markdown);
107
- const plan = store.applyJungleNote({ itemId, revisionId, editorial, ...(v1 ? { v1 } : {}) });
108
- Object.assign(entry, { itemId, editorial: plan.editorial, v1: plan.v1, ...(issues.length ? { v1Issues: issues } : {}), visibility: plan.visibility });
195
+ const itemId = imported.itemId, revisionId = store.imports.revision(itemId, importBundleHash(imported.sourceHash)), markdown = store.revisions.markdown(revisionId), { digest, issues, quotes } = jungleDigest(editorial, markdown);
196
+ const plan = store.imports.applyJungleNote({
197
+ itemId,
198
+ revisionId,
199
+ editorial,
200
+ ...(digest ? { digest } : {}),
201
+ firstNote,
202
+ });
203
+ Object.assign(entry, {
204
+ itemId,
205
+ editorial: plan.editorial,
206
+ digest: plan.digest,
207
+ ...(issues.length ? { digestIssues: issues } : {}),
208
+ ...(quotes ? { quotes } : {}),
209
+ uri: plan.uri,
210
+ title: plan.title,
211
+ visibility: plan.visibility,
212
+ });
109
213
  }
110
214
  bump(report.counts.editorial, entry.editorial);
111
- bump(report.counts.v1, entry.v1);
215
+ bump(report.counts.digest, entry.digest);
216
+ for (const i of new Set((entry.digestIssues ?? []).map((i) => i.replace(/\.\d+/g, '.N'))))
217
+ bump(report.counts.digestIssues, i);
218
+ if (entry.quotes) {
219
+ report.counts.quotes.assigned += entry.quotes.assigned;
220
+ for (const d of entry.quotes.dropped)
221
+ bump(report.counts.quotes.dropped, d.reason);
222
+ }
223
+ bump(report.counts.uri, entry.uri);
224
+ bump(report.counts.title, entry.title);
112
225
  bump(report.counts.visibility, entry.visibility);
113
226
  if (entry.item === 'new')
114
227
  report.counts.newItems++;