@hraness/sponge 0.2.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +23 -10
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +39 -17
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +10 -46
  64. package/dist/src/digest/editorial.js +73 -59
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +53 -32
  67. package/dist/src/digest/pipeline.js +357 -125
  68. package/dist/src/digest/schema.d.ts +44 -17
  69. package/dist/src/digest/schema.js +132 -44
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +31 -24
  82. package/dist/src/importers/jungle-digest.js +217 -115
  83. package/dist/src/importers/jungle-store.d.ts +20 -11
  84. package/dist/src/importers/jungle-store.js +136 -41
  85. package/dist/src/importers/jungle.d.ts +32 -16
  86. package/dist/src/importers/jungle.js +145 -32
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +108 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -77
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -1,37 +1,57 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { foldText, locateAll, words } from '../annotations/quote.js';
3
- import { checkDigest, digestLimits as L } from '../digest/schema.js';
4
- import { minQuoteWords, snapQuote } from '../digest/pipeline.js';
2
+ import { foldText, words } from '../annotations/quote.js';
3
+ import { checkDigest, digestLimits as L, parseDigest, } from '../digest/schema.js';
4
+ import { snapQuote } from '../digest/pipeline.js';
5
+ import { SpongeError } from '../errors.js';
5
6
  /**
6
- * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v1. Jungle keeps
7
- * quotes in their own section, so each quote is assigned to the idea it shares the most content words with.
7
+ * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v2. Jungle keeps
8
+ * quotes in their own section, so each verbatim quote is assigned to the idea it shares the most content words with.
9
+ * Owner text is never cut or rewritten: a note that does not fit v2 as written is reported, not converted.
8
10
  * Nothing here reads files; callers pass the note and, when available, its capture Markdown.
9
11
  */
10
- export const jungleTags = ['agent-eng', 'ai', 'work', 'business', 'creative-coding', 'finance', 'music', 'internet', 'security', 'history', 'art', 'hardware', 'climate', 'article', 'interactive', 'code', 'x', 'video'];
11
- /** Tags that name how a source is consumed; Digest v1 carries that as `form`. */
12
+ export const jungleTags = [
13
+ 'agent-eng',
14
+ 'ai',
15
+ 'work',
16
+ 'business',
17
+ 'creative-coding',
18
+ 'finance',
19
+ 'music',
20
+ 'internet',
21
+ 'security',
22
+ 'history',
23
+ 'art',
24
+ 'hardware',
25
+ 'climate',
26
+ 'article',
27
+ 'interactive',
28
+ 'code',
29
+ 'x',
30
+ 'video',
31
+ ];
32
+ /** Tags that name how a source is consumed; a digest carries that as `form`. */
12
33
  export const jungleFormatTags = ['article', 'interactive', 'x', 'video'];
13
- export class JungleDigestError extends Error {
14
- code;
15
- constructor(code) {
16
- super(code);
17
- this.code = code;
18
- this.name = 'JungleDigestError';
19
- }
20
- }
21
- const fail = (code = 'malformed') => { throw new JungleDigestError(code); };
34
+ const fail = (code = 'malformed') => {
35
+ throw new SpongeError(code);
36
+ };
22
37
  /** The frontmatter subset Jungle writes: block maps and lists of scalars, quoted or plain. */
23
38
  function parseYaml(lines) {
24
39
  let i = 0;
25
40
  const indent = (l) => l.length - l.trimStart().length;
26
- const scalar = (v) => { v = v.trim(); if (v.startsWith('"')) {
27
- try {
28
- return JSON.parse(v);
29
- }
30
- catch {
31
- return fail();
41
+ const scalar = (v) => {
42
+ v = v.trim();
43
+ if (v.startsWith('"')) {
44
+ try {
45
+ return JSON.parse(v);
46
+ }
47
+ catch {
48
+ return fail();
49
+ }
32
50
  }
33
- } if (v.startsWith("'"))
34
- return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail(); return v === 'true' ? true : v === 'false' ? false : v; };
51
+ if (v.startsWith("'"))
52
+ return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail();
53
+ return v === 'true' ? true : v === 'false' ? false : v;
54
+ };
35
55
  const block = (at) => {
36
56
  if (lines[i].trimStart().startsWith('- ')) {
37
57
  const list = [];
@@ -55,12 +75,18 @@ function parseYaml(lines) {
55
75
  while (i < lines.length && indent(lines[i]) === at && !lines[i].trimStart().startsWith('- ')) {
56
76
  const m = /^([\w-]+):(?:\s+(.*))?$/.exec(lines[i].trim()) ?? fail();
57
77
  i++;
58
- if (m[2] !== undefined && m[2] !== '')
59
- out[m[1]] = scalar(m[2]);
60
- else if (i < lines.length && (indent(lines[i]) > at || indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
61
- out[m[1]] = block(indent(lines[i]));
62
- else
63
- out[m[1]] = '';
78
+ // An own data property even for `__proto__`, which plain assignment would turn into the map's prototype.
79
+ Object.defineProperty(out, m[1], {
80
+ value: m[2] !== undefined && m[2] !== ''
81
+ ? scalar(m[2])
82
+ : i < lines.length &&
83
+ (indent(lines[i]) > at || (indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
84
+ ? block(indent(lines[i]))
85
+ : '',
86
+ enumerable: true,
87
+ writable: true,
88
+ configurable: true,
89
+ });
64
90
  }
65
91
  return out;
66
92
  };
@@ -68,14 +94,21 @@ function parseYaml(lines) {
68
94
  return i === lines.length ? value : fail();
69
95
  }
70
96
  const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
71
- const strings = (v) => Array.isArray(v) && v.every(x => typeof x === 'string') ? v : fail();
97
+ const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === 'string') ? v : fail());
72
98
  /** Split a Markdown file into its frontmatter lines and body. */
73
99
  function frontmatter(markdown) {
74
100
  const m = /^---\n([^]*?)\n---(?:\n|$)/.exec(markdown.replace(/\r\n/g, '\n'));
75
- return m ? { head: m[1].split('\n').filter(l => l.trim() && !l.trimStart().startsWith('#')), body: markdown.replace(/\r\n/g, '\n').slice(m[0].length) } : { head: [], body: markdown };
101
+ return m
102
+ ? {
103
+ head: m[1].split('\n').filter((l) => l.trim() && !l.trimStart().startsWith('#')),
104
+ body: markdown.replace(/\r\n/g, '\n').slice(m[0].length),
105
+ }
106
+ : { head: [], body: markdown };
76
107
  }
77
108
  /** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
78
- export function captureBody(capture) { return frontmatter(capture).body; }
109
+ export function captureBody(capture) {
110
+ return frontmatter(capture).body;
111
+ }
79
112
  /** `## Heading` sections of a note body, trimmed. */
80
113
  function sectionsOf(body) {
81
114
  const sections = new Map();
@@ -86,10 +119,19 @@ function sectionsOf(body) {
86
119
  return sections;
87
120
  }
88
121
  /** `- **Claim.** detail` bullets: Jungle's Ideas, and a conversation's Debate. */
89
- const ideaList = (text) => text.split('\n').filter(l => l.startsWith('- ')).map(l => { const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail(); return { claim: m[1].trim(), detail: m[2].trim() }; });
122
+ const ideaList = (text) => text
123
+ .split('\n')
124
+ .filter((l) => l.startsWith('- '))
125
+ .map((l) => {
126
+ const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail();
127
+ return { claim: m[1].trim(), detail: m[2].trim() };
128
+ });
90
129
  /** Blockquotes ending `> Attribution: Speaker, context`. */
91
- const quoteList = (text) => text.split(/\n\s*\n/).filter(b => b.trim()).map(b => {
92
- const lines = b.split('\n').map(l => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex(l => l.startsWith('Attribution:'));
130
+ const quoteList = (text) => text
131
+ .split(/\n\s*\n/)
132
+ .filter((b) => b.trim())
133
+ .map((b) => {
134
+ const lines = b.split('\n').map((l) => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex((l) => l.startsWith('Attribution:'));
93
135
  if (at < 1)
94
136
  fail();
95
137
  const text = lines.slice(0, at).join(' ').replace(/\s+/g, ' ').trim(), attribution = lines[at].slice('Attribution:'.length).trim();
@@ -100,7 +142,7 @@ export function parseJungleNote(markdown) {
100
142
  fail();
101
143
  const { head, body } = frontmatter(markdown), meta = parseYaml(head);
102
144
  if (!object(meta) || !object(meta.reading))
103
- throw new JungleDigestError('not-digest');
145
+ throw new SpongeError('not-digest');
104
146
  const reading = meta.reading;
105
147
  if (reading.form !== 'digest')
106
148
  fail('not-digest');
@@ -108,63 +150,111 @@ export function parseJungleNote(markdown) {
108
150
  const gist = (sections.get('Gist') ?? fail()).replace(/\s+/g, ' ').trim();
109
151
  const ideas = ideaList(sections.get('Ideas') ?? fail());
110
152
  const quotes = quoteList(sections.get('Quotes') ?? '');
111
- if (!gist || !ideas.length || quotes.some(q => !q.text || !q.speaker))
153
+ if (!gist || !ideas.length || quotes.some((q) => !q.text || !q.speaker))
112
154
  fail();
113
- const authors = Array.isArray(reading.authors) ? reading.authors.map(a => object(a) && typeof a.name === 'string' ? a.name : fail()) : [];
155
+ const authors = Array.isArray(reading.authors)
156
+ ? reading.authors.map((a) => (object(a) && typeof a.name === 'string' ? a.name : fail()))
157
+ : [];
114
158
  const relations = object(meta.relations) ? meta.relations : {}, synth = Array.isArray(relations.synthesizes) ? strings(relations.synthesizes) : [];
115
- return { slug: typeof reading.slug === 'string' ? reading.slug : fail(), title: typeof meta.title === 'string' ? meta.title : '', status: typeof reading.status === 'string' ? reading.status : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', authors, tags: meta.tags === undefined ? [] : strings(meta.tags), ...(synth[0] ? { capture: synth[0] } : {}), gist, ideas, quotes };
159
+ return {
160
+ slug: typeof reading.slug === 'string' ? reading.slug : fail(),
161
+ title: typeof meta.title === 'string' ? meta.title : '',
162
+ status: typeof reading.status === 'string' ? reading.status : '',
163
+ sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
164
+ authors,
165
+ tags: meta.tags === undefined ? [] : strings(meta.tags),
166
+ ...(synth[0] ? { capture: synth[0] } : {}),
167
+ gist,
168
+ ideas,
169
+ quotes,
170
+ };
116
171
  }
117
172
  export function parseJungleEditorial(markdown) {
118
173
  if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
119
174
  fail();
120
175
  const { head, body } = frontmatter(markdown), meta = parseYaml(head);
121
176
  if (!object(meta) || !object(meta.reading))
122
- throw new JungleDigestError('not-digest');
177
+ throw new SpongeError('not-digest');
123
178
  const reading = meta.reading, relations = object(meta.relations) ? meta.relations : {};
124
179
  const form = reading.form, slug = reading.slug;
125
180
  if (typeof form !== 'string' || !form || typeof slug !== 'string' || !/^[a-z0-9][a-z0-9-]{0,199}$/.test(slug))
126
181
  return fail();
127
- const capture = [relations.synthesizes, relations['evidenced-by']].flatMap(v => Array.isArray(v) ? strings(v) : [])[0] ?? fail('no-capture');
182
+ const capture = [relations.synthesizes, relations['evidenced-by']].flatMap((v) => (Array.isArray(v) ? strings(v) : []))[0] ??
183
+ fail('no-capture');
128
184
  // A digest must parse strictly; the other forms keep whatever structure they have, and the Markdown keeps the rest.
129
- const sections = sectionsOf(body), loose = (f) => { if (form === 'digest')
130
- return f(); try {
131
- return f();
132
- }
133
- catch {
134
- return undefined;
135
- } };
136
- const gistText = sections.get('Gist') ?? sections.get('Summary') ?? (typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
185
+ const sections = sectionsOf(body), loose = (f) => {
186
+ if (form === 'digest')
187
+ return f();
188
+ try {
189
+ return f();
190
+ }
191
+ catch {
192
+ return undefined;
193
+ }
194
+ };
195
+ const gistText = sections.get('Gist') ??
196
+ sections.get('Summary') ??
197
+ (typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
137
198
  if (form === 'digest')
138
199
  parseJungleNote(markdown);
139
200
  const gist = gistText?.replace(/\s+/g, ' ').trim(), ideas = ideaText === undefined ? undefined : loose(() => ideaList(ideaText)), quotes = quoteText === undefined ? undefined : loose(() => quoteList(quoteText));
140
- return { schemaVersion: 1, source: 'jungle-reading', form, slug, status: typeof reading.status === 'string' ? reading.status : '', title: typeof meta.title === 'string' ? meta.title : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', capture, frontmatter: meta, ...(gist ? { gist } : {}), ...(ideas?.length ? { ideas } : {}), ...(quotes?.length && quotes.every(q => q.text && q.speaker) ? { quotes } : {}), markdownSha256: createHash('sha256').update(markdown).digest('hex'), markdown };
201
+ return {
202
+ schemaVersion: 1,
203
+ source: 'jungle-reading',
204
+ form,
205
+ slug,
206
+ status: typeof reading.status === 'string' ? reading.status : '',
207
+ title: typeof meta.title === 'string' ? meta.title : '',
208
+ sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
209
+ capture,
210
+ frontmatter: meta,
211
+ ...(gist ? { gist } : {}),
212
+ ...(ideas?.length ? { ideas } : {}),
213
+ ...(quotes?.length && quotes.every((q) => q.text && q.speaker) ? { quotes } : {}),
214
+ markdownSha256: createHash('sha256').update(markdown).digest('hex'),
215
+ markdown,
216
+ };
141
217
  }
142
218
  const stop = new Set('the a an and or but of to in on for with by from at as is are was were be been it its this that these those not no than then so such into about over under more most less can could would should will just also only very their there they them we our you your he she his her i my me what which who how when where why do does did has have had'.split(' '));
143
219
  /** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
144
220
  export function contentTerms(text) {
145
- return new Set(foldText(text).toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length > 2 && !stop.has(w)).map(w => w.replace(/(?:ing|ed|es|s)$/, '')).filter(w => w.length > 2));
221
+ return new Set(foldText(text)
222
+ .toLowerCase()
223
+ .split(/[^a-z0-9]+/)
224
+ .filter((w) => w.length > 2 && !stop.has(w))
225
+ .map((w) => w.replace(/(?:ing|ed|es|s)$/, ''))
226
+ .filter((w) => w.length > 2));
146
227
  }
147
- export function termSimilarity(a, b) { if (!a.size || !b.size)
148
- return 0; let n = 0; for (const t of a)
149
- if (b.has(t))
150
- n++; return n / Math.sqrt(a.size * b.size); }
151
- /** Best-first assignment: every idea gets at most one quote before any gets a second; zero-overlap pairs are never made. */
152
- export function assignQuotes(ideas, quotes) { return assign(ideas, quotes); }
153
- function assign(ideas, quotes) {
154
- const ideaTerms = ideas.map(i => contentTerms(`${i.claim} ${i.detail}`)), pairs = [];
155
- quotes.forEach((q, qi) => { const t = contentTerms(q.text); ideaTerms.forEach((it, ii) => { const s = termSimilarity(t, it); if (s > 0)
156
- pairs.push({ q: qi, i: ii, s }); }); });
157
- pairs.sort((a, b) => b.s - a.s || Math.abs(a.q - a.i) - Math.abs(b.q - b.i) || a.q - b.q);
158
- const out = quotes.map(() => null), load = ideas.map(() => 0);
159
- for (const cap of [1, L.maxQuotesPerIdea])
160
- for (const p of pairs)
161
- if (out[p.q] === null && load[p.i] < cap) {
162
- out[p.q] = p.i;
163
- load[p.i]++;
164
- }
165
- return out;
228
+ export function termSimilarity(a, b) {
229
+ if (!a.size || !b.size)
230
+ return 0;
231
+ let n = 0;
232
+ for (const t of a)
233
+ if (b.has(t))
234
+ n++;
235
+ return n / Math.sqrt(a.size * b.size);
166
236
  }
167
- export const firstSentence = (text) => { const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text); return m ? [m[1], text.slice(m[0].length)] : [text, '']; };
237
+ /**
238
+ * Each quote in order goes to the idea sharing the most content words with its claim and detail (ties: the earliest idea),
239
+ * else to the next best idea with room; null when every idea already holds `cap` quotes.
240
+ */
241
+ export function assignQuotes(ideas, quotes, cap = L.maxQuotesPerIdea) {
242
+ const terms = ideas.map((i) => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
243
+ return quotes.map((q) => {
244
+ const t = contentTerms(q.text), pick = terms
245
+ .map((it, i) => ({ i, s: termSimilarity(t, it) }))
246
+ .sort((a, b) => b.s - a.s || a.i - b.i)
247
+ .find((r) => load[r.i] < cap);
248
+ if (!pick)
249
+ return null;
250
+ load[pick.i]++;
251
+ return pick.i;
252
+ });
253
+ }
254
+ export const firstSentence = (text) => {
255
+ const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text);
256
+ return m ? [m[1], text.slice(m[0].length)] : [text, ''];
257
+ };
168
258
  export function formOf(note) {
169
259
  if (note.tags.includes('video'))
170
260
  return 'video';
@@ -172,50 +262,62 @@ export function formOf(note) {
172
262
  return 'thread';
173
263
  return /\.pdf(?:$|[?#])|arxiv\.org\/pdf\//i.test(note.sourceUrl) ? 'pdf' : 'article';
174
264
  }
175
- /** Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), quotes are located and the gold is checked. */
265
+ /**
266
+ * Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), each quote is kept only when it
267
+ * occurs as written (folded typography only) and has at least four words; whole quotes then leave, longest and then latest in
268
+ * the note first, until the rest fit the quote budget (never the last one); survivors are assigned to ideas, and the gold is checked.
269
+ */
176
270
  export function convertJungleDigest(noteMarkdown, capture) {
177
- const note = parseJungleNote(noteMarkdown), lossy = new Set(), source = capture === undefined ? undefined : captureBody(capture);
178
- const [lead, rest] = firstSentence(note.gist);
179
- let context = '';
180
- for (let r = rest; r;) {
181
- const [s, next] = firstSentence(r);
182
- if (words(context ? `${context} ${s}` : s) > L.contextWords) {
183
- lossy.add('context-trimmed');
184
- break;
185
- }
186
- context = context ? `${context} ${s}` : s;
187
- r = next;
271
+ const note = parseJungleNote(noteMarkdown), source = capture === undefined ? undefined : captureBody(capture), [lead, context] = firstSentence(note.gist);
272
+ const located = note.quotes.map((q) => (source === undefined ? undefined : snapQuote(source, q.text).match));
273
+ const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped'
274
+ ? 'not-verbatim'
275
+ : words(q.text) < L.minQuoteWords
276
+ ? 'too-short'
277
+ : undefined);
278
+ let kept = note.quotes.flatMap((q, k) => (dropped[k] ? [] : [{ q, k, n: words(q.text) }]));
279
+ // The pipeline's budget rule, applied to owner quotes: drop whole quotes, never edit one. A single quote over budget still fails the check.
280
+ const budget = source === undefined
281
+ ? Infinity
282
+ : Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)));
283
+ while (kept.length > 1 && (kept.length > L.maxQuotes || kept.reduce((n, x) => n + x.n, 0) > budget)) {
284
+ const x = [...kept].sort((a, b) => b.n - a.n || b.k - a.k)[0];
285
+ dropped[x.k] = 'budget';
286
+ kept = kept.filter((y) => y !== x);
188
287
  }
189
- const assigned = assign(note.ideas, note.quotes), authors = new Set(note.authors.map(a => foldText(a).toLowerCase())), form = formOf(note);
190
- const located = note.quotes.map(q => source === undefined ? undefined : snapQuote(source, q.text));
191
- const ideas = note.ideas.map((idea, i) => {
192
- const mine = note.quotes.map((q, k) => ({ q, k })).filter(x => assigned[x.k] === i);
193
- const other = mine.find(x => !authors.has(foldText(x.q.speaker).toLowerCase()));
194
- // A non-author in a talk is a speaker; in writing, the author reports them.
195
- const voice = other ? { type: form === 'video' ? 'speaker' : 'reported', name: other.q.speaker } : { type: 'author' };
196
- const quotes = mine.map(x => ({ text: located[x.k]?.text ?? x.q.text, speaker: x.q.speaker }));
197
- return { claim: idea.claim, detail: idea.detail, kind: 'argument', voice, quotes };
288
+ const assigned = note.quotes.map(() => null);
289
+ assignQuotes(note.ideas, kept.map((x) => x.q)).forEach((idea, n) => {
290
+ if (idea === null)
291
+ dropped[kept[n].k] = 'overflow';
292
+ else
293
+ assigned[kept[n].k] = idea;
198
294
  });
199
- const content = note.tags.filter(t => t !== 'reading' && !jungleFormatTags.includes(t)), tags = (content.length ? content : note.tags.filter(t => t !== 'reading')).slice(0, L.maxTags);
200
- if (!content.length)
201
- lossy.add('no-content-tag');
202
- if (assigned.some(a => a === null))
203
- lossy.add('quote-unassigned');
204
- if (located.some(l => l?.match === 'missing'))
205
- lossy.add('quote-missing');
206
- if (ideas.some(i => !i.quotes.length))
207
- lossy.add('idea-unquoted');
208
- const gold = { schemaVersion: 1, form, gist: { lead, ...(context ? { context } : {}) }, ideas, tags };
209
- const options = { tags: jungleTags };
210
- const issues = source === undefined ? [] : checkDigest(gold, source, options).issues;
211
- let importable;
212
- if (source !== undefined) {
213
- const kept = note.quotes.map((_, k) => assigned[k] !== null && located[k]?.match !== 'missing');
214
- // Import keeps Sponge's ordering rule: ideas by their earliest quote in the capture, quotes of at least minQuoteWords.
215
- const at = (t) => locateAll(source, t)[0]?.start ?? Number.MAX_SAFE_INTEGER;
216
- const trimmed = { ...gold, ideas: note.ideas.map((_, i) => ({ ...ideas[i], quotes: note.quotes.flatMap((__, k) => assigned[k] === i && kept[k] && words(located[k].text) >= minQuoteWords ? [{ text: located[k].text, speaker: note.quotes[k].speaker }] : []).sort((a, b) => at(a.text) - at(b.text)) })).filter(i => i.quotes.length).sort((a, b) => at(a.quotes[0].text) - at(b.quotes[0].text)) };
217
- if (checkDigest(trimmed, source, options).ok)
218
- importable = trimmed;
219
- }
220
- return { note, gold, quotes: note.quotes.map((_, k) => ({ idea: assigned[k] ?? null, ...(located[k] ? { match: located[k].match } : {}) })), lossy: [...lossy], issues, ...(importable ? { importable } : {}), clean: source !== undefined && issues.length === 0 && lossy.size === 0 };
295
+ const ideas = note.ideas.map((idea, i) => ({
296
+ claim: idea.claim,
297
+ detail: idea.detail,
298
+ quotes: note.quotes.flatMap((q, k) => assigned[k] === i ? [{ text: q.text, ...(q.attribution ? { attribution: q.attribution } : {}) }] : []),
299
+ }));
300
+ const tags = note.tags.filter((t) => t !== 'reading' && !jungleFormatTags.includes(t));
301
+ const gold = {
302
+ schemaVersion: 2,
303
+ form: formOf(note),
304
+ gist: { lead, ...(context ? { context } : {}) },
305
+ ideas,
306
+ tags,
307
+ };
308
+ const issues = [
309
+ ...parseDigest(gold).issues,
310
+ ...(source === undefined ? [] : checkDigest(gold, source, { tags: jungleTags }).issues),
311
+ ];
312
+ return {
313
+ note,
314
+ gold,
315
+ quotes: note.quotes.map((_, k) => ({
316
+ idea: assigned[k] ?? null,
317
+ ...(located[k] ? { match: located[k] } : {}),
318
+ ...(dropped[k] ? { dropped: dropped[k] } : {}),
319
+ })),
320
+ issues,
321
+ ...(source !== undefined && !issues.length ? { digest: gold } : {}),
322
+ };
221
323
  }
@@ -1,33 +1,42 @@
1
1
  import type { DatabaseSync } from 'node:sqlite';
2
- import { type DigestV1 } from '../digest/schema.js';
2
+ import { type DigestV2 } from '../digest/schema.js';
3
3
  import { type VisibilityState } from '../identity/visibility.js';
4
4
  import { type JungleEditorial } from './jungle-digest.js';
5
5
  /**
6
6
  * The library side of `sponge import jungle`: one Jungle Reading note onto the item holding its capture.
7
7
  * `planJungleNote` only reads; `applyJungleNote` runs the same plan inside the caller's transaction, through the logged paths.
8
8
  */
9
- export declare const jungleNamespace = "jungle", jungleRealm = "import:jungle", jungleProducer: {
10
- readonly importer: "sponge-jungle-reading";
11
- readonly version: "1";
12
- };
9
+ export declare const jungleNamespace = "jungle", jungleRealm = "import:jungle";
13
10
  export type EditorialAction = 'new' | 'unchanged' | 'user-review-wins';
14
11
  export type DigestAction = 'new' | 'unchanged' | 'not-convertible' | 'not-digest' | 'user-review-wins' | 'user-digest-wins';
15
12
  export type VisibilityAction = 'set-public' | 'set-private' | 'restore-default' | 'unchanged' | 'kept-user-edit' | 'guarded' | 'default-policy';
13
+ /** Local captures only (no web URI of their own): `uri` records the note's `source_url`. Later notes on one capture leave it to the first (`other-note`). */
14
+ export type UriAction = 'new' | 'unchanged' | 'web-capture' | 'other-note' | 'no-source-url' | 'capability';
15
+ /** The first note's title is the item's default title (`noteTitle`); `kept-user-edit` records it while the user's override stays shown. */
16
+ export type TitleAction = 'set' | 'unchanged' | 'other-note' | 'kept-user-edit';
17
+ /** `firstNote` is false for a later note on a capture an earlier note in the same run already used; `firstNoteUri` is the URI that note records (a dry run has not written it). */
16
18
  export interface JungleNoteInput {
17
19
  itemId: string;
18
20
  revisionId: string;
19
21
  editorial: JungleEditorial;
20
- v1?: DigestV1;
22
+ digest?: DigestV2;
23
+ firstNote?: boolean;
24
+ firstNoteUri?: string;
21
25
  }
22
26
  export interface JunglePlan {
23
27
  editorialId: string;
24
28
  editorial: EditorialAction;
25
- v1Id?: string;
26
- v1: DigestAction;
29
+ digestId?: string;
30
+ digest: DigestAction;
31
+ uri: UriAction;
32
+ title: TitleAction;
27
33
  visibility: VisibilityAction;
28
34
  }
29
- /** What Jungle's `reading.status` asks for: `published` is Jungle's explicit public approval, `draft` stays private. */
30
- export declare function desiredVisibility(status: string): 'public' | 'private' | undefined;
35
+ /** What a local capture's URI action would be, before or after its item exists. */
36
+ export declare function uriAction(local: boolean, firstNote: boolean, sourceUrl: string, recorded?: string): {
37
+ action: UriAction;
38
+ uri?: string;
39
+ };
31
40
  /**
32
41
  * Jungle's decision becomes an explicit override only while nobody else has set one: once the latest override was not
33
42
  * written by this importer, the user's choice wins. A public decision never lifts the capability-URL or local-file guards.
@@ -39,7 +48,7 @@ export declare function visibilityAction(state: Pick<VisibilityState, 'access' |
39
48
  export declare function planJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;
40
49
  /**
41
50
  * Apply one note; the caller holds the write transaction. A new editorial object is accepted through `review.note` unless
42
- * the user already rejected or cleared an earlier one; a clean Digest v1 is stored and accepted per field, linked to the
51
+ * the user already rejected or cleared an earlier one; a converted Digest v2 is stored and accepted per field, linked to the
43
52
  * editorial object by its input hash, unless the user owns a digest of this item or changed an imported digest's review.
44
53
  */
45
54
  export declare function applyJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;