@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -3,84 +3,153 @@ import { join } from 'node:path';
3
3
  import { limits } from '../../contracts/index.js';
4
4
  import { validateAsset } from '../../contracts/validate.js';
5
5
  import { object, metadataFromMarkdown } from '../acquisition/normalize.js';
6
- import { readConfined, hash } from '../acquisition/files.js';
6
+ import { readConfined } from '../acquisition/files.js';
7
+ import { sha256 as hash } from '../util/hash.js';
7
8
  import { withImportVerification } from './custody.js';
8
9
  import { inspectCaptureRuntime } from '../acquisition/runtime.js';
9
10
  import { articleMetadata } from '../metadata/article.js';
10
11
  import { bundleBytes } from '../app/bundle-files.js';
12
+ import { SpongeError } from '../errors.js';
11
13
  export function releasedBundleVerifier(runtime, libraryRoot, signal) {
12
14
  return async (root, kind) => {
13
15
  const checked = await inspectCaptureRuntime(runtime.root, runtime.pins), helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
14
16
  return withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
15
17
  const args = [helper, checked.root, root, kind];
16
- const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin' ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args] : args, { env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' }, deadlineMs: 45000, maxOutputBytes: 65536, signal });
18
+ const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin'
19
+ ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args]
20
+ : args, {
21
+ env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' },
22
+ deadlineMs: 45000,
23
+ maxOutputBytes: 65536,
24
+ signal,
25
+ });
17
26
  if (result.cleanup !== 'settled')
18
- throw Error('cleanup-required');
27
+ throw new SpongeError('cleanup-required');
19
28
  if (result.stopped)
20
- throw Error(result.stopped);
29
+ throw new SpongeError(result.stopped);
21
30
  if (result.code !== 0)
22
- throw Error('unsupported');
31
+ throw new SpongeError('unsupported');
23
32
  return JSON.parse(result.stdout);
24
33
  });
25
34
  };
26
35
  }
27
- function imageType(bytes) { if (bytes[0] === 255 && bytes[1] === 216)
28
- return 'image/jpeg'; if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
29
- return 'image/png'; if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
30
- return 'image/webp'; if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
31
- return 'image/gif'; if (bytes.toString('ascii', 4, 12) === 'ftypavif')
32
- return 'image/avif'; throw Error('unsupported'); }
36
+ function imageType(bytes) {
37
+ if (bytes[0] === 255 && bytes[1] === 216)
38
+ return 'image/jpeg';
39
+ if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
40
+ return 'image/png';
41
+ if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
42
+ return 'image/webp';
43
+ if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
44
+ return 'image/gif';
45
+ if (bytes.toString('ascii', 4, 12) === 'ftypavif')
46
+ return 'image/avif';
47
+ throw new SpongeError('unsupported');
48
+ }
33
49
  /** An import-time digest never upgrades a missing historical document digest. */
34
50
  export async function inspectImportBundle(root, verify) {
51
+ // A bundle and its verifier report are untrusted: malformed JSON, a listed file that is missing, a bad URL or asset record is an integrity failure.
52
+ try {
53
+ return await inspectBundle(root, verify);
54
+ }
55
+ catch (error) {
56
+ if (error instanceof SpongeError)
57
+ throw error;
58
+ throw new SpongeError('integrity', { cause: error });
59
+ }
60
+ }
61
+ async function inspectBundle(root, verify) {
35
62
  const manifestBytes = await readConfined(root, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8'))), kind = manifest.kind === 'pdf' ? 'pdf' : 'article';
36
63
  if (kind === 'pdf' ? manifest.schemaVersion !== 1 : ![1, 2, 3, 4].includes(Number(manifest.schemaVersion)))
37
- throw Error('unsupported');
64
+ throw new SpongeError('unsupported');
38
65
  const result = object(await verify(root, kind)), inspection = object(result.inspection), document = object(kind === 'pdf' ? inspection.markdown : inspection.document);
39
- if (inspection.schemaVersion !== manifest.schemaVersion || !['verified', 'unavailable'].includes(String(document.integrity)) || !Array.isArray(inspection.assets) || inspection.assets.length > limits.imagesPerItem)
40
- throw Error('integrity');
41
- if (result.ok !== true && (!Array.isArray(result.issues) || !result.issues.length || result.issues.some(raw => { const issue = object(raw); return !(issue.kind === 'document-integrity' && issue.path === document.path && document.integrity === 'unavailable' && (kind === 'article' || issue.integrity === 'unavailable')); })))
42
- throw Error('integrity');
43
- if (inspection.status !== undefined && inspection.status !== null && !['complete', 'partial'].includes(String(inspection.status)))
44
- throw Error('unsupported');
45
- const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = { relativePath: String(document.path), bytes: markdownBytes.length, sha256: hash(markdownBytes), mediaType: 'text/markdown' };
66
+ if (inspection.schemaVersion !== manifest.schemaVersion ||
67
+ !['verified', 'unavailable'].includes(String(document.integrity)) ||
68
+ !Array.isArray(inspection.assets) ||
69
+ inspection.assets.length > limits.imagesPerItem)
70
+ throw new SpongeError('integrity');
71
+ if (result.ok !== true &&
72
+ (!Array.isArray(result.issues) ||
73
+ !result.issues.length ||
74
+ result.issues.some((raw) => {
75
+ const issue = object(raw);
76
+ return !(issue.kind === 'document-integrity' &&
77
+ issue.path === document.path &&
78
+ document.integrity === 'unavailable' &&
79
+ (kind === 'article' || issue.integrity === 'unavailable'));
80
+ })))
81
+ throw new SpongeError('integrity');
82
+ if (inspection.status !== undefined &&
83
+ inspection.status !== null &&
84
+ !['complete', 'partial'].includes(String(inspection.status)))
85
+ throw new SpongeError('unsupported');
86
+ const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = {
87
+ relativePath: String(document.path),
88
+ bytes: markdownBytes.length,
89
+ sha256: hash(markdownBytes),
90
+ mediaType: 'text/markdown',
91
+ };
46
92
  validateAsset(markdown);
47
93
  if (markdown.sha256 !== document.sha256 || markdown.bytes !== document.bytes)
48
- throw Error('integrity');
94
+ throw new SpongeError('integrity');
49
95
  let unavailable = document.integrity === 'unavailable';
50
96
  const assets = [];
51
97
  let total = 0;
52
98
  for (const raw of inspection.assets) {
53
99
  const asset = object(raw);
54
100
  if (!['verified', 'unavailable'].includes(String(asset.integrity)))
55
- throw Error('integrity');
101
+ throw new SpongeError('integrity');
56
102
  unavailable ||= asset.integrity === 'unavailable';
57
103
  const bytes = await readConfined(root, String(asset.path), limits.imageBytes);
58
104
  total += bytes.length;
59
105
  if (total > 500 * 1024 ** 2)
60
- throw Error('limit-exceeded');
61
- const value = { relativePath: String(asset.path), bytes: bytes.length, sha256: hash(bytes), mediaType: imageType(bytes), relationship: 'image' };
106
+ throw new SpongeError('limit-exceeded');
107
+ const value = {
108
+ relativePath: String(asset.path),
109
+ bytes: bytes.length,
110
+ sha256: hash(bytes),
111
+ mediaType: imageType(bytes),
112
+ relationship: 'image',
113
+ };
62
114
  validateAsset(value);
63
115
  const expectedHash = kind === 'pdf' ? asset.sha256 : asset.expectedSha256, expectedBytes = kind === 'pdf' ? asset.bytes : asset.expectedBytes;
64
116
  if (asset.integrity === 'verified' && (expectedHash !== value.sha256 || expectedBytes !== value.bytes))
65
- throw Error('integrity');
117
+ throw new SpongeError('integrity');
66
118
  assets.push(value);
67
119
  }
68
120
  if (kind === 'pdf') {
69
121
  const source = object(inspection.source);
70
122
  if (source.integrity !== 'verified')
71
- throw Error('integrity');
72
- const asset = { relativePath: String(source.path), sha256: String(source.sha256), bytes: Number(source.bytes), mediaType: 'application/pdf', relationship: 'source-pdf' };
123
+ throw new SpongeError('integrity');
124
+ const asset = {
125
+ relativePath: String(source.path),
126
+ sha256: String(source.sha256),
127
+ bytes: Number(source.bytes),
128
+ mediaType: 'application/pdf',
129
+ relationship: 'source-pdf',
130
+ };
73
131
  validateAsset(asset);
74
- for await (const _ of bundleBytes(root, asset)) { }
132
+ for await (const _ of bundleBytes(root, asset)) {
133
+ }
75
134
  assets.push(asset);
76
135
  }
77
- if (assets.length !== new Set(assets.map(a => a.relativePath)).size)
78
- throw Error('integrity');
79
- const sourceUrl = kind === 'pdf' ? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href) : inspection.sourceUrl;
80
- if (typeof sourceUrl !== 'string' || sourceUrl.length > 16384 || !['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
81
- throw Error('integrity');
82
- const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt)) ? manifest.capturedAt : '1970-01-01T00:00:00.000Z';
83
- const metadata = kind === 'article' ? (manifest.metadata === undefined ? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt) : articleMetadata(manifest.metadata, observedAt)) : [];
136
+ if (assets.length !== new Set(assets.map((a) => a.relativePath)).size)
137
+ throw new SpongeError('integrity');
138
+ const sourceUrl = kind === 'pdf'
139
+ ? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href)
140
+ : inspection.sourceUrl;
141
+ if (typeof sourceUrl !== 'string' ||
142
+ sourceUrl.length > 16384 ||
143
+ !['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
144
+ throw new SpongeError('integrity');
145
+ const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt))
146
+ ? manifest.capturedAt
147
+ : '1970-01-01T00:00:00.000Z';
148
+ const metadata = kind === 'article'
149
+ ? manifest.metadata === undefined
150
+ ? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt)
151
+ : articleMetadata(manifest.metadata, observedAt)
152
+ : [];
84
153
  if (kind === 'pdf') {
85
154
  const document = object(manifest.document ?? {});
86
155
  for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
@@ -89,9 +158,27 @@ export async function inspectImportBundle(root, verify) {
89
158
  metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
90
159
  }
91
160
  }
92
- metadata.push({ field: 'historicalIntegrity', value: unavailable ? 'unavailable' : 'verified', source: kind === 'pdf' ? 'pdf' : 'html', observedAt, rule: 'sponge-import-v1' });
161
+ metadata.push({
162
+ field: 'historicalIntegrity',
163
+ value: unavailable ? 'unavailable' : 'verified',
164
+ source: kind === 'pdf' ? 'pdf' : 'html',
165
+ observedAt,
166
+ rule: 'sponge-import-v1',
167
+ });
93
168
  if (hash(await readConfined(root, 'capture.json', limits.metadataBytes)) !== hash(manifestBytes))
94
- throw Error('integrity');
95
- const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map(a => [a.relativePath, a.sha256, a.bytes])]));
96
- return { root, kind, sourceUrl, schemaVersion: Number(manifest.schemaVersion), status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete', markdown, assets, metadata, manifestSha256, historicalIntegrity: unavailable ? 'unavailable' : 'verified', importHash };
169
+ throw new SpongeError('integrity');
170
+ const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map((a) => [a.relativePath, a.sha256, a.bytes])]));
171
+ return {
172
+ root,
173
+ kind,
174
+ sourceUrl,
175
+ schemaVersion: Number(manifest.schemaVersion),
176
+ status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete',
177
+ markdown,
178
+ assets,
179
+ metadata,
180
+ manifestSha256,
181
+ historicalIntegrity: unavailable ? 'unavailable' : 'verified',
182
+ importHash,
183
+ };
97
184
  }
@@ -9,11 +9,7 @@ import { type Form, type QuoteMatch } from '../digest/pipeline.js';
9
9
  export declare const jungleTags: readonly ["agent-eng", "ai", "work", "business", "creative-coding", "finance", "music", "internet", "security", "history", "art", "hardware", "climate", "article", "interactive", "code", "x", "video"];
10
10
  /** Tags that name how a source is consumed; a digest carries that as `form`. */
11
11
  export declare const jungleFormatTags: readonly ["article", "interactive", "x", "video"];
12
- export declare class JungleDigestError extends Error {
13
- readonly code: 'not-digest' | 'malformed' | 'no-capture';
14
- constructor(code: 'not-digest' | 'malformed' | 'no-capture');
15
- }
16
- export interface JungleQuote {
12
+ interface JungleQuote {
17
13
  text: string;
18
14
  attribution: string;
19
15
  speaker: string;
@@ -53,7 +49,7 @@ export interface JungleConversion {
53
49
  /** The gold, when a capture was given and the gold passes every v2 rule against it. */
54
50
  digest?: DigestV2;
55
51
  }
56
- export type Yaml = string | boolean | Yaml[] | {
52
+ type Yaml = string | boolean | Yaml[] | {
57
53
  [k: string]: Yaml;
58
54
  };
59
55
  /** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
@@ -105,3 +101,4 @@ export declare function formOf(note: JungleNote): Form;
105
101
  * the note first, until the rest fit the quote budget (never the last one); survivors are assigned to ideas, and the gold is checked.
106
102
  */
107
103
  export declare function convertJungleDigest(noteMarkdown: string, capture?: string): JungleConversion;
104
+ export {};
@@ -1,38 +1,57 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { foldText, words } from '../annotations/quote.js';
3
- import { checkDigest, digestLimits as L, parseDigest } from '../digest/schema.js';
3
+ import { checkDigest, digestLimits as L, parseDigest, } from '../digest/schema.js';
4
4
  import { snapQuote } from '../digest/pipeline.js';
5
+ import { SpongeError } from '../errors.js';
5
6
  /**
6
7
  * Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v2. Jungle keeps
7
8
  * quotes in their own section, so each verbatim quote is assigned to the idea it shares the most content words with.
8
9
  * Owner text is never cut or rewritten: a note that does not fit v2 as written is reported, not converted.
9
10
  * Nothing here reads files; callers pass the note and, when available, its capture Markdown.
10
11
  */
11
- export const jungleTags = ['agent-eng', 'ai', 'work', 'business', 'creative-coding', 'finance', 'music', 'internet', 'security', 'history', 'art', 'hardware', 'climate', 'article', 'interactive', 'code', 'x', 'video'];
12
+ export const jungleTags = [
13
+ 'agent-eng',
14
+ 'ai',
15
+ 'work',
16
+ 'business',
17
+ 'creative-coding',
18
+ 'finance',
19
+ 'music',
20
+ 'internet',
21
+ 'security',
22
+ 'history',
23
+ 'art',
24
+ 'hardware',
25
+ 'climate',
26
+ 'article',
27
+ 'interactive',
28
+ 'code',
29
+ 'x',
30
+ 'video',
31
+ ];
12
32
  /** Tags that name how a source is consumed; a digest carries that as `form`. */
13
33
  export const jungleFormatTags = ['article', 'interactive', 'x', 'video'];
14
- export class JungleDigestError extends Error {
15
- code;
16
- constructor(code) {
17
- super(code);
18
- this.code = code;
19
- this.name = 'JungleDigestError';
20
- }
21
- }
22
- const fail = (code = 'malformed') => { throw new JungleDigestError(code); };
34
+ const fail = (code = 'malformed') => {
35
+ throw new SpongeError(code);
36
+ };
23
37
  /** The frontmatter subset Jungle writes: block maps and lists of scalars, quoted or plain. */
24
38
  function parseYaml(lines) {
25
39
  let i = 0;
26
40
  const indent = (l) => l.length - l.trimStart().length;
27
- const scalar = (v) => { v = v.trim(); if (v.startsWith('"')) {
28
- try {
29
- return JSON.parse(v);
30
- }
31
- catch {
32
- return fail();
41
+ const scalar = (v) => {
42
+ v = v.trim();
43
+ if (v.startsWith('"')) {
44
+ try {
45
+ return JSON.parse(v);
46
+ }
47
+ catch {
48
+ return fail();
49
+ }
33
50
  }
34
- } if (v.startsWith("'"))
35
- return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail(); return v === 'true' ? true : v === 'false' ? false : v; };
51
+ if (v.startsWith("'"))
52
+ return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail();
53
+ return v === 'true' ? true : v === 'false' ? false : v;
54
+ };
36
55
  const block = (at) => {
37
56
  if (lines[i].trimStart().startsWith('- ')) {
38
57
  const list = [];
@@ -56,12 +75,18 @@ function parseYaml(lines) {
56
75
  while (i < lines.length && indent(lines[i]) === at && !lines[i].trimStart().startsWith('- ')) {
57
76
  const m = /^([\w-]+):(?:\s+(.*))?$/.exec(lines[i].trim()) ?? fail();
58
77
  i++;
59
- if (m[2] !== undefined && m[2] !== '')
60
- out[m[1]] = scalar(m[2]);
61
- else if (i < lines.length && (indent(lines[i]) > at || indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
62
- out[m[1]] = block(indent(lines[i]));
63
- else
64
- out[m[1]] = '';
78
+ // An own data property even for `__proto__`, which plain assignment would turn into the map's prototype.
79
+ Object.defineProperty(out, m[1], {
80
+ value: m[2] !== undefined && m[2] !== ''
81
+ ? scalar(m[2])
82
+ : i < lines.length &&
83
+ (indent(lines[i]) > at || (indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
84
+ ? block(indent(lines[i]))
85
+ : '',
86
+ enumerable: true,
87
+ writable: true,
88
+ configurable: true,
89
+ });
65
90
  }
66
91
  return out;
67
92
  };
@@ -69,14 +94,21 @@ function parseYaml(lines) {
69
94
  return i === lines.length ? value : fail();
70
95
  }
71
96
  const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
72
- const strings = (v) => Array.isArray(v) && v.every(x => typeof x === 'string') ? v : fail();
97
+ const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === 'string') ? v : fail());
73
98
  /** Split a Markdown file into its frontmatter lines and body. */
74
99
  function frontmatter(markdown) {
75
100
  const m = /^---\n([^]*?)\n---(?:\n|$)/.exec(markdown.replace(/\r\n/g, '\n'));
76
- return m ? { head: m[1].split('\n').filter(l => l.trim() && !l.trimStart().startsWith('#')), body: markdown.replace(/\r\n/g, '\n').slice(m[0].length) } : { head: [], body: markdown };
101
+ return m
102
+ ? {
103
+ head: m[1].split('\n').filter((l) => l.trim() && !l.trimStart().startsWith('#')),
104
+ body: markdown.replace(/\r\n/g, '\n').slice(m[0].length),
105
+ }
106
+ : { head: [], body: markdown };
77
107
  }
78
108
  /** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
79
- export function captureBody(capture) { return frontmatter(capture).body; }
109
+ export function captureBody(capture) {
110
+ return frontmatter(capture).body;
111
+ }
80
112
  /** `## Heading` sections of a note body, trimmed. */
81
113
  function sectionsOf(body) {
82
114
  const sections = new Map();
@@ -87,10 +119,19 @@ function sectionsOf(body) {
87
119
  return sections;
88
120
  }
89
121
  /** `- **Claim.** detail` bullets: Jungle's Ideas, and a conversation's Debate. */
90
- const ideaList = (text) => text.split('\n').filter(l => l.startsWith('- ')).map(l => { const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail(); return { claim: m[1].trim(), detail: m[2].trim() }; });
122
+ const ideaList = (text) => text
123
+ .split('\n')
124
+ .filter((l) => l.startsWith('- '))
125
+ .map((l) => {
126
+ const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail();
127
+ return { claim: m[1].trim(), detail: m[2].trim() };
128
+ });
91
129
  /** Blockquotes ending `> Attribution: Speaker, context`. */
92
- const quoteList = (text) => text.split(/\n\s*\n/).filter(b => b.trim()).map(b => {
93
- const lines = b.split('\n').map(l => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex(l => l.startsWith('Attribution:'));
130
+ const quoteList = (text) => text
131
+ .split(/\n\s*\n/)
132
+ .filter((b) => b.trim())
133
+ .map((b) => {
134
+ const lines = b.split('\n').map((l) => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex((l) => l.startsWith('Attribution:'));
94
135
  if (at < 1)
95
136
  fail();
96
137
  const text = lines.slice(0, at).join(' ').replace(/\s+/g, ' ').trim(), attribution = lines[at].slice('Attribution:'.length).trim();
@@ -101,7 +142,7 @@ export function parseJungleNote(markdown) {
101
142
  fail();
102
143
  const { head, body } = frontmatter(markdown), meta = parseYaml(head);
103
144
  if (!object(meta) || !object(meta.reading))
104
- throw new JungleDigestError('not-digest');
145
+ throw new SpongeError('not-digest');
105
146
  const reading = meta.reading;
106
147
  if (reading.form !== 'digest')
107
148
  fail('not-digest');
@@ -109,56 +150,111 @@ export function parseJungleNote(markdown) {
109
150
  const gist = (sections.get('Gist') ?? fail()).replace(/\s+/g, ' ').trim();
110
151
  const ideas = ideaList(sections.get('Ideas') ?? fail());
111
152
  const quotes = quoteList(sections.get('Quotes') ?? '');
112
- if (!gist || !ideas.length || quotes.some(q => !q.text || !q.speaker))
153
+ if (!gist || !ideas.length || quotes.some((q) => !q.text || !q.speaker))
113
154
  fail();
114
- const authors = Array.isArray(reading.authors) ? reading.authors.map(a => object(a) && typeof a.name === 'string' ? a.name : fail()) : [];
155
+ const authors = Array.isArray(reading.authors)
156
+ ? reading.authors.map((a) => (object(a) && typeof a.name === 'string' ? a.name : fail()))
157
+ : [];
115
158
  const relations = object(meta.relations) ? meta.relations : {}, synth = Array.isArray(relations.synthesizes) ? strings(relations.synthesizes) : [];
116
- return { slug: typeof reading.slug === 'string' ? reading.slug : fail(), title: typeof meta.title === 'string' ? meta.title : '', status: typeof reading.status === 'string' ? reading.status : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', authors, tags: meta.tags === undefined ? [] : strings(meta.tags), ...(synth[0] ? { capture: synth[0] } : {}), gist, ideas, quotes };
159
+ return {
160
+ slug: typeof reading.slug === 'string' ? reading.slug : fail(),
161
+ title: typeof meta.title === 'string' ? meta.title : '',
162
+ status: typeof reading.status === 'string' ? reading.status : '',
163
+ sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
164
+ authors,
165
+ tags: meta.tags === undefined ? [] : strings(meta.tags),
166
+ ...(synth[0] ? { capture: synth[0] } : {}),
167
+ gist,
168
+ ideas,
169
+ quotes,
170
+ };
117
171
  }
118
172
  export function parseJungleEditorial(markdown) {
119
173
  if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
120
174
  fail();
121
175
  const { head, body } = frontmatter(markdown), meta = parseYaml(head);
122
176
  if (!object(meta) || !object(meta.reading))
123
- throw new JungleDigestError('not-digest');
177
+ throw new SpongeError('not-digest');
124
178
  const reading = meta.reading, relations = object(meta.relations) ? meta.relations : {};
125
179
  const form = reading.form, slug = reading.slug;
126
180
  if (typeof form !== 'string' || !form || typeof slug !== 'string' || !/^[a-z0-9][a-z0-9-]{0,199}$/.test(slug))
127
181
  return fail();
128
- const capture = [relations.synthesizes, relations['evidenced-by']].flatMap(v => Array.isArray(v) ? strings(v) : [])[0] ?? fail('no-capture');
182
+ const capture = [relations.synthesizes, relations['evidenced-by']].flatMap((v) => (Array.isArray(v) ? strings(v) : []))[0] ??
183
+ fail('no-capture');
129
184
  // A digest must parse strictly; the other forms keep whatever structure they have, and the Markdown keeps the rest.
130
- const sections = sectionsOf(body), loose = (f) => { if (form === 'digest')
131
- return f(); try {
132
- return f();
133
- }
134
- catch {
135
- return undefined;
136
- } };
137
- const gistText = sections.get('Gist') ?? sections.get('Summary') ?? (typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
185
+ const sections = sectionsOf(body), loose = (f) => {
186
+ if (form === 'digest')
187
+ return f();
188
+ try {
189
+ return f();
190
+ }
191
+ catch {
192
+ return undefined;
193
+ }
194
+ };
195
+ const gistText = sections.get('Gist') ??
196
+ sections.get('Summary') ??
197
+ (typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
138
198
  if (form === 'digest')
139
199
  parseJungleNote(markdown);
140
200
  const gist = gistText?.replace(/\s+/g, ' ').trim(), ideas = ideaText === undefined ? undefined : loose(() => ideaList(ideaText)), quotes = quoteText === undefined ? undefined : loose(() => quoteList(quoteText));
141
- return { schemaVersion: 1, source: 'jungle-reading', form, slug, status: typeof reading.status === 'string' ? reading.status : '', title: typeof meta.title === 'string' ? meta.title : '', sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '', capture, frontmatter: meta, ...(gist ? { gist } : {}), ...(ideas?.length ? { ideas } : {}), ...(quotes?.length && quotes.every(q => q.text && q.speaker) ? { quotes } : {}), markdownSha256: createHash('sha256').update(markdown).digest('hex'), markdown };
201
+ return {
202
+ schemaVersion: 1,
203
+ source: 'jungle-reading',
204
+ form,
205
+ slug,
206
+ status: typeof reading.status === 'string' ? reading.status : '',
207
+ title: typeof meta.title === 'string' ? meta.title : '',
208
+ sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
209
+ capture,
210
+ frontmatter: meta,
211
+ ...(gist ? { gist } : {}),
212
+ ...(ideas?.length ? { ideas } : {}),
213
+ ...(quotes?.length && quotes.every((q) => q.text && q.speaker) ? { quotes } : {}),
214
+ markdownSha256: createHash('sha256').update(markdown).digest('hex'),
215
+ markdown,
216
+ };
142
217
  }
143
218
  const stop = new Set('the a an and or but of to in on for with by from at as is are was were be been it its this that these those not no than then so such into about over under more most less can could would should will just also only very their there they them we our you your he she his her i my me what which who how when where why do does did has have had'.split(' '));
144
219
  /** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
145
220
  export function contentTerms(text) {
146
- return new Set(foldText(text).toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length > 2 && !stop.has(w)).map(w => w.replace(/(?:ing|ed|es|s)$/, '')).filter(w => w.length > 2));
221
+ return new Set(foldText(text)
222
+ .toLowerCase()
223
+ .split(/[^a-z0-9]+/)
224
+ .filter((w) => w.length > 2 && !stop.has(w))
225
+ .map((w) => w.replace(/(?:ing|ed|es|s)$/, ''))
226
+ .filter((w) => w.length > 2));
227
+ }
228
+ export function termSimilarity(a, b) {
229
+ if (!a.size || !b.size)
230
+ return 0;
231
+ let n = 0;
232
+ for (const t of a)
233
+ if (b.has(t))
234
+ n++;
235
+ return n / Math.sqrt(a.size * b.size);
147
236
  }
148
- export function termSimilarity(a, b) { if (!a.size || !b.size)
149
- return 0; let n = 0; for (const t of a)
150
- if (b.has(t))
151
- n++; return n / Math.sqrt(a.size * b.size); }
152
237
  /**
153
238
  * Each quote in order goes to the idea sharing the most content words with its claim and detail (ties: the earliest idea),
154
239
  * else to the next best idea with room; null when every idea already holds `cap` quotes.
155
240
  */
156
241
  export function assignQuotes(ideas, quotes, cap = L.maxQuotesPerIdea) {
157
- const terms = ideas.map(i => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
158
- return quotes.map(q => { const t = contentTerms(q.text), pick = terms.map((it, i) => ({ i, s: termSimilarity(t, it) })).sort((a, b) => b.s - a.s || a.i - b.i).find(r => load[r.i] < cap); if (!pick)
159
- return null; load[pick.i]++; return pick.i; });
242
+ const terms = ideas.map((i) => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
243
+ return quotes.map((q) => {
244
+ const t = contentTerms(q.text), pick = terms
245
+ .map((it, i) => ({ i, s: termSimilarity(t, it) }))
246
+ .sort((a, b) => b.s - a.s || a.i - b.i)
247
+ .find((r) => load[r.i] < cap);
248
+ if (!pick)
249
+ return null;
250
+ load[pick.i]++;
251
+ return pick.i;
252
+ });
160
253
  }
161
- export const firstSentence = (text) => { const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text); return m ? [m[1], text.slice(m[0].length)] : [text, '']; };
254
+ export const firstSentence = (text) => {
255
+ const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text);
256
+ return m ? [m[1], text.slice(m[0].length)] : [text, ''];
257
+ };
162
258
  export function formOf(note) {
163
259
  if (note.tags.includes('video'))
164
260
  return 'video';
@@ -173,24 +269,55 @@ export function formOf(note) {
173
269
  */
174
270
  export function convertJungleDigest(noteMarkdown, capture) {
175
271
  const note = parseJungleNote(noteMarkdown), source = capture === undefined ? undefined : captureBody(capture), [lead, context] = firstSentence(note.gist);
176
- const located = note.quotes.map(q => source === undefined ? undefined : snapQuote(source, q.text).match);
177
- const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped' ? 'not-verbatim' : words(q.text) < L.minQuoteWords ? 'too-short' : undefined);
178
- let kept = note.quotes.flatMap((q, k) => dropped[k] ? [] : [{ q, k, n: words(q.text) }]);
272
+ const located = note.quotes.map((q) => (source === undefined ? undefined : snapQuote(source, q.text).match));
273
+ const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped'
274
+ ? 'not-verbatim'
275
+ : words(q.text) < L.minQuoteWords
276
+ ? 'too-short'
277
+ : undefined);
278
+ let kept = note.quotes.flatMap((q, k) => (dropped[k] ? [] : [{ q, k, n: words(q.text) }]));
179
279
  // The pipeline's budget rule, applied to owner quotes: drop whole quotes, never edit one. A single quote over budget still fails the check.
180
- const budget = source === undefined ? Infinity : Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)));
280
+ const budget = source === undefined
281
+ ? Infinity
282
+ : Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)));
181
283
  while (kept.length > 1 && (kept.length > L.maxQuotes || kept.reduce((n, x) => n + x.n, 0) > budget)) {
182
284
  const x = [...kept].sort((a, b) => b.n - a.n || b.k - a.k)[0];
183
285
  dropped[x.k] = 'budget';
184
- kept = kept.filter(y => y !== x);
286
+ kept = kept.filter((y) => y !== x);
185
287
  }
186
288
  const assigned = note.quotes.map(() => null);
187
- assignQuotes(note.ideas, kept.map(x => x.q)).forEach((idea, n) => { if (idea === null)
188
- dropped[kept[n].k] = 'overflow';
189
- else
190
- assigned[kept[n].k] = idea; });
191
- const ideas = note.ideas.map((idea, i) => ({ claim: idea.claim, detail: idea.detail, quotes: note.quotes.flatMap((q, k) => assigned[k] === i ? [{ text: q.text, ...(q.attribution ? { attribution: q.attribution } : {}) }] : []) }));
192
- const tags = note.tags.filter(t => t !== 'reading' && !jungleFormatTags.includes(t));
193
- const gold = { schemaVersion: 2, form: formOf(note), gist: { lead, ...(context ? { context } : {}) }, ideas, tags };
194
- const issues = [...parseDigest(gold).issues, ...source === undefined ? [] : checkDigest(gold, source, { tags: jungleTags }).issues];
195
- return { note, gold, quotes: note.quotes.map((_, k) => ({ idea: assigned[k] ?? null, ...(located[k] ? { match: located[k] } : {}), ...(dropped[k] ? { dropped: dropped[k] } : {}) })), issues, ...(source !== undefined && !issues.length ? { digest: gold } : {}) };
289
+ assignQuotes(note.ideas, kept.map((x) => x.q)).forEach((idea, n) => {
290
+ if (idea === null)
291
+ dropped[kept[n].k] = 'overflow';
292
+ else
293
+ assigned[kept[n].k] = idea;
294
+ });
295
+ const ideas = note.ideas.map((idea, i) => ({
296
+ claim: idea.claim,
297
+ detail: idea.detail,
298
+ quotes: note.quotes.flatMap((q, k) => assigned[k] === i ? [{ text: q.text, ...(q.attribution ? { attribution: q.attribution } : {}) }] : []),
299
+ }));
300
+ const tags = note.tags.filter((t) => t !== 'reading' && !jungleFormatTags.includes(t));
301
+ const gold = {
302
+ schemaVersion: 2,
303
+ form: formOf(note),
304
+ gist: { lead, ...(context ? { context } : {}) },
305
+ ideas,
306
+ tags,
307
+ };
308
+ const issues = [
309
+ ...parseDigest(gold).issues,
310
+ ...(source === undefined ? [] : checkDigest(gold, source, { tags: jungleTags }).issues),
311
+ ];
312
+ return {
313
+ note,
314
+ gold,
315
+ quotes: note.quotes.map((_, k) => ({
316
+ idea: assigned[k] ?? null,
317
+ ...(located[k] ? { match: located[k] } : {}),
318
+ ...(dropped[k] ? { dropped: dropped[k] } : {}),
319
+ })),
320
+ issues,
321
+ ...(source !== undefined && !issues.length ? { digest: gold } : {}),
322
+ };
196
323
  }