@hraness/sponge 0.2.6 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +369 -102
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +19 -6
- package/dist/src/importers/jungle-store.js +124 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +142 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -3,84 +3,153 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { limits } from '../../contracts/index.js';
|
|
4
4
|
import { validateAsset } from '../../contracts/validate.js';
|
|
5
5
|
import { object, metadataFromMarkdown } from '../acquisition/normalize.js';
|
|
6
|
-
import { readConfined
|
|
6
|
+
import { readConfined } from '../acquisition/files.js';
|
|
7
|
+
import { sha256 as hash } from '../util/hash.js';
|
|
7
8
|
import { withImportVerification } from './custody.js';
|
|
8
9
|
import { inspectCaptureRuntime } from '../acquisition/runtime.js';
|
|
9
10
|
import { articleMetadata } from '../metadata/article.js';
|
|
10
11
|
import { bundleBytes } from '../app/bundle-files.js';
|
|
12
|
+
import { SpongeError } from '../errors.js';
|
|
11
13
|
export function releasedBundleVerifier(runtime, libraryRoot, signal) {
|
|
12
14
|
return async (root, kind) => {
|
|
13
15
|
const checked = await inspectCaptureRuntime(runtime.root, runtime.pins), helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
|
|
14
16
|
return withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
|
|
15
17
|
const args = [helper, checked.root, root, kind];
|
|
16
|
-
const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin'
|
|
18
|
+
const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin'
|
|
19
|
+
? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args]
|
|
20
|
+
: args, {
|
|
21
|
+
env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' },
|
|
22
|
+
deadlineMs: 45000,
|
|
23
|
+
maxOutputBytes: 65536,
|
|
24
|
+
signal,
|
|
25
|
+
});
|
|
17
26
|
if (result.cleanup !== 'settled')
|
|
18
|
-
throw
|
|
27
|
+
throw new SpongeError('cleanup-required');
|
|
19
28
|
if (result.stopped)
|
|
20
|
-
throw
|
|
29
|
+
throw new SpongeError(result.stopped);
|
|
21
30
|
if (result.code !== 0)
|
|
22
|
-
throw
|
|
31
|
+
throw new SpongeError('unsupported');
|
|
23
32
|
return JSON.parse(result.stdout);
|
|
24
33
|
});
|
|
25
34
|
};
|
|
26
35
|
}
|
|
27
|
-
function imageType(bytes) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
36
|
+
function imageType(bytes) {
|
|
37
|
+
if (bytes[0] === 255 && bytes[1] === 216)
|
|
38
|
+
return 'image/jpeg';
|
|
39
|
+
if (bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])))
|
|
40
|
+
return 'image/png';
|
|
41
|
+
if (bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP')
|
|
42
|
+
return 'image/webp';
|
|
43
|
+
if (/^GIF8[79]a/.test(bytes.toString('ascii', 0, 6)))
|
|
44
|
+
return 'image/gif';
|
|
45
|
+
if (bytes.toString('ascii', 4, 12) === 'ftypavif')
|
|
46
|
+
return 'image/avif';
|
|
47
|
+
throw new SpongeError('unsupported');
|
|
48
|
+
}
|
|
33
49
|
/** An import-time digest never upgrades a missing historical document digest. */
|
|
34
50
|
export async function inspectImportBundle(root, verify) {
|
|
51
|
+
// A bundle and its verifier report are untrusted: malformed JSON, a listed file that is missing, a bad URL or asset record is an integrity failure.
|
|
52
|
+
try {
|
|
53
|
+
return await inspectBundle(root, verify);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
if (error instanceof SpongeError)
|
|
57
|
+
throw error;
|
|
58
|
+
throw new SpongeError('integrity', { cause: error });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async function inspectBundle(root, verify) {
|
|
35
62
|
const manifestBytes = await readConfined(root, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8'))), kind = manifest.kind === 'pdf' ? 'pdf' : 'article';
|
|
36
63
|
if (kind === 'pdf' ? manifest.schemaVersion !== 1 : ![1, 2, 3, 4].includes(Number(manifest.schemaVersion)))
|
|
37
|
-
throw
|
|
64
|
+
throw new SpongeError('unsupported');
|
|
38
65
|
const result = object(await verify(root, kind)), inspection = object(result.inspection), document = object(kind === 'pdf' ? inspection.markdown : inspection.document);
|
|
39
|
-
if (inspection.schemaVersion !== manifest.schemaVersion ||
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
66
|
+
if (inspection.schemaVersion !== manifest.schemaVersion ||
|
|
67
|
+
!['verified', 'unavailable'].includes(String(document.integrity)) ||
|
|
68
|
+
!Array.isArray(inspection.assets) ||
|
|
69
|
+
inspection.assets.length > limits.imagesPerItem)
|
|
70
|
+
throw new SpongeError('integrity');
|
|
71
|
+
if (result.ok !== true &&
|
|
72
|
+
(!Array.isArray(result.issues) ||
|
|
73
|
+
!result.issues.length ||
|
|
74
|
+
result.issues.some((raw) => {
|
|
75
|
+
const issue = object(raw);
|
|
76
|
+
return !(issue.kind === 'document-integrity' &&
|
|
77
|
+
issue.path === document.path &&
|
|
78
|
+
document.integrity === 'unavailable' &&
|
|
79
|
+
(kind === 'article' || issue.integrity === 'unavailable'));
|
|
80
|
+
})))
|
|
81
|
+
throw new SpongeError('integrity');
|
|
82
|
+
if (inspection.status !== undefined &&
|
|
83
|
+
inspection.status !== null &&
|
|
84
|
+
!['complete', 'partial'].includes(String(inspection.status)))
|
|
85
|
+
throw new SpongeError('unsupported');
|
|
86
|
+
const markdownBytes = await readConfined(root, String(document.path), limits.markdownBytes), markdown = {
|
|
87
|
+
relativePath: String(document.path),
|
|
88
|
+
bytes: markdownBytes.length,
|
|
89
|
+
sha256: hash(markdownBytes),
|
|
90
|
+
mediaType: 'text/markdown',
|
|
91
|
+
};
|
|
46
92
|
validateAsset(markdown);
|
|
47
93
|
if (markdown.sha256 !== document.sha256 || markdown.bytes !== document.bytes)
|
|
48
|
-
throw
|
|
94
|
+
throw new SpongeError('integrity');
|
|
49
95
|
let unavailable = document.integrity === 'unavailable';
|
|
50
96
|
const assets = [];
|
|
51
97
|
let total = 0;
|
|
52
98
|
for (const raw of inspection.assets) {
|
|
53
99
|
const asset = object(raw);
|
|
54
100
|
if (!['verified', 'unavailable'].includes(String(asset.integrity)))
|
|
55
|
-
throw
|
|
101
|
+
throw new SpongeError('integrity');
|
|
56
102
|
unavailable ||= asset.integrity === 'unavailable';
|
|
57
103
|
const bytes = await readConfined(root, String(asset.path), limits.imageBytes);
|
|
58
104
|
total += bytes.length;
|
|
59
105
|
if (total > 500 * 1024 ** 2)
|
|
60
|
-
throw
|
|
61
|
-
const value = {
|
|
106
|
+
throw new SpongeError('limit-exceeded');
|
|
107
|
+
const value = {
|
|
108
|
+
relativePath: String(asset.path),
|
|
109
|
+
bytes: bytes.length,
|
|
110
|
+
sha256: hash(bytes),
|
|
111
|
+
mediaType: imageType(bytes),
|
|
112
|
+
relationship: 'image',
|
|
113
|
+
};
|
|
62
114
|
validateAsset(value);
|
|
63
115
|
const expectedHash = kind === 'pdf' ? asset.sha256 : asset.expectedSha256, expectedBytes = kind === 'pdf' ? asset.bytes : asset.expectedBytes;
|
|
64
116
|
if (asset.integrity === 'verified' && (expectedHash !== value.sha256 || expectedBytes !== value.bytes))
|
|
65
|
-
throw
|
|
117
|
+
throw new SpongeError('integrity');
|
|
66
118
|
assets.push(value);
|
|
67
119
|
}
|
|
68
120
|
if (kind === 'pdf') {
|
|
69
121
|
const source = object(inspection.source);
|
|
70
122
|
if (source.integrity !== 'verified')
|
|
71
|
-
throw
|
|
72
|
-
const asset = {
|
|
123
|
+
throw new SpongeError('integrity');
|
|
124
|
+
const asset = {
|
|
125
|
+
relativePath: String(source.path),
|
|
126
|
+
sha256: String(source.sha256),
|
|
127
|
+
bytes: Number(source.bytes),
|
|
128
|
+
mediaType: 'application/pdf',
|
|
129
|
+
relationship: 'source-pdf',
|
|
130
|
+
};
|
|
73
131
|
validateAsset(asset);
|
|
74
|
-
for await (const _ of bundleBytes(root, asset)) {
|
|
132
|
+
for await (const _ of bundleBytes(root, asset)) {
|
|
133
|
+
}
|
|
75
134
|
assets.push(asset);
|
|
76
135
|
}
|
|
77
|
-
if (assets.length !== new Set(assets.map(a => a.relativePath)).size)
|
|
78
|
-
throw
|
|
79
|
-
const sourceUrl = kind === 'pdf'
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
136
|
+
if (assets.length !== new Set(assets.map((a) => a.relativePath)).size)
|
|
137
|
+
throw new SpongeError('integrity');
|
|
138
|
+
const sourceUrl = kind === 'pdf'
|
|
139
|
+
? (object(manifest.source).requestedUrl ?? pathToFileURL(join(root, String(object(inspection.source).path))).href)
|
|
140
|
+
: inspection.sourceUrl;
|
|
141
|
+
if (typeof sourceUrl !== 'string' ||
|
|
142
|
+
sourceUrl.length > 16384 ||
|
|
143
|
+
!['http:', 'https:', 'file:'].includes(new URL(sourceUrl).protocol))
|
|
144
|
+
throw new SpongeError('integrity');
|
|
145
|
+
const observedAt = typeof manifest.capturedAt === 'string' && Number.isFinite(Date.parse(manifest.capturedAt))
|
|
146
|
+
? manifest.capturedAt
|
|
147
|
+
: '1970-01-01T00:00:00.000Z';
|
|
148
|
+
const metadata = kind === 'article'
|
|
149
|
+
? manifest.metadata === undefined
|
|
150
|
+
? metadataFromMarkdown(markdownBytes.toString('utf8'), observedAt)
|
|
151
|
+
: articleMetadata(manifest.metadata, observedAt)
|
|
152
|
+
: [];
|
|
84
153
|
if (kind === 'pdf') {
|
|
85
154
|
const document = object(manifest.document ?? {});
|
|
86
155
|
for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
|
|
@@ -89,9 +158,27 @@ export async function inspectImportBundle(root, verify) {
|
|
|
89
158
|
metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
|
|
90
159
|
}
|
|
91
160
|
}
|
|
92
|
-
metadata.push({
|
|
161
|
+
metadata.push({
|
|
162
|
+
field: 'historicalIntegrity',
|
|
163
|
+
value: unavailable ? 'unavailable' : 'verified',
|
|
164
|
+
source: kind === 'pdf' ? 'pdf' : 'html',
|
|
165
|
+
observedAt,
|
|
166
|
+
rule: 'sponge-import-v1',
|
|
167
|
+
});
|
|
93
168
|
if (hash(await readConfined(root, 'capture.json', limits.metadataBytes)) !== hash(manifestBytes))
|
|
94
|
-
throw
|
|
95
|
-
const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map(a => [a.relativePath, a.sha256, a.bytes])]));
|
|
96
|
-
return {
|
|
169
|
+
throw new SpongeError('integrity');
|
|
170
|
+
const manifestSha256 = hash(manifestBytes), importHash = hash(JSON.stringify([manifestSha256, markdown.sha256, ...assets.map((a) => [a.relativePath, a.sha256, a.bytes])]));
|
|
171
|
+
return {
|
|
172
|
+
root,
|
|
173
|
+
kind,
|
|
174
|
+
sourceUrl,
|
|
175
|
+
schemaVersion: Number(manifest.schemaVersion),
|
|
176
|
+
status: unavailable || inspection.status !== 'complete' ? 'partial' : 'complete',
|
|
177
|
+
markdown,
|
|
178
|
+
assets,
|
|
179
|
+
metadata,
|
|
180
|
+
manifestSha256,
|
|
181
|
+
historicalIntegrity: unavailable ? 'unavailable' : 'verified',
|
|
182
|
+
importHash,
|
|
183
|
+
};
|
|
97
184
|
}
|
|
@@ -9,11 +9,7 @@ import { type Form, type QuoteMatch } from '../digest/pipeline.js';
|
|
|
9
9
|
export declare const jungleTags: readonly ["agent-eng", "ai", "work", "business", "creative-coding", "finance", "music", "internet", "security", "history", "art", "hardware", "climate", "article", "interactive", "code", "x", "video"];
|
|
10
10
|
/** Tags that name how a source is consumed; a digest carries that as `form`. */
|
|
11
11
|
export declare const jungleFormatTags: readonly ["article", "interactive", "x", "video"];
|
|
12
|
-
|
|
13
|
-
readonly code: 'not-digest' | 'malformed' | 'no-capture';
|
|
14
|
-
constructor(code: 'not-digest' | 'malformed' | 'no-capture');
|
|
15
|
-
}
|
|
16
|
-
export interface JungleQuote {
|
|
12
|
+
interface JungleQuote {
|
|
17
13
|
text: string;
|
|
18
14
|
attribution: string;
|
|
19
15
|
speaker: string;
|
|
@@ -53,7 +49,7 @@ export interface JungleConversion {
|
|
|
53
49
|
/** The gold, when a capture was given and the gold passes every v2 rule against it. */
|
|
54
50
|
digest?: DigestV2;
|
|
55
51
|
}
|
|
56
|
-
|
|
52
|
+
type Yaml = string | boolean | Yaml[] | {
|
|
57
53
|
[k: string]: Yaml;
|
|
58
54
|
};
|
|
59
55
|
/** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
|
|
@@ -105,3 +101,4 @@ export declare function formOf(note: JungleNote): Form;
|
|
|
105
101
|
* the note first, until the rest fit the quote budget (never the last one); survivors are assigned to ideas, and the gold is checked.
|
|
106
102
|
*/
|
|
107
103
|
export declare function convertJungleDigest(noteMarkdown: string, capture?: string): JungleConversion;
|
|
104
|
+
export {};
|
|
@@ -1,38 +1,57 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { foldText, words } from '../annotations/quote.js';
|
|
3
|
-
import { checkDigest, digestLimits as L, parseDigest } from '../digest/schema.js';
|
|
3
|
+
import { checkDigest, digestLimits as L, parseDigest, } from '../digest/schema.js';
|
|
4
4
|
import { snapQuote } from '../digest/pipeline.js';
|
|
5
|
+
import { SpongeError } from '../errors.js';
|
|
5
6
|
/**
|
|
6
7
|
* Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v2. Jungle keeps
|
|
7
8
|
* quotes in their own section, so each verbatim quote is assigned to the idea it shares the most content words with.
|
|
8
9
|
* Owner text is never cut or rewritten: a note that does not fit v2 as written is reported, not converted.
|
|
9
10
|
* Nothing here reads files; callers pass the note and, when available, its capture Markdown.
|
|
10
11
|
*/
|
|
11
|
-
export const jungleTags = [
|
|
12
|
+
export const jungleTags = [
|
|
13
|
+
'agent-eng',
|
|
14
|
+
'ai',
|
|
15
|
+
'work',
|
|
16
|
+
'business',
|
|
17
|
+
'creative-coding',
|
|
18
|
+
'finance',
|
|
19
|
+
'music',
|
|
20
|
+
'internet',
|
|
21
|
+
'security',
|
|
22
|
+
'history',
|
|
23
|
+
'art',
|
|
24
|
+
'hardware',
|
|
25
|
+
'climate',
|
|
26
|
+
'article',
|
|
27
|
+
'interactive',
|
|
28
|
+
'code',
|
|
29
|
+
'x',
|
|
30
|
+
'video',
|
|
31
|
+
];
|
|
12
32
|
/** Tags that name how a source is consumed; a digest carries that as `form`. */
|
|
13
33
|
export const jungleFormatTags = ['article', 'interactive', 'x', 'video'];
|
|
14
|
-
|
|
15
|
-
code;
|
|
16
|
-
|
|
17
|
-
super(code);
|
|
18
|
-
this.code = code;
|
|
19
|
-
this.name = 'JungleDigestError';
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
const fail = (code = 'malformed') => { throw new JungleDigestError(code); };
|
|
34
|
+
const fail = (code = 'malformed') => {
|
|
35
|
+
throw new SpongeError(code);
|
|
36
|
+
};
|
|
23
37
|
/** The frontmatter subset Jungle writes: block maps and lists of scalars, quoted or plain. */
|
|
24
38
|
function parseYaml(lines) {
|
|
25
39
|
let i = 0;
|
|
26
40
|
const indent = (l) => l.length - l.trimStart().length;
|
|
27
|
-
const scalar = (v) => {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
41
|
+
const scalar = (v) => {
|
|
42
|
+
v = v.trim();
|
|
43
|
+
if (v.startsWith('"')) {
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse(v);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return fail();
|
|
49
|
+
}
|
|
33
50
|
}
|
|
34
|
-
|
|
35
|
-
|
|
51
|
+
if (v.startsWith("'"))
|
|
52
|
+
return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail();
|
|
53
|
+
return v === 'true' ? true : v === 'false' ? false : v;
|
|
54
|
+
};
|
|
36
55
|
const block = (at) => {
|
|
37
56
|
if (lines[i].trimStart().startsWith('- ')) {
|
|
38
57
|
const list = [];
|
|
@@ -56,12 +75,18 @@ function parseYaml(lines) {
|
|
|
56
75
|
while (i < lines.length && indent(lines[i]) === at && !lines[i].trimStart().startsWith('- ')) {
|
|
57
76
|
const m = /^([\w-]+):(?:\s+(.*))?$/.exec(lines[i].trim()) ?? fail();
|
|
58
77
|
i++;
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
78
|
+
// An own data property even for `__proto__`, which plain assignment would turn into the map's prototype.
|
|
79
|
+
Object.defineProperty(out, m[1], {
|
|
80
|
+
value: m[2] !== undefined && m[2] !== ''
|
|
81
|
+
? scalar(m[2])
|
|
82
|
+
: i < lines.length &&
|
|
83
|
+
(indent(lines[i]) > at || (indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
|
|
84
|
+
? block(indent(lines[i]))
|
|
85
|
+
: '',
|
|
86
|
+
enumerable: true,
|
|
87
|
+
writable: true,
|
|
88
|
+
configurable: true,
|
|
89
|
+
});
|
|
65
90
|
}
|
|
66
91
|
return out;
|
|
67
92
|
};
|
|
@@ -69,14 +94,21 @@ function parseYaml(lines) {
|
|
|
69
94
|
return i === lines.length ? value : fail();
|
|
70
95
|
}
|
|
71
96
|
const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
72
|
-
const strings = (v) => Array.isArray(v) && v.every(x => typeof x === 'string') ? v : fail();
|
|
97
|
+
const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === 'string') ? v : fail());
|
|
73
98
|
/** Split a Markdown file into its frontmatter lines and body. */
|
|
74
99
|
function frontmatter(markdown) {
|
|
75
100
|
const m = /^---\n([^]*?)\n---(?:\n|$)/.exec(markdown.replace(/\r\n/g, '\n'));
|
|
76
|
-
return m
|
|
101
|
+
return m
|
|
102
|
+
? {
|
|
103
|
+
head: m[1].split('\n').filter((l) => l.trim() && !l.trimStart().startsWith('#')),
|
|
104
|
+
body: markdown.replace(/\r\n/g, '\n').slice(m[0].length),
|
|
105
|
+
}
|
|
106
|
+
: { head: [], body: markdown };
|
|
77
107
|
}
|
|
78
108
|
/** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
|
|
79
|
-
export function captureBody(capture) {
|
|
109
|
+
export function captureBody(capture) {
|
|
110
|
+
return frontmatter(capture).body;
|
|
111
|
+
}
|
|
80
112
|
/** `## Heading` sections of a note body, trimmed. */
|
|
81
113
|
function sectionsOf(body) {
|
|
82
114
|
const sections = new Map();
|
|
@@ -87,10 +119,19 @@ function sectionsOf(body) {
|
|
|
87
119
|
return sections;
|
|
88
120
|
}
|
|
89
121
|
/** `- **Claim.** detail` bullets: Jungle's Ideas, and a conversation's Debate. */
|
|
90
|
-
const ideaList = (text) => text
|
|
122
|
+
const ideaList = (text) => text
|
|
123
|
+
.split('\n')
|
|
124
|
+
.filter((l) => l.startsWith('- '))
|
|
125
|
+
.map((l) => {
|
|
126
|
+
const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail();
|
|
127
|
+
return { claim: m[1].trim(), detail: m[2].trim() };
|
|
128
|
+
});
|
|
91
129
|
/** Blockquotes ending `> Attribution: Speaker, context`. */
|
|
92
|
-
const quoteList = (text) => text
|
|
93
|
-
|
|
130
|
+
const quoteList = (text) => text
|
|
131
|
+
.split(/\n\s*\n/)
|
|
132
|
+
.filter((b) => b.trim())
|
|
133
|
+
.map((b) => {
|
|
134
|
+
const lines = b.split('\n').map((l) => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex((l) => l.startsWith('Attribution:'));
|
|
94
135
|
if (at < 1)
|
|
95
136
|
fail();
|
|
96
137
|
const text = lines.slice(0, at).join(' ').replace(/\s+/g, ' ').trim(), attribution = lines[at].slice('Attribution:'.length).trim();
|
|
@@ -101,7 +142,7 @@ export function parseJungleNote(markdown) {
|
|
|
101
142
|
fail();
|
|
102
143
|
const { head, body } = frontmatter(markdown), meta = parseYaml(head);
|
|
103
144
|
if (!object(meta) || !object(meta.reading))
|
|
104
|
-
throw new
|
|
145
|
+
throw new SpongeError('not-digest');
|
|
105
146
|
const reading = meta.reading;
|
|
106
147
|
if (reading.form !== 'digest')
|
|
107
148
|
fail('not-digest');
|
|
@@ -109,56 +150,111 @@ export function parseJungleNote(markdown) {
|
|
|
109
150
|
const gist = (sections.get('Gist') ?? fail()).replace(/\s+/g, ' ').trim();
|
|
110
151
|
const ideas = ideaList(sections.get('Ideas') ?? fail());
|
|
111
152
|
const quotes = quoteList(sections.get('Quotes') ?? '');
|
|
112
|
-
if (!gist || !ideas.length || quotes.some(q => !q.text || !q.speaker))
|
|
153
|
+
if (!gist || !ideas.length || quotes.some((q) => !q.text || !q.speaker))
|
|
113
154
|
fail();
|
|
114
|
-
const authors = Array.isArray(reading.authors)
|
|
155
|
+
const authors = Array.isArray(reading.authors)
|
|
156
|
+
? reading.authors.map((a) => (object(a) && typeof a.name === 'string' ? a.name : fail()))
|
|
157
|
+
: [];
|
|
115
158
|
const relations = object(meta.relations) ? meta.relations : {}, synth = Array.isArray(relations.synthesizes) ? strings(relations.synthesizes) : [];
|
|
116
|
-
return {
|
|
159
|
+
return {
|
|
160
|
+
slug: typeof reading.slug === 'string' ? reading.slug : fail(),
|
|
161
|
+
title: typeof meta.title === 'string' ? meta.title : '',
|
|
162
|
+
status: typeof reading.status === 'string' ? reading.status : '',
|
|
163
|
+
sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
|
|
164
|
+
authors,
|
|
165
|
+
tags: meta.tags === undefined ? [] : strings(meta.tags),
|
|
166
|
+
...(synth[0] ? { capture: synth[0] } : {}),
|
|
167
|
+
gist,
|
|
168
|
+
ideas,
|
|
169
|
+
quotes,
|
|
170
|
+
};
|
|
117
171
|
}
|
|
118
172
|
export function parseJungleEditorial(markdown) {
|
|
119
173
|
if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
|
|
120
174
|
fail();
|
|
121
175
|
const { head, body } = frontmatter(markdown), meta = parseYaml(head);
|
|
122
176
|
if (!object(meta) || !object(meta.reading))
|
|
123
|
-
throw new
|
|
177
|
+
throw new SpongeError('not-digest');
|
|
124
178
|
const reading = meta.reading, relations = object(meta.relations) ? meta.relations : {};
|
|
125
179
|
const form = reading.form, slug = reading.slug;
|
|
126
180
|
if (typeof form !== 'string' || !form || typeof slug !== 'string' || !/^[a-z0-9][a-z0-9-]{0,199}$/.test(slug))
|
|
127
181
|
return fail();
|
|
128
|
-
const capture = [relations.synthesizes, relations['evidenced-by']].flatMap(v => Array.isArray(v) ? strings(v) : [])[0] ??
|
|
182
|
+
const capture = [relations.synthesizes, relations['evidenced-by']].flatMap((v) => (Array.isArray(v) ? strings(v) : []))[0] ??
|
|
183
|
+
fail('no-capture');
|
|
129
184
|
// A digest must parse strictly; the other forms keep whatever structure they have, and the Markdown keeps the rest.
|
|
130
|
-
const sections = sectionsOf(body), loose = (f) => {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
185
|
+
const sections = sectionsOf(body), loose = (f) => {
|
|
186
|
+
if (form === 'digest')
|
|
187
|
+
return f();
|
|
188
|
+
try {
|
|
189
|
+
return f();
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
const gistText = sections.get('Gist') ??
|
|
196
|
+
sections.get('Summary') ??
|
|
197
|
+
(typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
|
|
138
198
|
if (form === 'digest')
|
|
139
199
|
parseJungleNote(markdown);
|
|
140
200
|
const gist = gistText?.replace(/\s+/g, ' ').trim(), ideas = ideaText === undefined ? undefined : loose(() => ideaList(ideaText)), quotes = quoteText === undefined ? undefined : loose(() => quoteList(quoteText));
|
|
141
|
-
return {
|
|
201
|
+
return {
|
|
202
|
+
schemaVersion: 1,
|
|
203
|
+
source: 'jungle-reading',
|
|
204
|
+
form,
|
|
205
|
+
slug,
|
|
206
|
+
status: typeof reading.status === 'string' ? reading.status : '',
|
|
207
|
+
title: typeof meta.title === 'string' ? meta.title : '',
|
|
208
|
+
sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
|
|
209
|
+
capture,
|
|
210
|
+
frontmatter: meta,
|
|
211
|
+
...(gist ? { gist } : {}),
|
|
212
|
+
...(ideas?.length ? { ideas } : {}),
|
|
213
|
+
...(quotes?.length && quotes.every((q) => q.text && q.speaker) ? { quotes } : {}),
|
|
214
|
+
markdownSha256: createHash('sha256').update(markdown).digest('hex'),
|
|
215
|
+
markdown,
|
|
216
|
+
};
|
|
142
217
|
}
|
|
143
218
|
const stop = new Set('the a an and or but of to in on for with by from at as is are was were be been it its this that these those not no than then so such into about over under more most less can could would should will just also only very their there they them we our you your he she his her i my me what which who how when where why do does did has have had'.split(' '));
|
|
144
219
|
/** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
|
|
145
220
|
export function contentTerms(text) {
|
|
146
|
-
return new Set(foldText(text)
|
|
221
|
+
return new Set(foldText(text)
|
|
222
|
+
.toLowerCase()
|
|
223
|
+
.split(/[^a-z0-9]+/)
|
|
224
|
+
.filter((w) => w.length > 2 && !stop.has(w))
|
|
225
|
+
.map((w) => w.replace(/(?:ing|ed|es|s)$/, ''))
|
|
226
|
+
.filter((w) => w.length > 2));
|
|
227
|
+
}
|
|
228
|
+
export function termSimilarity(a, b) {
|
|
229
|
+
if (!a.size || !b.size)
|
|
230
|
+
return 0;
|
|
231
|
+
let n = 0;
|
|
232
|
+
for (const t of a)
|
|
233
|
+
if (b.has(t))
|
|
234
|
+
n++;
|
|
235
|
+
return n / Math.sqrt(a.size * b.size);
|
|
147
236
|
}
|
|
148
|
-
export function termSimilarity(a, b) { if (!a.size || !b.size)
|
|
149
|
-
return 0; let n = 0; for (const t of a)
|
|
150
|
-
if (b.has(t))
|
|
151
|
-
n++; return n / Math.sqrt(a.size * b.size); }
|
|
152
237
|
/**
|
|
153
238
|
* Each quote in order goes to the idea sharing the most content words with its claim and detail (ties: the earliest idea),
|
|
154
239
|
* else to the next best idea with room; null when every idea already holds `cap` quotes.
|
|
155
240
|
*/
|
|
156
241
|
export function assignQuotes(ideas, quotes, cap = L.maxQuotesPerIdea) {
|
|
157
|
-
const terms = ideas.map(i => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
|
|
158
|
-
return quotes.map(
|
|
159
|
-
|
|
242
|
+
const terms = ideas.map((i) => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
|
|
243
|
+
return quotes.map((q) => {
|
|
244
|
+
const t = contentTerms(q.text), pick = terms
|
|
245
|
+
.map((it, i) => ({ i, s: termSimilarity(t, it) }))
|
|
246
|
+
.sort((a, b) => b.s - a.s || a.i - b.i)
|
|
247
|
+
.find((r) => load[r.i] < cap);
|
|
248
|
+
if (!pick)
|
|
249
|
+
return null;
|
|
250
|
+
load[pick.i]++;
|
|
251
|
+
return pick.i;
|
|
252
|
+
});
|
|
160
253
|
}
|
|
161
|
-
export const firstSentence = (text) => {
|
|
254
|
+
export const firstSentence = (text) => {
|
|
255
|
+
const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text);
|
|
256
|
+
return m ? [m[1], text.slice(m[0].length)] : [text, ''];
|
|
257
|
+
};
|
|
162
258
|
export function formOf(note) {
|
|
163
259
|
if (note.tags.includes('video'))
|
|
164
260
|
return 'video';
|
|
@@ -173,24 +269,55 @@ export function formOf(note) {
|
|
|
173
269
|
*/
|
|
174
270
|
export function convertJungleDigest(noteMarkdown, capture) {
|
|
175
271
|
const note = parseJungleNote(noteMarkdown), source = capture === undefined ? undefined : captureBody(capture), [lead, context] = firstSentence(note.gist);
|
|
176
|
-
const located = note.quotes.map(q => source === undefined ? undefined : snapQuote(source, q.text).match);
|
|
177
|
-
const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped'
|
|
178
|
-
|
|
272
|
+
const located = note.quotes.map((q) => (source === undefined ? undefined : snapQuote(source, q.text).match));
|
|
273
|
+
const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped'
|
|
274
|
+
? 'not-verbatim'
|
|
275
|
+
: words(q.text) < L.minQuoteWords
|
|
276
|
+
? 'too-short'
|
|
277
|
+
: undefined);
|
|
278
|
+
let kept = note.quotes.flatMap((q, k) => (dropped[k] ? [] : [{ q, k, n: words(q.text) }]));
|
|
179
279
|
// The pipeline's budget rule, applied to owner quotes: drop whole quotes, never edit one. A single quote over budget still fails the check.
|
|
180
|
-
const budget = source === undefined
|
|
280
|
+
const budget = source === undefined
|
|
281
|
+
? Infinity
|
|
282
|
+
: Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)));
|
|
181
283
|
while (kept.length > 1 && (kept.length > L.maxQuotes || kept.reduce((n, x) => n + x.n, 0) > budget)) {
|
|
182
284
|
const x = [...kept].sort((a, b) => b.n - a.n || b.k - a.k)[0];
|
|
183
285
|
dropped[x.k] = 'budget';
|
|
184
|
-
kept = kept.filter(y => y !== x);
|
|
286
|
+
kept = kept.filter((y) => y !== x);
|
|
185
287
|
}
|
|
186
288
|
const assigned = note.quotes.map(() => null);
|
|
187
|
-
assignQuotes(note.ideas, kept.map(x => x.q)).forEach((idea, n) => {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
289
|
+
assignQuotes(note.ideas, kept.map((x) => x.q)).forEach((idea, n) => {
|
|
290
|
+
if (idea === null)
|
|
291
|
+
dropped[kept[n].k] = 'overflow';
|
|
292
|
+
else
|
|
293
|
+
assigned[kept[n].k] = idea;
|
|
294
|
+
});
|
|
295
|
+
const ideas = note.ideas.map((idea, i) => ({
|
|
296
|
+
claim: idea.claim,
|
|
297
|
+
detail: idea.detail,
|
|
298
|
+
quotes: note.quotes.flatMap((q, k) => assigned[k] === i ? [{ text: q.text, ...(q.attribution ? { attribution: q.attribution } : {}) }] : []),
|
|
299
|
+
}));
|
|
300
|
+
const tags = note.tags.filter((t) => t !== 'reading' && !jungleFormatTags.includes(t));
|
|
301
|
+
const gold = {
|
|
302
|
+
schemaVersion: 2,
|
|
303
|
+
form: formOf(note),
|
|
304
|
+
gist: { lead, ...(context ? { context } : {}) },
|
|
305
|
+
ideas,
|
|
306
|
+
tags,
|
|
307
|
+
};
|
|
308
|
+
const issues = [
|
|
309
|
+
...parseDigest(gold).issues,
|
|
310
|
+
...(source === undefined ? [] : checkDigest(gold, source, { tags: jungleTags }).issues),
|
|
311
|
+
];
|
|
312
|
+
return {
|
|
313
|
+
note,
|
|
314
|
+
gold,
|
|
315
|
+
quotes: note.quotes.map((_, k) => ({
|
|
316
|
+
idea: assigned[k] ?? null,
|
|
317
|
+
...(located[k] ? { match: located[k] } : {}),
|
|
318
|
+
...(dropped[k] ? { dropped: dropped[k] } : {}),
|
|
319
|
+
})),
|
|
320
|
+
issues,
|
|
321
|
+
...(source !== undefined && !issues.length ? { digest: gold } : {}),
|
|
322
|
+
};
|
|
196
323
|
}
|