@hraness/sponge 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +39 -17
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +10 -46
- package/dist/src/digest/editorial.js +73 -59
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +53 -32
- package/dist/src/digest/pipeline.js +357 -125
- package/dist/src/digest/schema.d.ts +44 -17
- package/dist/src/digest/schema.js +132 -44
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +31 -24
- package/dist/src/importers/jungle-digest.js +217 -115
- package/dist/src/importers/jungle-store.d.ts +20 -11
- package/dist/src/importers/jungle-store.js +136 -41
- package/dist/src/importers/jungle.d.ts +32 -16
- package/dist/src/importers/jungle.js +145 -32
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +108 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -77
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -1,37 +1,57 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { foldText,
|
|
3
|
-
import { checkDigest, digestLimits as L } from '../digest/schema.js';
|
|
4
|
-
import {
|
|
2
|
+
import { foldText, words } from '../annotations/quote.js';
|
|
3
|
+
import { checkDigest, digestLimits as L, parseDigest, } from '../digest/schema.js';
|
|
4
|
+
import { snapQuote } from '../digest/pipeline.js';
|
|
5
|
+
import { SpongeError } from '../errors.js';
|
|
5
6
|
/**
|
|
6
|
-
* Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest
|
|
7
|
-
* quotes in their own section, so each quote is assigned to the idea it shares the most content words with.
|
|
7
|
+
* Pure conversion of one reviewed Jungle Reading digest note (`kb/notes/reading/<slug>.md`) into Digest v2. Jungle keeps
|
|
8
|
+
* quotes in their own section, so each verbatim quote is assigned to the idea it shares the most content words with.
|
|
9
|
+
* Owner text is never cut or rewritten: a note that does not fit v2 as written is reported, not converted.
|
|
8
10
|
* Nothing here reads files; callers pass the note and, when available, its capture Markdown.
|
|
9
11
|
*/
|
|
10
|
-
export const jungleTags = [
|
|
11
|
-
|
|
12
|
+
export const jungleTags = [
|
|
13
|
+
'agent-eng',
|
|
14
|
+
'ai',
|
|
15
|
+
'work',
|
|
16
|
+
'business',
|
|
17
|
+
'creative-coding',
|
|
18
|
+
'finance',
|
|
19
|
+
'music',
|
|
20
|
+
'internet',
|
|
21
|
+
'security',
|
|
22
|
+
'history',
|
|
23
|
+
'art',
|
|
24
|
+
'hardware',
|
|
25
|
+
'climate',
|
|
26
|
+
'article',
|
|
27
|
+
'interactive',
|
|
28
|
+
'code',
|
|
29
|
+
'x',
|
|
30
|
+
'video',
|
|
31
|
+
];
|
|
32
|
+
/** Tags that name how a source is consumed; a digest carries that as `form`. */
|
|
12
33
|
export const jungleFormatTags = ['article', 'interactive', 'x', 'video'];
|
|
13
|
-
|
|
14
|
-
code;
|
|
15
|
-
|
|
16
|
-
super(code);
|
|
17
|
-
this.code = code;
|
|
18
|
-
this.name = 'JungleDigestError';
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
const fail = (code = 'malformed') => { throw new JungleDigestError(code); };
|
|
34
|
+
const fail = (code = 'malformed') => {
|
|
35
|
+
throw new SpongeError(code);
|
|
36
|
+
};
|
|
22
37
|
/** The frontmatter subset Jungle writes: block maps and lists of scalars, quoted or plain. */
|
|
23
38
|
function parseYaml(lines) {
|
|
24
39
|
let i = 0;
|
|
25
40
|
const indent = (l) => l.length - l.trimStart().length;
|
|
26
|
-
const scalar = (v) => {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
41
|
+
const scalar = (v) => {
|
|
42
|
+
v = v.trim();
|
|
43
|
+
if (v.startsWith('"')) {
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse(v);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return fail();
|
|
49
|
+
}
|
|
32
50
|
}
|
|
33
|
-
|
|
34
|
-
|
|
51
|
+
if (v.startsWith("'"))
|
|
52
|
+
return v.endsWith("'") && v.length > 1 ? v.slice(1, -1).replace(/''/g, "'") : fail();
|
|
53
|
+
return v === 'true' ? true : v === 'false' ? false : v;
|
|
54
|
+
};
|
|
35
55
|
const block = (at) => {
|
|
36
56
|
if (lines[i].trimStart().startsWith('- ')) {
|
|
37
57
|
const list = [];
|
|
@@ -55,12 +75,18 @@ function parseYaml(lines) {
|
|
|
55
75
|
while (i < lines.length && indent(lines[i]) === at && !lines[i].trimStart().startsWith('- ')) {
|
|
56
76
|
const m = /^([\w-]+):(?:\s+(.*))?$/.exec(lines[i].trim()) ?? fail();
|
|
57
77
|
i++;
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
78
|
+
// An own data property even for `__proto__`, which plain assignment would turn into the map's prototype.
|
|
79
|
+
Object.defineProperty(out, m[1], {
|
|
80
|
+
value: m[2] !== undefined && m[2] !== ''
|
|
81
|
+
? scalar(m[2])
|
|
82
|
+
: i < lines.length &&
|
|
83
|
+
(indent(lines[i]) > at || (indent(lines[i]) === at && lines[i].trimStart().startsWith('- ')))
|
|
84
|
+
? block(indent(lines[i]))
|
|
85
|
+
: '',
|
|
86
|
+
enumerable: true,
|
|
87
|
+
writable: true,
|
|
88
|
+
configurable: true,
|
|
89
|
+
});
|
|
64
90
|
}
|
|
65
91
|
return out;
|
|
66
92
|
};
|
|
@@ -68,14 +94,21 @@ function parseYaml(lines) {
|
|
|
68
94
|
return i === lines.length ? value : fail();
|
|
69
95
|
}
|
|
70
96
|
const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
71
|
-
const strings = (v) => Array.isArray(v) && v.every(x => typeof x === 'string') ? v : fail();
|
|
97
|
+
const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === 'string') ? v : fail());
|
|
72
98
|
/** Split a Markdown file into its frontmatter lines and body. */
|
|
73
99
|
function frontmatter(markdown) {
|
|
74
100
|
const m = /^---\n([^]*?)\n---(?:\n|$)/.exec(markdown.replace(/\r\n/g, '\n'));
|
|
75
|
-
return m
|
|
101
|
+
return m
|
|
102
|
+
? {
|
|
103
|
+
head: m[1].split('\n').filter((l) => l.trim() && !l.trimStart().startsWith('#')),
|
|
104
|
+
body: markdown.replace(/\r\n/g, '\n').slice(m[0].length),
|
|
105
|
+
}
|
|
106
|
+
: { head: [], body: markdown };
|
|
76
107
|
}
|
|
77
108
|
/** Capture Markdown without its YAML frontmatter: what a Sponge revision of the same page would hold. */
|
|
78
|
-
export function captureBody(capture) {
|
|
109
|
+
export function captureBody(capture) {
|
|
110
|
+
return frontmatter(capture).body;
|
|
111
|
+
}
|
|
79
112
|
/** `## Heading` sections of a note body, trimmed. */
|
|
80
113
|
function sectionsOf(body) {
|
|
81
114
|
const sections = new Map();
|
|
@@ -86,10 +119,19 @@ function sectionsOf(body) {
|
|
|
86
119
|
return sections;
|
|
87
120
|
}
|
|
88
121
|
/** `- **Claim.** detail` bullets: Jungle's Ideas, and a conversation's Debate. */
|
|
89
|
-
const ideaList = (text) => text
|
|
122
|
+
const ideaList = (text) => text
|
|
123
|
+
.split('\n')
|
|
124
|
+
.filter((l) => l.startsWith('- '))
|
|
125
|
+
.map((l) => {
|
|
126
|
+
const m = /^- \*\*(.+?)\*\*\s*(.*)$/.exec(l.trim()) ?? fail();
|
|
127
|
+
return { claim: m[1].trim(), detail: m[2].trim() };
|
|
128
|
+
});
|
|
90
129
|
/** Blockquotes ending `> Attribution: Speaker, context`. */
|
|
91
|
-
const quoteList = (text) => text
|
|
92
|
-
|
|
130
|
+
const quoteList = (text) => text
|
|
131
|
+
.split(/\n\s*\n/)
|
|
132
|
+
.filter((b) => b.trim())
|
|
133
|
+
.map((b) => {
|
|
134
|
+
const lines = b.split('\n').map((l) => /^>\s?(.*)$/.exec(l)?.[1] ?? fail()), at = lines.findIndex((l) => l.startsWith('Attribution:'));
|
|
93
135
|
if (at < 1)
|
|
94
136
|
fail();
|
|
95
137
|
const text = lines.slice(0, at).join(' ').replace(/\s+/g, ' ').trim(), attribution = lines[at].slice('Attribution:'.length).trim();
|
|
@@ -100,7 +142,7 @@ export function parseJungleNote(markdown) {
|
|
|
100
142
|
fail();
|
|
101
143
|
const { head, body } = frontmatter(markdown), meta = parseYaml(head);
|
|
102
144
|
if (!object(meta) || !object(meta.reading))
|
|
103
|
-
throw new
|
|
145
|
+
throw new SpongeError('not-digest');
|
|
104
146
|
const reading = meta.reading;
|
|
105
147
|
if (reading.form !== 'digest')
|
|
106
148
|
fail('not-digest');
|
|
@@ -108,63 +150,111 @@ export function parseJungleNote(markdown) {
|
|
|
108
150
|
const gist = (sections.get('Gist') ?? fail()).replace(/\s+/g, ' ').trim();
|
|
109
151
|
const ideas = ideaList(sections.get('Ideas') ?? fail());
|
|
110
152
|
const quotes = quoteList(sections.get('Quotes') ?? '');
|
|
111
|
-
if (!gist || !ideas.length || quotes.some(q => !q.text || !q.speaker))
|
|
153
|
+
if (!gist || !ideas.length || quotes.some((q) => !q.text || !q.speaker))
|
|
112
154
|
fail();
|
|
113
|
-
const authors = Array.isArray(reading.authors)
|
|
155
|
+
const authors = Array.isArray(reading.authors)
|
|
156
|
+
? reading.authors.map((a) => (object(a) && typeof a.name === 'string' ? a.name : fail()))
|
|
157
|
+
: [];
|
|
114
158
|
const relations = object(meta.relations) ? meta.relations : {}, synth = Array.isArray(relations.synthesizes) ? strings(relations.synthesizes) : [];
|
|
115
|
-
return {
|
|
159
|
+
return {
|
|
160
|
+
slug: typeof reading.slug === 'string' ? reading.slug : fail(),
|
|
161
|
+
title: typeof meta.title === 'string' ? meta.title : '',
|
|
162
|
+
status: typeof reading.status === 'string' ? reading.status : '',
|
|
163
|
+
sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
|
|
164
|
+
authors,
|
|
165
|
+
tags: meta.tags === undefined ? [] : strings(meta.tags),
|
|
166
|
+
...(synth[0] ? { capture: synth[0] } : {}),
|
|
167
|
+
gist,
|
|
168
|
+
ideas,
|
|
169
|
+
quotes,
|
|
170
|
+
};
|
|
116
171
|
}
|
|
117
172
|
export function parseJungleEditorial(markdown) {
|
|
118
173
|
if (typeof markdown !== 'string' || markdown.length > 4 * 1024 * 1024)
|
|
119
174
|
fail();
|
|
120
175
|
const { head, body } = frontmatter(markdown), meta = parseYaml(head);
|
|
121
176
|
if (!object(meta) || !object(meta.reading))
|
|
122
|
-
throw new
|
|
177
|
+
throw new SpongeError('not-digest');
|
|
123
178
|
const reading = meta.reading, relations = object(meta.relations) ? meta.relations : {};
|
|
124
179
|
const form = reading.form, slug = reading.slug;
|
|
125
180
|
if (typeof form !== 'string' || !form || typeof slug !== 'string' || !/^[a-z0-9][a-z0-9-]{0,199}$/.test(slug))
|
|
126
181
|
return fail();
|
|
127
|
-
const capture = [relations.synthesizes, relations['evidenced-by']].flatMap(v => Array.isArray(v) ? strings(v) : [])[0] ??
|
|
182
|
+
const capture = [relations.synthesizes, relations['evidenced-by']].flatMap((v) => (Array.isArray(v) ? strings(v) : []))[0] ??
|
|
183
|
+
fail('no-capture');
|
|
128
184
|
// A digest must parse strictly; the other forms keep whatever structure they have, and the Markdown keeps the rest.
|
|
129
|
-
const sections = sectionsOf(body), loose = (f) => {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
185
|
+
const sections = sectionsOf(body), loose = (f) => {
|
|
186
|
+
if (form === 'digest')
|
|
187
|
+
return f();
|
|
188
|
+
try {
|
|
189
|
+
return f();
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
const gistText = sections.get('Gist') ??
|
|
196
|
+
sections.get('Summary') ??
|
|
197
|
+
(typeof reading.summary === 'string' ? reading.summary : undefined), ideaText = sections.get('Ideas') ?? sections.get('Debate'), quoteText = sections.get('Quotes');
|
|
137
198
|
if (form === 'digest')
|
|
138
199
|
parseJungleNote(markdown);
|
|
139
200
|
const gist = gistText?.replace(/\s+/g, ' ').trim(), ideas = ideaText === undefined ? undefined : loose(() => ideaList(ideaText)), quotes = quoteText === undefined ? undefined : loose(() => quoteList(quoteText));
|
|
140
|
-
return {
|
|
201
|
+
return {
|
|
202
|
+
schemaVersion: 1,
|
|
203
|
+
source: 'jungle-reading',
|
|
204
|
+
form,
|
|
205
|
+
slug,
|
|
206
|
+
status: typeof reading.status === 'string' ? reading.status : '',
|
|
207
|
+
title: typeof meta.title === 'string' ? meta.title : '',
|
|
208
|
+
sourceUrl: typeof reading.source_url === 'string' ? reading.source_url : '',
|
|
209
|
+
capture,
|
|
210
|
+
frontmatter: meta,
|
|
211
|
+
...(gist ? { gist } : {}),
|
|
212
|
+
...(ideas?.length ? { ideas } : {}),
|
|
213
|
+
...(quotes?.length && quotes.every((q) => q.text && q.speaker) ? { quotes } : {}),
|
|
214
|
+
markdownSha256: createHash('sha256').update(markdown).digest('hex'),
|
|
215
|
+
markdown,
|
|
216
|
+
};
|
|
141
217
|
}
|
|
142
218
|
const stop = new Set('the a an and or but of to in on for with by from at as is are was were be been it its this that these those not no than then so such into about over under more most less can could would should will just also only very their there they them we our you your he she his her i my me what which who how when where why do does did has have had'.split(' '));
|
|
143
219
|
/** Content-word stems: lowercase, stopwords out, a crude plural/tense strip. Shared by idea assignment and lexical coverage. */
|
|
144
220
|
export function contentTerms(text) {
|
|
145
|
-
return new Set(foldText(text)
|
|
221
|
+
return new Set(foldText(text)
|
|
222
|
+
.toLowerCase()
|
|
223
|
+
.split(/[^a-z0-9]+/)
|
|
224
|
+
.filter((w) => w.length > 2 && !stop.has(w))
|
|
225
|
+
.map((w) => w.replace(/(?:ing|ed|es|s)$/, ''))
|
|
226
|
+
.filter((w) => w.length > 2));
|
|
146
227
|
}
|
|
147
|
-
export function termSimilarity(a, b) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
quotes.forEach((q, qi) => { const t = contentTerms(q.text); ideaTerms.forEach((it, ii) => { const s = termSimilarity(t, it); if (s > 0)
|
|
156
|
-
pairs.push({ q: qi, i: ii, s }); }); });
|
|
157
|
-
pairs.sort((a, b) => b.s - a.s || Math.abs(a.q - a.i) - Math.abs(b.q - b.i) || a.q - b.q);
|
|
158
|
-
const out = quotes.map(() => null), load = ideas.map(() => 0);
|
|
159
|
-
for (const cap of [1, L.maxQuotesPerIdea])
|
|
160
|
-
for (const p of pairs)
|
|
161
|
-
if (out[p.q] === null && load[p.i] < cap) {
|
|
162
|
-
out[p.q] = p.i;
|
|
163
|
-
load[p.i]++;
|
|
164
|
-
}
|
|
165
|
-
return out;
|
|
228
|
+
export function termSimilarity(a, b) {
|
|
229
|
+
if (!a.size || !b.size)
|
|
230
|
+
return 0;
|
|
231
|
+
let n = 0;
|
|
232
|
+
for (const t of a)
|
|
233
|
+
if (b.has(t))
|
|
234
|
+
n++;
|
|
235
|
+
return n / Math.sqrt(a.size * b.size);
|
|
166
236
|
}
|
|
167
|
-
|
|
237
|
+
/**
|
|
238
|
+
* Each quote in order goes to the idea sharing the most content words with its claim and detail (ties: the earliest idea),
|
|
239
|
+
* else to the next best idea with room; null when every idea already holds `cap` quotes.
|
|
240
|
+
*/
|
|
241
|
+
export function assignQuotes(ideas, quotes, cap = L.maxQuotesPerIdea) {
|
|
242
|
+
const terms = ideas.map((i) => contentTerms(`${i.claim} ${i.detail}`)), load = ideas.map(() => 0);
|
|
243
|
+
return quotes.map((q) => {
|
|
244
|
+
const t = contentTerms(q.text), pick = terms
|
|
245
|
+
.map((it, i) => ({ i, s: termSimilarity(t, it) }))
|
|
246
|
+
.sort((a, b) => b.s - a.s || a.i - b.i)
|
|
247
|
+
.find((r) => load[r.i] < cap);
|
|
248
|
+
if (!pick)
|
|
249
|
+
return null;
|
|
250
|
+
load[pick.i]++;
|
|
251
|
+
return pick.i;
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
export const firstSentence = (text) => {
|
|
255
|
+
const m = /^(.+?[.!?]["”’)]*)\s+(?=["“(]?[A-Z0-9])/.exec(text);
|
|
256
|
+
return m ? [m[1], text.slice(m[0].length)] : [text, ''];
|
|
257
|
+
};
|
|
168
258
|
export function formOf(note) {
|
|
169
259
|
if (note.tags.includes('video'))
|
|
170
260
|
return 'video';
|
|
@@ -172,50 +262,62 @@ export function formOf(note) {
|
|
|
172
262
|
return 'thread';
|
|
173
263
|
return /\.pdf(?:$|[?#])|arxiv\.org\/pdf\//i.test(note.sourceUrl) ? 'pdf' : 'article';
|
|
174
264
|
}
|
|
175
|
-
/**
|
|
265
|
+
/**
|
|
266
|
+
* Convert one Jungle digest note. With `capture` (the capture Markdown, frontmatter allowed), each quote is kept only when it
|
|
267
|
+
* occurs as written (folded typography only) and has at least four words; whole quotes then leave, longest and then latest in
|
|
268
|
+
* the note first, until the rest fit the quote budget (never the last one); survivors are assigned to ideas, and the gold is checked.
|
|
269
|
+
*/
|
|
176
270
|
export function convertJungleDigest(noteMarkdown, capture) {
|
|
177
|
-
const note = parseJungleNote(noteMarkdown),
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
271
|
+
const note = parseJungleNote(noteMarkdown), source = capture === undefined ? undefined : captureBody(capture), [lead, context] = firstSentence(note.gist);
|
|
272
|
+
const located = note.quotes.map((q) => (source === undefined ? undefined : snapQuote(source, q.text).match));
|
|
273
|
+
const dropped = note.quotes.map((q, k) => located[k] === 'missing' || located[k] === 'snapped'
|
|
274
|
+
? 'not-verbatim'
|
|
275
|
+
: words(q.text) < L.minQuoteWords
|
|
276
|
+
? 'too-short'
|
|
277
|
+
: undefined);
|
|
278
|
+
let kept = note.quotes.flatMap((q, k) => (dropped[k] ? [] : [{ q, k, n: words(q.text) }]));
|
|
279
|
+
// The pipeline's budget rule, applied to owner quotes: drop whole quotes, never edit one. A single quote over budget still fails the check.
|
|
280
|
+
const budget = source === undefined
|
|
281
|
+
? Infinity
|
|
282
|
+
: Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)));
|
|
283
|
+
while (kept.length > 1 && (kept.length > L.maxQuotes || kept.reduce((n, x) => n + x.n, 0) > budget)) {
|
|
284
|
+
const x = [...kept].sort((a, b) => b.n - a.n || b.k - a.k)[0];
|
|
285
|
+
dropped[x.k] = 'budget';
|
|
286
|
+
kept = kept.filter((y) => y !== x);
|
|
188
287
|
}
|
|
189
|
-
const assigned =
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
const voice = other ? { type: form === 'video' ? 'speaker' : 'reported', name: other.q.speaker } : { type: 'author' };
|
|
196
|
-
const quotes = mine.map(x => ({ text: located[x.k]?.text ?? x.q.text, speaker: x.q.speaker }));
|
|
197
|
-
return { claim: idea.claim, detail: idea.detail, kind: 'argument', voice, quotes };
|
|
288
|
+
const assigned = note.quotes.map(() => null);
|
|
289
|
+
assignQuotes(note.ideas, kept.map((x) => x.q)).forEach((idea, n) => {
|
|
290
|
+
if (idea === null)
|
|
291
|
+
dropped[kept[n].k] = 'overflow';
|
|
292
|
+
else
|
|
293
|
+
assigned[kept[n].k] = idea;
|
|
198
294
|
});
|
|
199
|
-
const
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
295
|
+
const ideas = note.ideas.map((idea, i) => ({
|
|
296
|
+
claim: idea.claim,
|
|
297
|
+
detail: idea.detail,
|
|
298
|
+
quotes: note.quotes.flatMap((q, k) => assigned[k] === i ? [{ text: q.text, ...(q.attribution ? { attribution: q.attribution } : {}) }] : []),
|
|
299
|
+
}));
|
|
300
|
+
const tags = note.tags.filter((t) => t !== 'reading' && !jungleFormatTags.includes(t));
|
|
301
|
+
const gold = {
|
|
302
|
+
schemaVersion: 2,
|
|
303
|
+
form: formOf(note),
|
|
304
|
+
gist: { lead, ...(context ? { context } : {}) },
|
|
305
|
+
ideas,
|
|
306
|
+
tags,
|
|
307
|
+
};
|
|
308
|
+
const issues = [
|
|
309
|
+
...parseDigest(gold).issues,
|
|
310
|
+
...(source === undefined ? [] : checkDigest(gold, source, { tags: jungleTags }).issues),
|
|
311
|
+
];
|
|
312
|
+
return {
|
|
313
|
+
note,
|
|
314
|
+
gold,
|
|
315
|
+
quotes: note.quotes.map((_, k) => ({
|
|
316
|
+
idea: assigned[k] ?? null,
|
|
317
|
+
...(located[k] ? { match: located[k] } : {}),
|
|
318
|
+
...(dropped[k] ? { dropped: dropped[k] } : {}),
|
|
319
|
+
})),
|
|
320
|
+
issues,
|
|
321
|
+
...(source !== undefined && !issues.length ? { digest: gold } : {}),
|
|
322
|
+
};
|
|
221
323
|
}
|
|
@@ -1,33 +1,42 @@
|
|
|
1
1
|
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
-
import { type
|
|
2
|
+
import { type DigestV2 } from '../digest/schema.js';
|
|
3
3
|
import { type VisibilityState } from '../identity/visibility.js';
|
|
4
4
|
import { type JungleEditorial } from './jungle-digest.js';
|
|
5
5
|
/**
|
|
6
6
|
* The library side of `sponge import jungle`: one Jungle Reading note onto the item holding its capture.
|
|
7
7
|
* `planJungleNote` only reads; `applyJungleNote` runs the same plan inside the caller's transaction, through the logged paths.
|
|
8
8
|
*/
|
|
9
|
-
export declare const jungleNamespace = "jungle", jungleRealm = "import:jungle"
|
|
10
|
-
readonly importer: "sponge-jungle-reading";
|
|
11
|
-
readonly version: "1";
|
|
12
|
-
};
|
|
9
|
+
export declare const jungleNamespace = "jungle", jungleRealm = "import:jungle";
|
|
13
10
|
export type EditorialAction = 'new' | 'unchanged' | 'user-review-wins';
|
|
14
11
|
export type DigestAction = 'new' | 'unchanged' | 'not-convertible' | 'not-digest' | 'user-review-wins' | 'user-digest-wins';
|
|
15
12
|
export type VisibilityAction = 'set-public' | 'set-private' | 'restore-default' | 'unchanged' | 'kept-user-edit' | 'guarded' | 'default-policy';
|
|
13
|
+
/** Local captures only (no web URI of their own): `uri` records the note's `source_url`. Later notes on one capture leave it to the first (`other-note`). */
|
|
14
|
+
export type UriAction = 'new' | 'unchanged' | 'web-capture' | 'other-note' | 'no-source-url' | 'capability';
|
|
15
|
+
/** The first note's title is the item's default title (`noteTitle`); `kept-user-edit` records it while the user's override stays shown. */
|
|
16
|
+
export type TitleAction = 'set' | 'unchanged' | 'other-note' | 'kept-user-edit';
|
|
17
|
+
/** `firstNote` is false for a later note on a capture an earlier note in the same run already used; `firstNoteUri` is the URI that note records (a dry run has not written it). */
|
|
16
18
|
export interface JungleNoteInput {
|
|
17
19
|
itemId: string;
|
|
18
20
|
revisionId: string;
|
|
19
21
|
editorial: JungleEditorial;
|
|
20
|
-
|
|
22
|
+
digest?: DigestV2;
|
|
23
|
+
firstNote?: boolean;
|
|
24
|
+
firstNoteUri?: string;
|
|
21
25
|
}
|
|
22
26
|
export interface JunglePlan {
|
|
23
27
|
editorialId: string;
|
|
24
28
|
editorial: EditorialAction;
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
digestId?: string;
|
|
30
|
+
digest: DigestAction;
|
|
31
|
+
uri: UriAction;
|
|
32
|
+
title: TitleAction;
|
|
27
33
|
visibility: VisibilityAction;
|
|
28
34
|
}
|
|
29
|
-
/** What
|
|
30
|
-
export declare function
|
|
35
|
+
/** What a local capture's URI action would be, before or after its item exists. */
|
|
36
|
+
export declare function uriAction(local: boolean, firstNote: boolean, sourceUrl: string, recorded?: string): {
|
|
37
|
+
action: UriAction;
|
|
38
|
+
uri?: string;
|
|
39
|
+
};
|
|
31
40
|
/**
|
|
32
41
|
* Jungle's decision becomes an explicit override only while nobody else has set one: once the latest override was not
|
|
33
42
|
* written by this importer, the user's choice wins. A public decision never lifts the capability-URL or local-file guards.
|
|
@@ -39,7 +48,7 @@ export declare function visibilityAction(state: Pick<VisibilityState, 'access' |
|
|
|
39
48
|
export declare function planJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;
|
|
40
49
|
/**
|
|
41
50
|
* Apply one note; the caller holds the write transaction. A new editorial object is accepted through `review.note` unless
|
|
42
|
-
* the user already rejected or cleared an earlier one; a
|
|
51
|
+
* the user already rejected or cleared an earlier one; a converted Digest v2 is stored and accepted per field, linked to the
|
|
43
52
|
* editorial object by its input hash, unless the user owns a digest of this item or changed an imported digest's review.
|
|
44
53
|
*/
|
|
45
54
|
export declare function applyJungleNote(db: DatabaseSync, input: JungleNoteInput): JunglePlan;
|