@hraness/sponge 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +343 -100
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +15 -6
- package/dist/src/importers/jungle-store.js +122 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +136 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -1,19 +1,26 @@
|
|
|
1
|
+
import { SpongeError } from '../errors.js';
|
|
1
2
|
// Executed only by the pinned Bun runtime. The import is Wordcell's public API.
|
|
2
3
|
const runtime = process.argv[2], bundle = process.argv[3];
|
|
3
4
|
try {
|
|
4
5
|
const { verifyCaptureBundle } = await import(import.meta.resolve('@hraness/wordcell/clip/bundle-reader', `${runtime}/package.json`));
|
|
5
|
-
const result = await verifyCaptureBundle(bundle, {
|
|
6
|
+
const result = await verifyCaptureBundle(bundle, {
|
|
7
|
+
verifyAssets: true,
|
|
8
|
+
maxDocumentBytes: 16 * 1024 ** 2,
|
|
9
|
+
maxVerifiedAssets: 200,
|
|
10
|
+
maxAssetBytes: 20 * 1024 ** 2,
|
|
11
|
+
maxTotalAssetBytes: 500 * 1024 ** 2,
|
|
12
|
+
maxAssetVerificationMs: 30000,
|
|
13
|
+
});
|
|
6
14
|
// The document travels via the verified file, never a giant protocol frame.
|
|
7
15
|
const { markdown: _, ...document } = result.inspection.document;
|
|
8
16
|
// Metadata is validated upstream and read from the confined manifest by the parent.
|
|
9
17
|
const { metadata: _metadata, ...inspection } = result.inspection;
|
|
10
18
|
const output = JSON.stringify({ ...result, inspection: { ...inspection, document } });
|
|
11
19
|
if (Buffer.byteLength(output) > 65536)
|
|
12
|
-
throw
|
|
20
|
+
throw new SpongeError('limit-exceeded');
|
|
13
21
|
process.stdout.write(output);
|
|
14
22
|
}
|
|
15
23
|
catch {
|
|
16
24
|
process.stdout.write(JSON.stringify({ ok: false }));
|
|
17
25
|
process.exitCode = 1;
|
|
18
26
|
}
|
|
19
|
-
export {};
|
|
@@ -10,10 +10,13 @@ const escapable = /^[\\`*_{}[\]()#+\-.!|>~<"']$/;
|
|
|
10
10
|
function fold(raw) {
|
|
11
11
|
let text = '', space = false;
|
|
12
12
|
const map = [], ends = [], clusters = [...graphemes.segment(raw)];
|
|
13
|
-
const emit = (out, start, end) => {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
const emit = (out, start, end) => {
|
|
14
|
+
text += out;
|
|
15
|
+
for (let k = 0; k < out.length; k++) {
|
|
16
|
+
map.push(start);
|
|
17
|
+
ends.push(end);
|
|
18
|
+
}
|
|
19
|
+
};
|
|
17
20
|
for (let c = 0; c < clusters.length; c++) {
|
|
18
21
|
const { segment, index } = clusters[c];
|
|
19
22
|
// An escaped character maps to its backslash, so a match starting there keeps the escape.
|
|
@@ -29,7 +32,11 @@ function fold(raw) {
|
|
|
29
32
|
continue;
|
|
30
33
|
}
|
|
31
34
|
space = false;
|
|
32
|
-
emit(ch
|
|
35
|
+
emit(ch
|
|
36
|
+
.replace(/[‘’‚′]/g, "'")
|
|
37
|
+
.replace(/[“”„″]/g, '"')
|
|
38
|
+
.replace(/[\u2010-\u2015\u2212]/g, '-')
|
|
39
|
+
.normalize('NFC'), index, end);
|
|
33
40
|
}
|
|
34
41
|
if (text.endsWith(' ')) {
|
|
35
42
|
text = text.slice(0, -1);
|
|
@@ -38,7 +45,9 @@ function fold(raw) {
|
|
|
38
45
|
}
|
|
39
46
|
return { text, map, ends };
|
|
40
47
|
}
|
|
41
|
-
export function foldText(value) {
|
|
48
|
+
export function foldText(value) {
|
|
49
|
+
return fold(value).text;
|
|
50
|
+
}
|
|
42
51
|
/** Every occurrence of `quote` in `source`, compared after folding. */
|
|
43
52
|
export function locateAll(source, quote) {
|
|
44
53
|
const s = fold(source), q = fold(quote).text, out = [];
|
|
@@ -47,14 +56,21 @@ export function locateAll(source, quote) {
|
|
|
47
56
|
for (let at = s.text.indexOf(q); at >= 0; at = s.text.indexOf(q, at + 1)) {
|
|
48
57
|
// A match must start and end on cluster boundaries; half a cluster has no raw offsets.
|
|
49
58
|
const last = at + q.length - 1;
|
|
50
|
-
if (at > 0 && s.map[at - 1] === s.map[at] || last + 1 < s.text.length && s.map[last + 1] === s.map[last])
|
|
59
|
+
if ((at > 0 && s.map[at - 1] === s.map[at]) || (last + 1 < s.text.length && s.map[last + 1] === s.map[last]))
|
|
51
60
|
continue;
|
|
52
61
|
out.push({ start: s.map[at], end: s.ends[last] });
|
|
53
62
|
}
|
|
54
63
|
return out;
|
|
55
64
|
}
|
|
56
65
|
export function textQuote(source, start, end) {
|
|
57
|
-
return {
|
|
66
|
+
return {
|
|
67
|
+
type: 'text-quote',
|
|
68
|
+
exact: source.slice(start, end),
|
|
69
|
+
prefix: source.slice(Math.max(0, start - context), start),
|
|
70
|
+
suffix: source.slice(end, end + context),
|
|
71
|
+
start,
|
|
72
|
+
end,
|
|
73
|
+
};
|
|
58
74
|
}
|
|
59
75
|
/** The single match for `quote`, or undefined when absent or ambiguous. */
|
|
60
76
|
export function locateQuote(source, quote) {
|
|
@@ -63,9 +79,16 @@ export function locateQuote(source, quote) {
|
|
|
63
79
|
}
|
|
64
80
|
/** A stored selector still names its exact text in `source`. */
|
|
65
81
|
export function anchorHolds(source, anchor) {
|
|
66
|
-
return Number.isSafeInteger(anchor.start) &&
|
|
82
|
+
return (Number.isSafeInteger(anchor.start) &&
|
|
83
|
+
Number.isSafeInteger(anchor.end) &&
|
|
84
|
+
anchor.start >= 0 &&
|
|
85
|
+
anchor.end > anchor.start &&
|
|
86
|
+
source.slice(anchor.start, anchor.end) === anchor.exact);
|
|
87
|
+
}
|
|
88
|
+
export function words(value) {
|
|
89
|
+
const t = foldText(value);
|
|
90
|
+
return t ? t.split(' ').length : 0;
|
|
67
91
|
}
|
|
68
|
-
export function words(value) { const t = foldText(value); return t ? t.split(' ').length : 0; }
|
|
69
92
|
/**
|
|
70
93
|
* Rendered-text projection for snapping a near-verbatim quote: drops inline Markdown syntax (link targets, brackets,
|
|
71
94
|
* emphasis, code ticks, line-start block markers), folds like `fold`, and lowercases. Offsets map back to raw text.
|
|
@@ -90,7 +113,7 @@ function rendered(raw) {
|
|
|
90
113
|
continue;
|
|
91
114
|
}
|
|
92
115
|
}
|
|
93
|
-
if (ch === '!' && raw[i + 1] === '[' || /[[\]*_`]/.test(ch))
|
|
116
|
+
if ((ch === '!' && raw[i + 1] === '[') || /[[\]*_`]/.test(ch))
|
|
94
117
|
continue;
|
|
95
118
|
if (ch === '\\' && i + 1 < raw.length && /[\\`*_{}[\]()#+\-.!|>~<"']/.test(raw[i + 1])) {
|
|
96
119
|
ch = raw[++i];
|
|
@@ -105,7 +128,12 @@ function rendered(raw) {
|
|
|
105
128
|
continue;
|
|
106
129
|
}
|
|
107
130
|
space = false;
|
|
108
|
-
ch = ch
|
|
131
|
+
ch = ch
|
|
132
|
+
.replace(/[‘’‚′]/g, "'")
|
|
133
|
+
.replace(/[“”„″]/g, '"')
|
|
134
|
+
.replace(/[‐-―−]/g, '-')
|
|
135
|
+
.normalize('NFC')
|
|
136
|
+
.toLowerCase();
|
|
109
137
|
// One entry per output code unit: normalization and case mapping may change the length.
|
|
110
138
|
text += ch;
|
|
111
139
|
for (let k = 0; k < ch.length; k++) {
|
|
@@ -43,7 +43,6 @@ export declare function updateAnnotation(db: DatabaseSync, id: string, body: str
|
|
|
43
43
|
export declare function deleteAnnotation(db: DatabaseSync, id: string): void;
|
|
44
44
|
/** Reading state is one field per item; `updatedAt` travels with it. */
|
|
45
45
|
export declare function setReadingState(db: DatabaseSync, itemId: string, state: unknown): void;
|
|
46
|
-
export declare function readAnnotation(db: DatabaseSync, id: string): Annotation | undefined;
|
|
47
46
|
/** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
|
|
48
47
|
export declare function listAnnotations(db: DatabaseSync, itemId: string): (Annotation & {
|
|
49
48
|
anchored?: boolean;
|
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { anchorHolds, locateAll, textQuote } from './quote.js';
|
|
3
3
|
import { mutate } from '../store/changes.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { text } from '../store/files.js';
|
|
5
|
+
import { sha256 } from '../util/hash.js';
|
|
6
|
+
import { SpongeError } from '../errors.js';
|
|
6
7
|
const now = () => new Date().toISOString();
|
|
7
|
-
const fail = (code = 'invalid-input') => {
|
|
8
|
+
const fail = (code = 'invalid-input') => {
|
|
9
|
+
throw new SpongeError(code);
|
|
10
|
+
};
|
|
8
11
|
export function markdownFor(db, revisionId) {
|
|
9
|
-
const row = db
|
|
12
|
+
const row = db
|
|
13
|
+
.prepare("SELECT markdown FROM representations WHERE revision_id=? AND kind='markdown'")
|
|
14
|
+
.get(revisionId);
|
|
10
15
|
return typeof row?.markdown === 'string' ? row.markdown : fail('unavailable');
|
|
11
16
|
}
|
|
12
17
|
/** Create through the log with the full row image, so a replay rebuilds the same row. */
|
|
13
|
-
function create(db, id, row) {
|
|
18
|
+
function create(db, id, row) {
|
|
19
|
+
mutate(db, 'annotation', id, 'created', row);
|
|
20
|
+
}
|
|
14
21
|
/**
|
|
15
22
|
* Highlight `quote` in the item's current revision. An ambiguous quote needs `occurrence` (0-based),
|
|
16
23
|
* so a highlight never silently lands on the wrong passage.
|
|
@@ -27,7 +34,15 @@ export function addHighlight(db, input) {
|
|
|
27
34
|
if (!Number.isSafeInteger(index) || index < 0 || index >= matches.length)
|
|
28
35
|
fail();
|
|
29
36
|
const anchor = textQuote(source, matches[index].start, matches[index].end), id = randomUUID();
|
|
30
|
-
create(db, id, {
|
|
37
|
+
create(db, id, {
|
|
38
|
+
itemId: input.itemId,
|
|
39
|
+
kind: 'highlight',
|
|
40
|
+
revisionId,
|
|
41
|
+
parentId: null,
|
|
42
|
+
anchor,
|
|
43
|
+
body: input.note ?? '',
|
|
44
|
+
createdAt: now(),
|
|
45
|
+
});
|
|
31
46
|
return readAnnotation(db, id);
|
|
32
47
|
}
|
|
33
48
|
export function addComment(db, input) {
|
|
@@ -40,7 +55,15 @@ export function addComment(db, input) {
|
|
|
40
55
|
fail('unavailable');
|
|
41
56
|
}
|
|
42
57
|
const id = randomUUID();
|
|
43
|
-
create(db, id, {
|
|
58
|
+
create(db, id, {
|
|
59
|
+
itemId: input.itemId,
|
|
60
|
+
kind: 'comment',
|
|
61
|
+
revisionId: null,
|
|
62
|
+
parentId: input.parentId ?? null,
|
|
63
|
+
anchor: null,
|
|
64
|
+
body: input.body,
|
|
65
|
+
createdAt: now(),
|
|
66
|
+
});
|
|
44
67
|
return readAnnotation(db, id);
|
|
45
68
|
}
|
|
46
69
|
/**
|
|
@@ -48,14 +71,23 @@ export function addComment(db, input) {
|
|
|
48
71
|
* once the user edits or deletes it, the user's choice wins and the import skips it.
|
|
49
72
|
*/
|
|
50
73
|
export function importNote(db, input) {
|
|
51
|
-
const id =
|
|
74
|
+
const id = sha256(JSON.stringify(['import-note-v1', input.namespace, input.sourceKey, input.noteId]));
|
|
52
75
|
const prior = db.prepare('SELECT item_id,body,updated_at,deleted_at FROM annotations WHERE id=?').get(id);
|
|
53
76
|
if (prior) {
|
|
54
|
-
if (prior.item_id !== input.itemId ||
|
|
77
|
+
if (prior.item_id !== input.itemId ||
|
|
78
|
+
(prior.body !== input.body && prior.updated_at === null && prior.deleted_at === null))
|
|
55
79
|
fail('integrity');
|
|
56
80
|
return;
|
|
57
81
|
}
|
|
58
|
-
create(db, id, {
|
|
82
|
+
create(db, id, {
|
|
83
|
+
itemId: input.itemId,
|
|
84
|
+
kind: 'note',
|
|
85
|
+
revisionId: input.revisionId,
|
|
86
|
+
parentId: null,
|
|
87
|
+
anchor: { kind: 'imported-note', sourceId: input.noteId },
|
|
88
|
+
body: input.body,
|
|
89
|
+
createdAt: now(),
|
|
90
|
+
});
|
|
59
91
|
}
|
|
60
92
|
export function updateAnnotation(db, id, body) {
|
|
61
93
|
text(body, 16384);
|
|
@@ -75,12 +107,32 @@ export function deleteAnnotation(db, id) {
|
|
|
75
107
|
mutate(db, 'annotation', id, 'deleted', now());
|
|
76
108
|
}
|
|
77
109
|
/** Reading state is one field per item; `updatedAt` travels with it. */
|
|
78
|
-
export function setReadingState(db, itemId, state) {
|
|
79
|
-
|
|
80
|
-
|
|
110
|
+
export function setReadingState(db, itemId, state) {
|
|
111
|
+
mutate(db, 'reading', itemId, 'state', { state, updatedAt: now() });
|
|
112
|
+
}
|
|
113
|
+
function annotationRow(r) {
|
|
114
|
+
return {
|
|
115
|
+
id: String(r.id),
|
|
116
|
+
itemId: String(r.item_id),
|
|
117
|
+
kind: r.kind,
|
|
118
|
+
revisionId: r.revision_id === null ? null : String(r.revision_id),
|
|
119
|
+
parentId: r.parent_id === null ? null : String(r.parent_id),
|
|
120
|
+
anchor: JSON.parse(String(r.anchor_json)),
|
|
121
|
+
body: String(r.body),
|
|
122
|
+
createdAt: String(r.created_at),
|
|
123
|
+
updatedAt: r.updated_at === null ? null : String(r.updated_at),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function readAnnotation(db, id) {
|
|
127
|
+
const r = db.prepare('SELECT * FROM annotations WHERE id=? AND deleted_at IS NULL').get(id);
|
|
128
|
+
return r ? annotationRow(r) : undefined;
|
|
129
|
+
}
|
|
81
130
|
/** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
|
|
82
131
|
export function listAnnotations(db, itemId) {
|
|
83
|
-
return db
|
|
132
|
+
return db
|
|
133
|
+
.prepare('SELECT * FROM annotations WHERE item_id=? AND deleted_at IS NULL ORDER BY created_at,rowid')
|
|
134
|
+
.all(itemId)
|
|
135
|
+
.map((r) => {
|
|
84
136
|
const a = annotationRow(r);
|
|
85
137
|
if (a.kind !== 'highlight' || !a.revisionId)
|
|
86
138
|
return a;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
|
|
2
2
|
import { type CaptureRequest, type CaptureOutcome, type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
|
|
3
3
|
import type { LibraryStore, AttemptToken } from '../store/index.js';
|
|
4
|
-
|
|
4
|
+
interface ArticleResult {
|
|
5
5
|
ok: true;
|
|
6
6
|
itemId: string;
|
|
7
7
|
revisionId: string;
|
|
@@ -21,3 +21,4 @@ export interface ArticleOptions {
|
|
|
21
21
|
}
|
|
22
22
|
/** One attempt. Durable queue policy and process recovery live above this function. */
|
|
23
23
|
export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;
|
|
24
|
+
export {};
|
package/dist/src/app/article.js
CHANGED
|
@@ -1,82 +1,78 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
1
|
import { join } from 'node:path';
|
|
3
2
|
import { validateCapture } from '../../contracts/validate.js';
|
|
4
|
-
import { captureArticle } from '../acquisition/index.js';
|
|
5
|
-
import {
|
|
3
|
+
import { captureArticle, } from '../acquisition/index.js';
|
|
4
|
+
import { runAttempt, classifyErrors } from '../ingest/attempt.js';
|
|
5
|
+
import { sha256 } from '../util/hash.js';
|
|
6
6
|
import { bundleBytes, bundleText } from './bundle-files.js';
|
|
7
7
|
import { selectedMetadata } from '../metadata/article.js';
|
|
8
8
|
import { localizeMarkdown } from './markdown.js';
|
|
9
|
-
|
|
9
|
+
import { SpongeError } from '../errors.js';
|
|
10
|
+
const classify = classifyErrors([
|
|
11
|
+
'integrity',
|
|
12
|
+
'limit-exceeded',
|
|
13
|
+
'cancelled',
|
|
14
|
+
'invalid-input',
|
|
15
|
+
'owner-busy',
|
|
16
|
+
'unavailable',
|
|
17
|
+
]);
|
|
10
18
|
/** One attempt. Durable queue policy and process recovery live above this function. */
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
let cleanup = 'settled';
|
|
15
|
-
try {
|
|
16
|
-
const source = store.sourceForJob(token.jobId);
|
|
19
|
+
export function ingestArticle(store, token, options) {
|
|
20
|
+
return runAttempt(store, token, classify, async (attempt) => {
|
|
21
|
+
const source = store.queue.source(token.jobId);
|
|
17
22
|
if (!source || source.kind !== 'article')
|
|
18
|
-
throw
|
|
19
|
-
const result = await (options.capture ?? captureArticle)({
|
|
23
|
+
throw new SpongeError('invalid-input');
|
|
24
|
+
const result = await (options.capture ?? captureArticle)({
|
|
25
|
+
runProcess: options.runProcess,
|
|
26
|
+
runtime: options.runtime,
|
|
27
|
+
stagingRoot: join(store.root, 'staging'),
|
|
28
|
+
sourceUrl: source.locator,
|
|
29
|
+
requestId: token.attemptId,
|
|
30
|
+
connection: options.connection,
|
|
31
|
+
signal: options.signal,
|
|
32
|
+
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
33
|
+
});
|
|
20
34
|
if (!result.ok) {
|
|
21
|
-
|
|
22
|
-
|
|
35
|
+
if (result.code === 'cleanup-required')
|
|
36
|
+
attempt.cleanup = 'pending';
|
|
37
|
+
return result;
|
|
23
38
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
assets.push({ ...blob, mediaType: asset.mediaType });
|
|
38
|
-
}
|
|
39
|
-
if (options.signal?.aborted) {
|
|
40
|
-
store.requestCancellation(token.jobId);
|
|
41
|
-
throw Object.assign(Error('cancelled'), { code: 'cancelled' });
|
|
42
|
-
}
|
|
43
|
-
const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
|
|
44
|
-
const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
|
|
45
|
-
const status = localized.missingImages ? 'partial' : capture.status;
|
|
46
|
-
// Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
|
|
47
|
-
const bundleHash = createHash('sha256').update(JSON.stringify(['article-local-markdown-v1', capture.manifestSha256])).digest('hex');
|
|
48
|
-
await options.checkpoint?.('promoting');
|
|
49
|
-
const revision = store.commitRevision({ ...token, itemId: source.itemId, bundleHash, markdown: localized.markdown, title, extractor: capture.extractor, status, metadata: capture.metadata, metadataWarnings: capture.metadataWarnings, assets, warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])] });
|
|
50
|
-
committed = true;
|
|
51
|
-
outcome = { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
|
|
39
|
+
attempt.hold(result);
|
|
40
|
+
await options.checkpoint?.('extracting');
|
|
41
|
+
const capture = result.capture;
|
|
42
|
+
validateCapture(capture);
|
|
43
|
+
if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
|
|
44
|
+
throw new SpongeError('integrity');
|
|
45
|
+
const original = await bundleText(result.bundleRoot, capture.markdown), localized = localizeMarkdown(original, capture.assets);
|
|
46
|
+
const assets = [];
|
|
47
|
+
for (const asset of capture.assets) {
|
|
48
|
+
if (options.signal?.aborted)
|
|
49
|
+
throw new SpongeError('cancelled');
|
|
50
|
+
const blob = await store.blobs.put(token, asset.sha256, asset.bytes, bundleBytes(result.bundleRoot, asset));
|
|
51
|
+
assets.push({ ...blob, mediaType: asset.mediaType });
|
|
52
52
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
outcome = failure(token.attemptId, code && known.has(code) ? code : 'unclassified', 'storage');
|
|
57
|
-
}
|
|
58
|
-
if (captured)
|
|
59
|
-
try {
|
|
60
|
-
await captured.dispose();
|
|
53
|
+
if (options.signal?.aborted) {
|
|
54
|
+
store.queue.cancel(token.jobId);
|
|
55
|
+
throw new SpongeError('cancelled');
|
|
61
56
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
57
|
+
const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
|
|
58
|
+
const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
|
|
59
|
+
const status = localized.missingImages ? 'partial' : capture.status;
|
|
60
|
+
// Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
|
|
61
|
+
const bundleHash = sha256(JSON.stringify(['article-local-markdown-v1', capture.manifestSha256]));
|
|
62
|
+
await options.checkpoint?.('promoting');
|
|
63
|
+
const revision = store.revisions.commit({
|
|
64
|
+
...token,
|
|
65
|
+
itemId: source.itemId,
|
|
66
|
+
bundleHash,
|
|
67
|
+
markdown: localized.markdown,
|
|
68
|
+
title,
|
|
69
|
+
extractor: capture.extractor,
|
|
70
|
+
status,
|
|
71
|
+
metadata: capture.metadata,
|
|
72
|
+
metadataWarnings: capture.metadataWarnings,
|
|
73
|
+
assets,
|
|
74
|
+
warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
|
|
75
|
+
});
|
|
76
|
+
return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
|
|
77
|
+
});
|
|
82
78
|
}
|
|
@@ -1,8 +1,4 @@
|
|
|
1
1
|
import { type Asset } from '../../contracts/index.js';
|
|
2
|
-
export declare class BundleIntegrityError extends Error {
|
|
3
|
-
readonly code = "integrity";
|
|
4
|
-
constructor();
|
|
5
|
-
}
|
|
6
2
|
/** Stream only the verified regular inode; parent path checks never follow symlinks. */
|
|
7
3
|
export declare function bundleBytes(root: string, asset: Asset): AsyncGenerator<Uint8Array>;
|
|
8
4
|
export declare function bundleText(root: string, asset: Asset): Promise<string>;
|
|
@@ -4,32 +4,29 @@ import { createHash } from 'node:crypto';
|
|
|
4
4
|
import { dirname, join, isAbsolute, relative, sep } from 'node:path';
|
|
5
5
|
import { limits } from '../../contracts/index.js';
|
|
6
6
|
import { validateAsset } from '../../contracts/validate.js';
|
|
7
|
-
|
|
8
|
-
code = 'integrity';
|
|
9
|
-
constructor() { super('integrity'); }
|
|
10
|
-
}
|
|
7
|
+
import { SpongeError } from '../errors.js';
|
|
11
8
|
/** Stream only the verified regular inode; parent path checks never follow symlinks. */
|
|
12
9
|
export async function* bundleBytes(root, asset) {
|
|
13
10
|
validateAsset(asset);
|
|
14
11
|
if (asset.bytes > limits.pdfBytes)
|
|
15
|
-
throw new
|
|
16
|
-
if (!isAbsolute(root) || await realpath(root) !== root)
|
|
17
|
-
throw new
|
|
12
|
+
throw new SpongeError('integrity');
|
|
13
|
+
if (!isAbsolute(root) || (await realpath(root)) !== root)
|
|
14
|
+
throw new SpongeError('integrity');
|
|
18
15
|
const path = join(root, asset.relativePath), rel = relative(root, path);
|
|
19
16
|
if (!rel || rel.startsWith('..' + sep) || isAbsolute(rel))
|
|
20
|
-
throw new
|
|
17
|
+
throw new SpongeError('integrity');
|
|
21
18
|
let parent = root;
|
|
22
19
|
for (const part of relative(root, dirname(path)).split(sep).filter(Boolean)) {
|
|
23
20
|
parent = join(parent, part);
|
|
24
21
|
const stat = await lstat(parent);
|
|
25
22
|
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
26
|
-
throw new
|
|
23
|
+
throw new SpongeError('integrity');
|
|
27
24
|
}
|
|
28
25
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
29
26
|
try {
|
|
30
27
|
const stat = await file.stat();
|
|
31
28
|
if (!stat.isFile() || stat.nlink !== 1 || stat.size !== asset.bytes)
|
|
32
|
-
throw new
|
|
29
|
+
throw new SpongeError('integrity');
|
|
33
30
|
const digest = createHash('sha256');
|
|
34
31
|
let count = 0;
|
|
35
32
|
for (;;) {
|
|
@@ -39,13 +36,13 @@ export async function* bundleBytes(root, asset) {
|
|
|
39
36
|
break;
|
|
40
37
|
count += bytesRead;
|
|
41
38
|
if (count > asset.bytes)
|
|
42
|
-
throw new
|
|
39
|
+
throw new SpongeError('integrity');
|
|
43
40
|
const bytes = buffer.subarray(0, bytesRead);
|
|
44
41
|
digest.update(bytes);
|
|
45
42
|
yield bytes;
|
|
46
43
|
}
|
|
47
44
|
if (count !== asset.bytes || digest.digest('hex') !== asset.sha256)
|
|
48
|
-
throw new
|
|
45
|
+
throw new SpongeError('integrity');
|
|
49
46
|
}
|
|
50
47
|
finally {
|
|
51
48
|
await file.close();
|
|
@@ -53,7 +50,7 @@ export async function* bundleBytes(root, asset) {
|
|
|
53
50
|
}
|
|
54
51
|
export async function bundleText(root, asset) {
|
|
55
52
|
if (asset.mediaType !== 'text/markdown' || asset.bytes > limits.markdownBytes)
|
|
56
|
-
throw new
|
|
53
|
+
throw new SpongeError('integrity');
|
|
57
54
|
const chunks = [];
|
|
58
55
|
for await (const chunk of bundleBytes(root, asset))
|
|
59
56
|
chunks.push(chunk);
|
|
@@ -61,6 +58,6 @@ export async function bundleText(root, asset) {
|
|
|
61
58
|
return new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks));
|
|
62
59
|
}
|
|
63
60
|
catch {
|
|
64
|
-
throw new
|
|
61
|
+
throw new SpongeError('integrity');
|
|
65
62
|
}
|
|
66
63
|
}
|
package/dist/src/app/markdown.js
CHANGED
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import { fromMarkdown } from 'mdast-util-from-markdown';
|
|
2
2
|
import { toMarkdown } from 'mdast-util-to-markdown';
|
|
3
|
+
import { SpongeError } from '../errors.js';
|
|
3
4
|
const assetPrefix = 'sponge-asset:';
|
|
4
5
|
function localKey(url) {
|
|
5
6
|
try {
|
|
6
7
|
const key = decodeURIComponent(url).replace(/^\.\//u, '');
|
|
7
|
-
return key.split('/').some(p => p === '..' || p === '') || key.includes('\\') || key.includes(':') ? null : key;
|
|
8
|
+
return key.split('/').some((p) => p === '..' || p === '') || key.includes('\\') || key.includes(':') ? null : key;
|
|
8
9
|
}
|
|
9
10
|
catch {
|
|
10
11
|
return null;
|
|
11
12
|
}
|
|
12
13
|
}
|
|
13
|
-
function text(value) {
|
|
14
|
+
function text(value) {
|
|
15
|
+
return toMarkdown({ type: 'text', value }).trimEnd();
|
|
16
|
+
}
|
|
14
17
|
/** Preserve original formatting/frontmatter; edit parsed destination spans, never regex Markdown. */
|
|
15
18
|
export function localizeMarkdown(markdown, assets) {
|
|
16
19
|
const frontmatter = /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0];
|
|
@@ -18,7 +21,7 @@ export function localizeMarkdown(markdown, assets) {
|
|
|
18
21
|
const body = localizeMarkdown(markdown.slice(frontmatter.length), assets);
|
|
19
22
|
return { ...body, markdown: frontmatter + body.markdown };
|
|
20
23
|
}
|
|
21
|
-
const tree = fromMarkdown(markdown), byPath = new Map(assets.map(a => [a.relativePath, a.sha256]));
|
|
24
|
+
const tree = fromMarkdown(markdown), byPath = new Map(assets.map((a) => [a.relativePath, a.sha256]));
|
|
22
25
|
const definitions = new Map();
|
|
23
26
|
const patches = [];
|
|
24
27
|
let missingImages = false;
|
|
@@ -33,7 +36,7 @@ export function localizeMarkdown(markdown, assets) {
|
|
|
33
36
|
function replace(node, value) {
|
|
34
37
|
const start = node.position?.start.offset, end = node.position?.end.offset;
|
|
35
38
|
if (start === undefined || end === undefined)
|
|
36
|
-
throw
|
|
39
|
+
throw new SpongeError('integrity');
|
|
37
40
|
patches.push({ start, end, value });
|
|
38
41
|
}
|
|
39
42
|
function walk(node) {
|
|
@@ -41,7 +44,12 @@ export function localizeMarkdown(markdown, assets) {
|
|
|
41
44
|
const url = node.type === 'image' ? node.url : definitions.get(node.identifier)?.url;
|
|
42
45
|
const key = url ? localKey(url) : null, sha = key ? byPath.get(key) : undefined;
|
|
43
46
|
if (sha) {
|
|
44
|
-
const image = {
|
|
47
|
+
const image = {
|
|
48
|
+
type: 'image',
|
|
49
|
+
url: assetPrefix + sha,
|
|
50
|
+
alt: node.alt ?? '',
|
|
51
|
+
title: node.type === 'image' ? node.title : definitions.get(node.identifier)?.title,
|
|
52
|
+
};
|
|
45
53
|
replace(node, toMarkdown(image).trimEnd());
|
|
46
54
|
Object.assign(node, image);
|
|
47
55
|
}
|
|
@@ -73,7 +81,11 @@ export function localizeMarkdown(markdown, assets) {
|
|
|
73
81
|
if (node.type === 'definition')
|
|
74
82
|
replace(node, '');
|
|
75
83
|
else {
|
|
76
|
-
const plain = (n) => 'value' in n
|
|
84
|
+
const plain = (n) => 'value' in n
|
|
85
|
+
? String(n.value)
|
|
86
|
+
: 'children' in n
|
|
87
|
+
? n.children.map((c) => plain(c)).join('')
|
|
88
|
+
: '';
|
|
77
89
|
replace(node, text(plain(node)));
|
|
78
90
|
}
|
|
79
91
|
return;
|
package/dist/src/app/source.d.ts
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
export declare class InputError extends Error {
|
|
2
|
-
readonly code = "invalid-input";
|
|
3
|
-
constructor();
|
|
4
|
-
}
|
|
5
1
|
/** The identity form of an http(s) locator; source keys and shareable URIs both use it. Its output pins every existing source key. */
|
|
6
2
|
export declare function normalizeWebLocator(locator: string): string;
|
|
7
3
|
export declare function articleSource(locator: string, realm?: string): {
|