@hraness/sponge 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -10
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +39 -17
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +10 -46
- package/dist/src/digest/editorial.js +73 -59
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +53 -32
- package/dist/src/digest/pipeline.js +357 -125
- package/dist/src/digest/schema.d.ts +44 -17
- package/dist/src/digest/schema.js +132 -44
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +31 -24
- package/dist/src/importers/jungle-digest.js +217 -115
- package/dist/src/importers/jungle-store.d.ts +20 -11
- package/dist/src/importers/jungle-store.js +136 -41
- package/dist/src/importers/jungle.d.ts +32 -16
- package/dist/src/importers/jungle.js +145 -32
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +108 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -77
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -1,38 +1,56 @@
|
|
|
1
1
|
import { type TextQuote } from '../annotations/quote.js';
|
|
2
2
|
/**
|
|
3
|
-
* Digest
|
|
4
|
-
*
|
|
3
|
+
* Digest v2: a gist led by one sentence, and 3-5 ideas in the author's order, each a claim with detail and 0-2 verbatim
|
|
4
|
+
* quotes; the digest carries 1-6 quotes in all. Digest v1 (one short lead, 1-2 quotes on every idea, ideas in source
|
|
5
|
+
* order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
|
|
6
|
+
* decides. Nothing here calls a model.
|
|
5
7
|
*/
|
|
6
|
-
|
|
8
|
+
declare const ideaKinds: readonly ["finding", "argument", "method", "evidence", "recommendation", "counterpoint"];
|
|
7
9
|
export declare const forms: readonly ["article", "video", "paper", "thread", "pdf"];
|
|
8
|
-
|
|
10
|
+
type Voice = {
|
|
9
11
|
type: 'author';
|
|
10
12
|
} | {
|
|
11
13
|
type: 'speaker' | 'reported';
|
|
12
14
|
name: string;
|
|
13
15
|
};
|
|
14
|
-
|
|
16
|
+
type QuoteLocator = {
|
|
15
17
|
type: 'timestamp';
|
|
16
18
|
ms: number;
|
|
17
19
|
} | {
|
|
18
20
|
type: 'page';
|
|
19
21
|
page: number;
|
|
20
22
|
};
|
|
23
|
+
/** `attribution` is v2 only: who said it and where, as the reviewer wrote it. */
|
|
21
24
|
export interface DigestQuote {
|
|
22
25
|
text: string;
|
|
23
26
|
speaker?: string;
|
|
27
|
+
attribution?: string;
|
|
24
28
|
locator?: QuoteLocator;
|
|
25
29
|
}
|
|
26
|
-
|
|
30
|
+
interface DigestIdea {
|
|
27
31
|
claim: string;
|
|
28
32
|
detail: string;
|
|
29
|
-
kind
|
|
30
|
-
voice
|
|
33
|
+
kind?: (typeof ideaKinds)[number];
|
|
34
|
+
voice?: Voice;
|
|
31
35
|
quotes: DigestQuote[];
|
|
32
36
|
}
|
|
37
|
+
interface DigestIdeaV1 extends DigestIdea {
|
|
38
|
+
kind: (typeof ideaKinds)[number];
|
|
39
|
+
voice: Voice;
|
|
40
|
+
}
|
|
33
41
|
export interface DigestV1 {
|
|
34
42
|
schemaVersion: 1;
|
|
35
|
-
form: typeof forms[number];
|
|
43
|
+
form: (typeof forms)[number];
|
|
44
|
+
gist: {
|
|
45
|
+
lead: string;
|
|
46
|
+
context?: string;
|
|
47
|
+
};
|
|
48
|
+
ideas: DigestIdeaV1[];
|
|
49
|
+
tags: string[];
|
|
50
|
+
}
|
|
51
|
+
export interface DigestV2 {
|
|
52
|
+
schemaVersion: 2;
|
|
53
|
+
form: (typeof forms)[number];
|
|
36
54
|
gist: {
|
|
37
55
|
lead: string;
|
|
38
56
|
context?: string;
|
|
@@ -40,17 +58,25 @@ export interface DigestV1 {
|
|
|
40
58
|
ideas: DigestIdea[];
|
|
41
59
|
tags: string[];
|
|
42
60
|
}
|
|
61
|
+
export type Digest = DigestV1 | DigestV2;
|
|
62
|
+
export declare const digestSchemaVersion = 2;
|
|
63
|
+
/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
|
|
43
64
|
export declare const digestLimits: {
|
|
44
|
-
readonly leadChars:
|
|
45
|
-
readonly contextWords:
|
|
65
|
+
readonly leadChars: 240;
|
|
66
|
+
readonly contextWords: 80;
|
|
67
|
+
readonly gistWords: 100;
|
|
68
|
+
readonly gistSentences: 5;
|
|
46
69
|
readonly minIdeas: 3;
|
|
47
70
|
readonly maxIdeas: 5;
|
|
48
71
|
readonly claimWords: 20;
|
|
49
72
|
readonly detailWords: 60;
|
|
50
73
|
readonly detailSentences: 3;
|
|
51
74
|
readonly maxQuotesPerIdea: 2;
|
|
75
|
+
readonly minQuotes: 1;
|
|
76
|
+
readonly maxQuotes: 6;
|
|
52
77
|
readonly minQuoteWords: 4;
|
|
53
78
|
readonly quoteWords: 25;
|
|
79
|
+
readonly attributionWords: 25;
|
|
54
80
|
readonly totalQuoteWords: 150;
|
|
55
81
|
readonly sourceQuoteShare: 0.1;
|
|
56
82
|
readonly maxTags: 3;
|
|
@@ -59,7 +85,7 @@ export interface DigestIssue {
|
|
|
59
85
|
path: string;
|
|
60
86
|
code: string;
|
|
61
87
|
}
|
|
62
|
-
|
|
88
|
+
interface LocatedQuote {
|
|
63
89
|
idea: number;
|
|
64
90
|
quote: number;
|
|
65
91
|
anchor: TextQuote;
|
|
@@ -70,16 +96,17 @@ export interface DigestCheck {
|
|
|
70
96
|
quotes: LocatedQuote[];
|
|
71
97
|
}
|
|
72
98
|
export interface DigestOptions {
|
|
73
|
-
/** Controlled tag vocabulary; omitted means any 1-3
|
|
99
|
+
/** Controlled tag vocabulary; omitted means any lowercase kebab-case tags (1-3 in v1, 0-3 in v2). */
|
|
74
100
|
tags?: readonly string[];
|
|
75
101
|
/** Optional semantic check that a quote supports its claim, e.g. embedding cosine. */
|
|
76
102
|
supports?: (claim: string, quote: string) => number;
|
|
77
103
|
supportThreshold?: number;
|
|
78
104
|
}
|
|
79
|
-
/** Parse an untrusted
|
|
105
|
+
/** Parse an untrusted digest (v1 or v2) into its shape, or report why not. */
|
|
80
106
|
export declare function parseDigest(value: unknown): {
|
|
81
|
-
digest?:
|
|
107
|
+
digest?: Digest;
|
|
82
108
|
issues: DigestIssue[];
|
|
83
109
|
};
|
|
84
|
-
/** Check every
|
|
85
|
-
export declare function checkDigest(digest:
|
|
110
|
+
/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
|
|
111
|
+
export declare function checkDigest(digest: Digest, source: string, options?: DigestOptions): DigestCheck;
|
|
112
|
+
export {};
|
|
@@ -1,92 +1,173 @@
|
|
|
1
1
|
import { foldText, locateAll, textQuote, words } from '../annotations/quote.js';
|
|
2
2
|
/**
|
|
3
|
-
* Digest
|
|
4
|
-
*
|
|
3
|
+
* Digest v2: a gist led by one sentence, and 3-5 ideas in the author's order, each a claim with detail and 0-2 verbatim
|
|
4
|
+
* quotes; the digest carries 1-6 quotes in all. Digest v1 (one short lead, 1-2 quotes on every idea, ideas in source
|
|
5
|
+
* order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
|
|
6
|
+
* decides. Nothing here calls a model.
|
|
5
7
|
*/
|
|
6
|
-
|
|
8
|
+
const ideaKinds = ['finding', 'argument', 'method', 'evidence', 'recommendation', 'counterpoint'];
|
|
7
9
|
export const forms = ['article', 'video', 'paper', 'thread', 'pdf'];
|
|
8
|
-
export const
|
|
10
|
+
export const digestSchemaVersion = 2;
|
|
11
|
+
const digestLimitsV1 = {
|
|
12
|
+
leadChars: 155,
|
|
13
|
+
contextWords: 50,
|
|
14
|
+
minIdeas: 3,
|
|
15
|
+
maxIdeas: 5,
|
|
16
|
+
claimWords: 20,
|
|
17
|
+
detailWords: 60,
|
|
18
|
+
detailSentences: 3,
|
|
19
|
+
maxQuotesPerIdea: 2,
|
|
20
|
+
minQuoteWords: 4,
|
|
21
|
+
quoteWords: 25,
|
|
22
|
+
totalQuoteWords: 150,
|
|
23
|
+
sourceQuoteShare: 0.1,
|
|
24
|
+
maxTags: 3,
|
|
25
|
+
};
|
|
26
|
+
/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
|
|
27
|
+
export const digestLimits = {
|
|
28
|
+
leadChars: 240,
|
|
29
|
+
contextWords: 80,
|
|
30
|
+
gistWords: 100,
|
|
31
|
+
gistSentences: 5,
|
|
32
|
+
minIdeas: 3,
|
|
33
|
+
maxIdeas: 5,
|
|
34
|
+
claimWords: 20,
|
|
35
|
+
detailWords: 60,
|
|
36
|
+
detailSentences: 3,
|
|
37
|
+
maxQuotesPerIdea: 2,
|
|
38
|
+
minQuotes: 1,
|
|
39
|
+
maxQuotes: 6,
|
|
40
|
+
minQuoteWords: 4,
|
|
41
|
+
quoteWords: 25,
|
|
42
|
+
attributionWords: 25,
|
|
43
|
+
totalQuoteWords: 150,
|
|
44
|
+
sourceQuoteShare: 0.1,
|
|
45
|
+
maxTags: 3,
|
|
46
|
+
};
|
|
9
47
|
const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
10
|
-
const keys = (v, allowed) => Object.keys(v).every(k => allowed.includes(k));
|
|
48
|
+
const keys = (v, allowed) => Object.keys(v).every((k) => allowed.includes(k));
|
|
11
49
|
const str = (v, max = 2000) => typeof v === 'string' && v.trim().length > 0 && v.length <= max && !/[\u0000-\u0008\u000b-\u001f\u007f]/.test(v);
|
|
12
|
-
/**
|
|
13
|
-
const
|
|
14
|
-
/**
|
|
50
|
+
/** v1: sentence terminators followed by more text; abbreviations are rare enough in a 155-character lead to reject. */
|
|
51
|
+
const sentencesV1 = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
|
|
52
|
+
/** v2: a terminator (and closing quotes) followed by a capital or digit, the same split Jungle's first-sentence lead uses, so `e.g. this` stays one sentence. */
|
|
53
|
+
const sentences = (v) => foldText(v).split(/(?<=[.!?]["')]*)\s+(?=["(]?[A-Z0-9])/).length;
|
|
54
|
+
/** Parse an untrusted digest (v1 or v2) into its shape, or report why not. */
|
|
15
55
|
export function parseDigest(value) {
|
|
16
|
-
const issues = [], bad = (path, code = 'shape') => {
|
|
17
|
-
|
|
56
|
+
const issues = [], bad = (path, code = 'shape') => {
|
|
57
|
+
issues.push({ path, code });
|
|
58
|
+
};
|
|
59
|
+
if (!object(value) ||
|
|
60
|
+
!keys(value, ['schemaVersion', 'form', 'gist', 'ideas', 'tags']) ||
|
|
61
|
+
(value.schemaVersion !== 1 && value.schemaVersion !== 2))
|
|
18
62
|
return { issues: [{ path: '', code: 'shape' }] };
|
|
63
|
+
const v2 = value.schemaVersion === 2;
|
|
19
64
|
if (!forms.includes(value.form))
|
|
20
65
|
bad('form');
|
|
21
66
|
const gist = value.gist;
|
|
22
|
-
if (!object(gist) ||
|
|
67
|
+
if (!object(gist) ||
|
|
68
|
+
!keys(gist, ['lead', 'context']) ||
|
|
69
|
+
!str(gist.lead) ||
|
|
70
|
+
(gist.context !== undefined && !str(gist.context)))
|
|
23
71
|
bad('gist');
|
|
24
72
|
if (!Array.isArray(value.ideas) || value.ideas.length > 8)
|
|
25
73
|
bad('ideas');
|
|
26
74
|
else
|
|
27
75
|
value.ideas.forEach((idea, i) => {
|
|
28
76
|
const at = `ideas.${i}`;
|
|
29
|
-
if (!object(idea) ||
|
|
77
|
+
if (!object(idea) ||
|
|
78
|
+
!keys(idea, ['claim', 'detail', 'kind', 'voice', 'quotes']) ||
|
|
79
|
+
!str(idea.claim) ||
|
|
80
|
+
!str(idea.detail) ||
|
|
81
|
+
!((v2 && idea.kind === undefined) || ideaKinds.includes(idea.kind)))
|
|
30
82
|
return bad(at);
|
|
31
83
|
const voice = idea.voice;
|
|
32
|
-
if (!
|
|
84
|
+
if (!(v2 && voice === undefined) &&
|
|
85
|
+
!(object(voice) &&
|
|
86
|
+
((voice.type === 'author' && keys(voice, ['type'])) ||
|
|
87
|
+
((voice.type === 'speaker' || voice.type === 'reported') &&
|
|
88
|
+
keys(voice, ['type', 'name']) &&
|
|
89
|
+
str(voice.name, 200)))))
|
|
33
90
|
bad(at + '.voice');
|
|
34
91
|
if (!Array.isArray(idea.quotes) || idea.quotes.length > 4)
|
|
35
92
|
return bad(at + '.quotes');
|
|
36
93
|
idea.quotes.forEach((quote, j) => {
|
|
37
|
-
if (!object(quote) ||
|
|
94
|
+
if (!object(quote) ||
|
|
95
|
+
!keys(quote, v2 ? ['text', 'speaker', 'attribution', 'locator'] : ['text', 'speaker', 'locator']) ||
|
|
96
|
+
!str(quote.text, 600) ||
|
|
97
|
+
(quote.speaker !== undefined && !str(quote.speaker, 200)) ||
|
|
98
|
+
(quote.attribution !== undefined && !str(quote.attribution, 600)))
|
|
38
99
|
return bad(`${at}.quotes.${j}`);
|
|
39
100
|
const loc = quote.locator;
|
|
40
|
-
if (loc !== undefined &&
|
|
101
|
+
if (loc !== undefined &&
|
|
102
|
+
!(object(loc) &&
|
|
103
|
+
((loc.type === 'timestamp' &&
|
|
104
|
+
keys(loc, ['type', 'ms']) &&
|
|
105
|
+
Number.isSafeInteger(loc.ms) &&
|
|
106
|
+
loc.ms >= 0) ||
|
|
107
|
+
(loc.type === 'page' &&
|
|
108
|
+
keys(loc, ['type', 'page']) &&
|
|
109
|
+
Number.isSafeInteger(loc.page) &&
|
|
110
|
+
loc.page >= 1))))
|
|
41
111
|
bad(`${at}.quotes.${j}.locator`);
|
|
42
112
|
});
|
|
43
113
|
});
|
|
44
|
-
if (!Array.isArray(value.tags) || value.tags.some(t => !str(t, 64)))
|
|
114
|
+
if (!Array.isArray(value.tags) || value.tags.some((t) => !str(t, 64)))
|
|
45
115
|
bad('tags');
|
|
46
116
|
return issues.length ? { issues } : { digest: value, issues };
|
|
47
117
|
}
|
|
48
|
-
/** Check every
|
|
118
|
+
/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
|
|
49
119
|
export function checkDigest(digest, source, options = {}) {
|
|
50
|
-
const issues = [], located = [], issue = (path, code) => {
|
|
51
|
-
|
|
120
|
+
const issues = [], located = [], issue = (path, code) => {
|
|
121
|
+
issues.push({ path, code });
|
|
122
|
+
}, v2 = digest.schemaVersion === 2, L = v2 ? digestLimits : digestLimitsV1, count = v2 ? sentences : sentencesV1;
|
|
123
|
+
const { lead, context } = digest.gist;
|
|
124
|
+
if (lead.length > L.leadChars)
|
|
52
125
|
issue('gist.lead', 'too-long');
|
|
53
|
-
if (
|
|
126
|
+
if (count(lead) !== 1)
|
|
54
127
|
issue('gist.lead', 'one-sentence');
|
|
55
|
-
if (
|
|
128
|
+
if (context !== undefined && words(context) > L.contextWords)
|
|
56
129
|
issue('gist.context', 'too-long');
|
|
130
|
+
if (v2 && words(lead) + (context === undefined ? 0 : words(context)) > digestLimits.gistWords)
|
|
131
|
+
issue('gist', 'too-long');
|
|
132
|
+
if (v2 && count(lead) + (context === undefined ? 0 : count(context)) > digestLimits.gistSentences)
|
|
133
|
+
issue('gist', 'too-many-sentences');
|
|
57
134
|
if (digest.ideas.length < L.minIdeas || digest.ideas.length > L.maxIdeas)
|
|
58
135
|
issue('ideas', 'count');
|
|
59
136
|
const claims = new Set();
|
|
60
|
-
let quoteWords = 0;
|
|
137
|
+
let quoteWords = 0, quoteCount = 0;
|
|
61
138
|
digest.ideas.forEach((idea, i) => {
|
|
62
139
|
const at = `ideas.${i}`;
|
|
63
140
|
if (words(idea.claim) > L.claimWords)
|
|
64
141
|
issue(at + '.claim', 'too-long');
|
|
65
|
-
if (
|
|
142
|
+
if (count(idea.claim) !== 1)
|
|
66
143
|
issue(at + '.claim', 'one-sentence');
|
|
67
144
|
if (claims.has(foldText(idea.claim).toLowerCase()))
|
|
68
145
|
issue(at + '.claim', 'duplicate');
|
|
69
146
|
claims.add(foldText(idea.claim).toLowerCase());
|
|
70
147
|
if (words(idea.detail) > L.detailWords)
|
|
71
148
|
issue(at + '.detail', 'too-long');
|
|
72
|
-
if (
|
|
149
|
+
if (count(idea.detail) > L.detailSentences)
|
|
73
150
|
issue(at + '.detail', 'too-many-sentences');
|
|
74
|
-
|
|
151
|
+
// v1 needs a quote on every idea; v2 allows quoteless ideas and counts quotes per digest below.
|
|
152
|
+
if (idea.quotes.length < (v2 ? 0 : 1) || idea.quotes.length > L.maxQuotesPerIdea)
|
|
75
153
|
issue(at + '.quotes', 'count');
|
|
76
154
|
idea.quotes.forEach((quote, j) => {
|
|
77
|
-
const path = `${at}.quotes.${j}`,
|
|
78
|
-
quoteWords +=
|
|
79
|
-
|
|
155
|
+
const path = `${at}.quotes.${j}`, n = words(quote.text);
|
|
156
|
+
quoteWords += n;
|
|
157
|
+
quoteCount++;
|
|
158
|
+
if (n > L.quoteWords)
|
|
80
159
|
issue(path, 'too-long');
|
|
81
|
-
if (
|
|
160
|
+
if (n < L.minQuoteWords)
|
|
82
161
|
issue(path, 'too-short');
|
|
83
|
-
if (
|
|
162
|
+
if (quote.attribution !== undefined && words(quote.attribution) > digestLimits.attributionWords)
|
|
163
|
+
issue(path + '.attribution', 'too-long');
|
|
164
|
+
if (idea.voice && idea.voice.type !== 'author' && !quote.speaker)
|
|
84
165
|
issue(path, 'speaker-required');
|
|
85
166
|
const matches = locateAll(source, quote.text);
|
|
86
167
|
if (!matches.length)
|
|
87
168
|
return issue(path, 'not-verbatim');
|
|
88
169
|
// Prefer the first occurrence not already claimed by an earlier quote.
|
|
89
|
-
const match = matches.find(m => !located.some(q => m.start < q.anchor.end && q.anchor.start < m.end));
|
|
170
|
+
const match = matches.find((m) => !located.some((q) => m.start < q.anchor.end && q.anchor.start < m.end));
|
|
90
171
|
if (!match)
|
|
91
172
|
return issue(path, 'overlap');
|
|
92
173
|
located.push({ idea: i, quote: j, anchor: textQuote(source, match.start, match.end) });
|
|
@@ -94,25 +175,32 @@ export function checkDigest(digest, source, options = {}) {
|
|
|
94
175
|
issue(path, 'unsupported');
|
|
95
176
|
});
|
|
96
177
|
});
|
|
97
|
-
|
|
178
|
+
if (v2 && (quoteCount < digestLimits.minQuotes || quoteCount > digestLimits.maxQuotes))
|
|
179
|
+
issue('ideas', 'quote-count');
|
|
180
|
+
// v1 only: ideas follow the source, each idea's earliest located quote starting no earlier than the previous idea's.
|
|
98
181
|
let previous = -1;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
182
|
+
if (!v2)
|
|
183
|
+
digest.ideas.forEach((_, i) => {
|
|
184
|
+
const starts = located.filter((q) => q.idea === i).map((q) => q.anchor.start);
|
|
185
|
+
if (!starts.length)
|
|
186
|
+
return;
|
|
187
|
+
const first = Math.min(...starts);
|
|
188
|
+
if (first < previous)
|
|
189
|
+
issue(`ideas.${i}`, 'out-of-order');
|
|
190
|
+
previous = first;
|
|
191
|
+
});
|
|
108
192
|
if (quoteWords > L.totalQuoteWords)
|
|
109
193
|
issue('ideas', 'quote-budget');
|
|
110
194
|
// The 10% share has a 25-word floor: an exemption so a short source can still carry one full quote.
|
|
111
195
|
if (quoteWords > Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)))
|
|
112
196
|
issue('ideas', 'quote-share');
|
|
113
|
-
if (digest.tags.length <
|
|
197
|
+
if (digest.tags.length < (v2 ? 0 : 1) ||
|
|
198
|
+
digest.tags.length > L.maxTags ||
|
|
199
|
+
new Set(digest.tags).size !== digest.tags.length)
|
|
114
200
|
issue('tags', 'count');
|
|
115
|
-
digest.tags.forEach((tag, i) => {
|
|
116
|
-
|
|
201
|
+
digest.tags.forEach((tag, i) => {
|
|
202
|
+
if (options.tags ? !options.tags.includes(tag) : !/^[a-z0-9][a-z0-9-]{0,63}$/.test(tag))
|
|
203
|
+
issue(`tags.${i}`, 'vocabulary');
|
|
204
|
+
});
|
|
117
205
|
return { ok: issues.length === 0, issues, quotes: located };
|
|
118
206
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { ErrorCode } from '../contracts/index.js';
|
|
2
|
+
export type { ErrorCode };
|
|
3
|
+
/** Public codes: the frozen v1 ErrorEnvelope vocabulary. Exit statuses and hints are keyed by these. */
|
|
4
|
+
export declare const publicCodes: readonly ["grant-permission", "repair-auth", "unsupported", "unavailable", "rate-limited", "partial", "cleanup-required", "cancelled", "invalid-input", "unclassified", "integrity", "limit-exceeded", "runtime-missing", "schema-newer", "owner-busy"];
|
|
5
|
+
/** Detail codes narrow a public code. They are stored as job reasons, matched by callers, and printed as `reason`. */
|
|
6
|
+
declare const parents: {
|
|
7
|
+
readonly 'invalid-index': "invalid-input";
|
|
8
|
+
readonly 'stale-index': "unavailable";
|
|
9
|
+
readonly 'incomplete-index': "unavailable";
|
|
10
|
+
readonly 'cancelled-index': "unavailable";
|
|
11
|
+
readonly 'invalid-search-query': "invalid-input";
|
|
12
|
+
readonly 'invalid-search-vector': "invalid-input";
|
|
13
|
+
readonly 'invalid-search-index': "integrity";
|
|
14
|
+
readonly 'search-generation-missing': "unavailable";
|
|
15
|
+
readonly 'unsupported-vector-runtime': "unsupported";
|
|
16
|
+
readonly 'not-digest': "unsupported";
|
|
17
|
+
readonly malformed: "invalid-input";
|
|
18
|
+
readonly 'no-capture': "invalid-input";
|
|
19
|
+
};
|
|
20
|
+
export type DetailCode = keyof typeof parents;
|
|
21
|
+
export type SpongeCode = ErrorCode | DetailCode;
|
|
22
|
+
export declare const indexCodes: readonly DetailCode[];
|
|
23
|
+
export declare const isSpongeCode: (value: unknown) => value is SpongeCode;
|
|
24
|
+
/** The one code→exit-status table (contracts `exitCodes`: 0 success, 1 failure, 2 invalid input, 3 partial, 4 action required, 130 cancelled). */
|
|
25
|
+
export declare const exitStatus: Readonly<Record<ErrorCode, number>>;
|
|
26
|
+
/** The one code→user-facing hint table. Fixed text only: never paths, sources, credentials or upstream prose. */
|
|
27
|
+
export declare const hints: Readonly<Record<ErrorCode, string>>;
|
|
28
|
+
export interface SpongeErrorOptions {
|
|
29
|
+
detail?: Record<string, unknown>;
|
|
30
|
+
cause?: unknown;
|
|
31
|
+
issues?: readonly unknown[];
|
|
32
|
+
}
|
|
33
|
+
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
34
|
+
export declare class SpongeError extends Error {
|
|
35
|
+
readonly code: SpongeCode;
|
|
36
|
+
readonly detail?: Record<string, unknown>;
|
|
37
|
+
readonly issues?: readonly any[];
|
|
38
|
+
constructor(code: SpongeCode, options?: SpongeErrorOptions);
|
|
39
|
+
}
|
|
40
|
+
/** A thrown value's code: a SpongeError's, or any object's `code` when it is a Sponge code; otherwise the fallback. */
|
|
41
|
+
export declare function codeOf(error: unknown, fallback?: SpongeCode): SpongeCode;
|
|
42
|
+
/** The public code for a code or thrown value. */
|
|
43
|
+
export declare function publicCode(value: unknown, fallback?: SpongeCode): ErrorCode;
|
|
44
|
+
export declare const exitFor: (value: unknown) => number;
|
|
45
|
+
/** The cause chain for --debug only: names, codes, details, messages and the first stack. */
|
|
46
|
+
export declare function debugChain(error: unknown): string;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/** Public codes: the frozen v1 ErrorEnvelope vocabulary. Exit statuses and hints are keyed by these. */
|
|
2
|
+
export const publicCodes = [
|
|
3
|
+
'grant-permission',
|
|
4
|
+
'repair-auth',
|
|
5
|
+
'unsupported',
|
|
6
|
+
'unavailable',
|
|
7
|
+
'rate-limited',
|
|
8
|
+
'partial',
|
|
9
|
+
'cleanup-required',
|
|
10
|
+
'cancelled',
|
|
11
|
+
'invalid-input',
|
|
12
|
+
'unclassified',
|
|
13
|
+
'integrity',
|
|
14
|
+
'limit-exceeded',
|
|
15
|
+
'runtime-missing',
|
|
16
|
+
'schema-newer',
|
|
17
|
+
'owner-busy',
|
|
18
|
+
];
|
|
19
|
+
/** Detail codes narrow a public code. They are stored as job reasons, matched by callers, and printed as `reason`. */
|
|
20
|
+
const parents = {
|
|
21
|
+
'invalid-index': 'invalid-input',
|
|
22
|
+
'stale-index': 'unavailable',
|
|
23
|
+
'incomplete-index': 'unavailable',
|
|
24
|
+
'cancelled-index': 'unavailable',
|
|
25
|
+
'invalid-search-query': 'invalid-input',
|
|
26
|
+
'invalid-search-vector': 'invalid-input',
|
|
27
|
+
'invalid-search-index': 'integrity',
|
|
28
|
+
'search-generation-missing': 'unavailable',
|
|
29
|
+
'unsupported-vector-runtime': 'unsupported',
|
|
30
|
+
'not-digest': 'unsupported',
|
|
31
|
+
malformed: 'invalid-input',
|
|
32
|
+
'no-capture': 'invalid-input',
|
|
33
|
+
};
|
|
34
|
+
export const indexCodes = [
|
|
35
|
+
'invalid-index',
|
|
36
|
+
'stale-index',
|
|
37
|
+
'incomplete-index',
|
|
38
|
+
'cancelled-index',
|
|
39
|
+
];
|
|
40
|
+
const known = new Set([...publicCodes, ...Object.keys(parents)]);
|
|
41
|
+
export const isSpongeCode = (value) => typeof value === 'string' && known.has(value);
|
|
42
|
+
/** The one code→exit-status table (contracts `exitCodes`: 0 success, 1 failure, 2 invalid input, 3 partial, 4 action required, 130 cancelled). */
|
|
43
|
+
export const exitStatus = {
|
|
44
|
+
'grant-permission': 4,
|
|
45
|
+
'repair-auth': 4,
|
|
46
|
+
'cleanup-required': 4,
|
|
47
|
+
'runtime-missing': 4,
|
|
48
|
+
'owner-busy': 4,
|
|
49
|
+
'invalid-input': 2,
|
|
50
|
+
partial: 3,
|
|
51
|
+
cancelled: 130,
|
|
52
|
+
unsupported: 1,
|
|
53
|
+
unavailable: 1,
|
|
54
|
+
'rate-limited': 1,
|
|
55
|
+
unclassified: 1,
|
|
56
|
+
integrity: 1,
|
|
57
|
+
'limit-exceeded': 1,
|
|
58
|
+
'schema-newer': 1,
|
|
59
|
+
};
|
|
60
|
+
/** The one code→user-facing hint table. Fixed text only: never paths, sources, credentials or upstream prose. */
|
|
61
|
+
export const hints = {
|
|
62
|
+
'grant-permission': 'Grant the requested permission, then retry.',
|
|
63
|
+
'repair-auth': 'Sign in again or repair the saved connection, then retry.',
|
|
64
|
+
unsupported: 'This source or platform is not supported.',
|
|
65
|
+
unavailable: 'Not found or not reachable now. Check the identifier, or retry later.',
|
|
66
|
+
'rate-limited': 'The source is rate limiting requests. Retry later.',
|
|
67
|
+
partial: 'Saved with missing parts. Refresh later to try again.',
|
|
68
|
+
'cleanup-required': 'A previous run left work to clean up. Run `sponge queue recover`.',
|
|
69
|
+
cancelled: 'Cancelled.',
|
|
70
|
+
'invalid-input': 'Check the command and its arguments. Run `sponge help COMMAND`.',
|
|
71
|
+
unclassified: 'Unexpected failure. Run again with --debug for details.',
|
|
72
|
+
integrity: 'A verification check failed. Nothing was changed. Run `sponge doctor`.',
|
|
73
|
+
'limit-exceeded': 'A size, count or time limit was exceeded.',
|
|
74
|
+
'runtime-missing': 'A required runtime is not installed. Run `sponge setup`.',
|
|
75
|
+
'schema-newer': 'This library was written by a newer Sponge. Upgrade Sponge.',
|
|
76
|
+
'owner-busy': 'Another Sponge process owns the library. Wait for it, or run `sponge queue recover` after it stops.',
|
|
77
|
+
};
|
|
78
|
+
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
79
|
+
export class SpongeError extends Error {
|
|
80
|
+
code;
|
|
81
|
+
detail;
|
|
82
|
+
issues;
|
|
83
|
+
constructor(code, options = {}) {
|
|
84
|
+
super(code, options.cause === undefined ? undefined : { cause: options.cause });
|
|
85
|
+
this.code = code;
|
|
86
|
+
this.name = 'SpongeError';
|
|
87
|
+
if (options.detail)
|
|
88
|
+
this.detail = options.detail;
|
|
89
|
+
if (options.issues)
|
|
90
|
+
this.issues = options.issues;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** A thrown value's code: a SpongeError's, or any object's `code` when it is a Sponge code; otherwise the fallback. */
|
|
94
|
+
export function codeOf(error, fallback = 'unclassified') {
|
|
95
|
+
const code = error?.code;
|
|
96
|
+
return isSpongeCode(code) ? code : fallback;
|
|
97
|
+
}
|
|
98
|
+
/** The public code for a code or thrown value. */
|
|
99
|
+
export function publicCode(value, fallback = 'unclassified') {
|
|
100
|
+
const code = isSpongeCode(value) ? value : codeOf(value, fallback);
|
|
101
|
+
return Object.hasOwn(parents, code) ? parents[code] : code;
|
|
102
|
+
}
|
|
103
|
+
export const exitFor = (value) => exitStatus[publicCode(value)];
|
|
104
|
+
/** The cause chain for --debug only: names, codes, details, messages and the first stack. */
|
|
105
|
+
export function debugChain(error) {
|
|
106
|
+
const lines = [];
|
|
107
|
+
let current = error, depth = 0;
|
|
108
|
+
while (current !== undefined && current !== null && depth < 8) {
|
|
109
|
+
const e = current;
|
|
110
|
+
lines.push(`${depth ? 'caused by' : 'error'}: ${String(e.name ?? typeof current)}${e.code !== undefined ? ` [${String(e.code)}]` : ''}${e.message !== undefined && e.message !== e.code ? ` ${String(e.message)}` : ''}${e.detail !== undefined ? ` ${JSON.stringify(e.detail)}` : ''}`);
|
|
111
|
+
if (!depth && typeof e.stack === 'string')
|
|
112
|
+
lines.push(...e.stack.split('\n').slice(1, 8));
|
|
113
|
+
current = current instanceof Error || typeof current === 'object' ? e.cause : undefined;
|
|
114
|
+
depth++;
|
|
115
|
+
}
|
|
116
|
+
return lines.map((line) => 'debug: ' + line).join('\n') + '\n';
|
|
117
|
+
}
|
|
@@ -6,11 +6,6 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export declare const campaignFields: ReadonlySet<string>;
|
|
8
8
|
export declare const clickTrackers: ReadonlySet<string>;
|
|
9
|
-
/** Credential-like query keys, compared case-insensitively. Presence makes a capture private by default. */
|
|
10
|
-
export declare const secretFields: ReadonlySet<string>;
|
|
11
|
-
export declare const secretPrefixes: readonly string[];
|
|
12
|
-
/** The decoded, lowercased key of one raw `a=b` field, or undefined when it does not decode. */
|
|
13
|
-
export declare function fieldKey(field: string): string | undefined;
|
|
14
9
|
export declare function isSecretField(key: string): boolean;
|
|
15
10
|
/** Filter raw fields: URLSearchParams mutation rewrites unknown signed query bytes. */
|
|
16
11
|
export declare function filterQuery(search: string, drop: (key: string) => boolean): string;
|