@hraness/sponge 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +343 -100
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +15 -6
- package/dist/src/importers/jungle-store.js +122 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +136 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -5,15 +5,15 @@ import { type TextQuote } from '../annotations/quote.js';
|
|
|
5
5
|
* order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
|
|
6
6
|
* decides. Nothing here calls a model.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
declare const ideaKinds: readonly ["finding", "argument", "method", "evidence", "recommendation", "counterpoint"];
|
|
9
9
|
export declare const forms: readonly ["article", "video", "paper", "thread", "pdf"];
|
|
10
|
-
|
|
10
|
+
type Voice = {
|
|
11
11
|
type: 'author';
|
|
12
12
|
} | {
|
|
13
13
|
type: 'speaker' | 'reported';
|
|
14
14
|
name: string;
|
|
15
15
|
};
|
|
16
|
-
|
|
16
|
+
type QuoteLocator = {
|
|
17
17
|
type: 'timestamp';
|
|
18
18
|
ms: number;
|
|
19
19
|
} | {
|
|
@@ -27,20 +27,20 @@ export interface DigestQuote {
|
|
|
27
27
|
attribution?: string;
|
|
28
28
|
locator?: QuoteLocator;
|
|
29
29
|
}
|
|
30
|
-
|
|
30
|
+
interface DigestIdea {
|
|
31
31
|
claim: string;
|
|
32
32
|
detail: string;
|
|
33
|
-
kind?: typeof ideaKinds[number];
|
|
33
|
+
kind?: (typeof ideaKinds)[number];
|
|
34
34
|
voice?: Voice;
|
|
35
35
|
quotes: DigestQuote[];
|
|
36
36
|
}
|
|
37
|
-
|
|
38
|
-
kind: typeof ideaKinds[number];
|
|
37
|
+
interface DigestIdeaV1 extends DigestIdea {
|
|
38
|
+
kind: (typeof ideaKinds)[number];
|
|
39
39
|
voice: Voice;
|
|
40
40
|
}
|
|
41
41
|
export interface DigestV1 {
|
|
42
42
|
schemaVersion: 1;
|
|
43
|
-
form: typeof forms[number];
|
|
43
|
+
form: (typeof forms)[number];
|
|
44
44
|
gist: {
|
|
45
45
|
lead: string;
|
|
46
46
|
context?: string;
|
|
@@ -50,7 +50,7 @@ export interface DigestV1 {
|
|
|
50
50
|
}
|
|
51
51
|
export interface DigestV2 {
|
|
52
52
|
schemaVersion: 2;
|
|
53
|
-
form: typeof forms[number];
|
|
53
|
+
form: (typeof forms)[number];
|
|
54
54
|
gist: {
|
|
55
55
|
lead: string;
|
|
56
56
|
context?: string;
|
|
@@ -60,21 +60,6 @@ export interface DigestV2 {
|
|
|
60
60
|
}
|
|
61
61
|
export type Digest = DigestV1 | DigestV2;
|
|
62
62
|
export declare const digestSchemaVersion = 2;
|
|
63
|
-
export declare const digestLimitsV1: {
|
|
64
|
-
readonly leadChars: 155;
|
|
65
|
-
readonly contextWords: 50;
|
|
66
|
-
readonly minIdeas: 3;
|
|
67
|
-
readonly maxIdeas: 5;
|
|
68
|
-
readonly claimWords: 20;
|
|
69
|
-
readonly detailWords: 60;
|
|
70
|
-
readonly detailSentences: 3;
|
|
71
|
-
readonly maxQuotesPerIdea: 2;
|
|
72
|
-
readonly minQuoteWords: 4;
|
|
73
|
-
readonly quoteWords: 25;
|
|
74
|
-
readonly totalQuoteWords: 150;
|
|
75
|
-
readonly sourceQuoteShare: 0.1;
|
|
76
|
-
readonly maxTags: 3;
|
|
77
|
-
};
|
|
78
63
|
/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
|
|
79
64
|
export declare const digestLimits: {
|
|
80
65
|
readonly leadChars: 240;
|
|
@@ -100,7 +85,7 @@ export interface DigestIssue {
|
|
|
100
85
|
path: string;
|
|
101
86
|
code: string;
|
|
102
87
|
}
|
|
103
|
-
|
|
88
|
+
interface LocatedQuote {
|
|
104
89
|
idea: number;
|
|
105
90
|
quote: number;
|
|
106
91
|
anchor: TextQuote;
|
|
@@ -124,3 +109,4 @@ export declare function parseDigest(value: unknown): {
|
|
|
124
109
|
};
|
|
125
110
|
/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
|
|
126
111
|
export declare function checkDigest(digest: Digest, source: string, options?: DigestOptions): DigestCheck;
|
|
112
|
+
export {};
|
|
@@ -5,14 +5,47 @@ import { foldText, locateAll, textQuote, words } from '../annotations/quote.js';
|
|
|
5
5
|
* order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
|
|
6
6
|
* decides. Nothing here calls a model.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
const ideaKinds = ['finding', 'argument', 'method', 'evidence', 'recommendation', 'counterpoint'];
|
|
9
9
|
export const forms = ['article', 'video', 'paper', 'thread', 'pdf'];
|
|
10
10
|
export const digestSchemaVersion = 2;
|
|
11
|
-
|
|
11
|
+
const digestLimitsV1 = {
|
|
12
|
+
leadChars: 155,
|
|
13
|
+
contextWords: 50,
|
|
14
|
+
minIdeas: 3,
|
|
15
|
+
maxIdeas: 5,
|
|
16
|
+
claimWords: 20,
|
|
17
|
+
detailWords: 60,
|
|
18
|
+
detailSentences: 3,
|
|
19
|
+
maxQuotesPerIdea: 2,
|
|
20
|
+
minQuoteWords: 4,
|
|
21
|
+
quoteWords: 25,
|
|
22
|
+
totalQuoteWords: 150,
|
|
23
|
+
sourceQuoteShare: 0.1,
|
|
24
|
+
maxTags: 3,
|
|
25
|
+
};
|
|
12
26
|
/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
|
|
13
|
-
export const digestLimits = {
|
|
27
|
+
export const digestLimits = {
|
|
28
|
+
leadChars: 240,
|
|
29
|
+
contextWords: 80,
|
|
30
|
+
gistWords: 100,
|
|
31
|
+
gistSentences: 5,
|
|
32
|
+
minIdeas: 3,
|
|
33
|
+
maxIdeas: 5,
|
|
34
|
+
claimWords: 20,
|
|
35
|
+
detailWords: 60,
|
|
36
|
+
detailSentences: 3,
|
|
37
|
+
maxQuotesPerIdea: 2,
|
|
38
|
+
minQuotes: 1,
|
|
39
|
+
maxQuotes: 6,
|
|
40
|
+
minQuoteWords: 4,
|
|
41
|
+
quoteWords: 25,
|
|
42
|
+
attributionWords: 25,
|
|
43
|
+
totalQuoteWords: 150,
|
|
44
|
+
sourceQuoteShare: 0.1,
|
|
45
|
+
maxTags: 3,
|
|
46
|
+
};
|
|
14
47
|
const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
15
|
-
const keys = (v, allowed) => Object.keys(v).every(k => allowed.includes(k));
|
|
48
|
+
const keys = (v, allowed) => Object.keys(v).every((k) => allowed.includes(k));
|
|
16
49
|
const str = (v, max = 2000) => typeof v === 'string' && v.trim().length > 0 && v.length <= max && !/[\u0000-\u0008\u000b-\u001f\u007f]/.test(v);
|
|
17
50
|
/** v1: sentence terminators followed by more text; abbreviations are rare enough in a 155-character lead to reject. */
|
|
18
51
|
const sentencesV1 = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
|
|
@@ -20,42 +53,73 @@ const sentencesV1 = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
|
|
|
20
53
|
const sentences = (v) => foldText(v).split(/(?<=[.!?]["')]*)\s+(?=["(]?[A-Z0-9])/).length;
|
|
21
54
|
/** Parse an untrusted digest (v1 or v2) into its shape, or report why not. */
|
|
22
55
|
export function parseDigest(value) {
|
|
23
|
-
const issues = [], bad = (path, code = 'shape') => {
|
|
24
|
-
|
|
56
|
+
const issues = [], bad = (path, code = 'shape') => {
|
|
57
|
+
issues.push({ path, code });
|
|
58
|
+
};
|
|
59
|
+
if (!object(value) ||
|
|
60
|
+
!keys(value, ['schemaVersion', 'form', 'gist', 'ideas', 'tags']) ||
|
|
61
|
+
(value.schemaVersion !== 1 && value.schemaVersion !== 2))
|
|
25
62
|
return { issues: [{ path: '', code: 'shape' }] };
|
|
26
63
|
const v2 = value.schemaVersion === 2;
|
|
27
64
|
if (!forms.includes(value.form))
|
|
28
65
|
bad('form');
|
|
29
66
|
const gist = value.gist;
|
|
30
|
-
if (!object(gist) ||
|
|
67
|
+
if (!object(gist) ||
|
|
68
|
+
!keys(gist, ['lead', 'context']) ||
|
|
69
|
+
!str(gist.lead) ||
|
|
70
|
+
(gist.context !== undefined && !str(gist.context)))
|
|
31
71
|
bad('gist');
|
|
32
72
|
if (!Array.isArray(value.ideas) || value.ideas.length > 8)
|
|
33
73
|
bad('ideas');
|
|
34
74
|
else
|
|
35
75
|
value.ideas.forEach((idea, i) => {
|
|
36
76
|
const at = `ideas.${i}`;
|
|
37
|
-
if (!object(idea) ||
|
|
77
|
+
if (!object(idea) ||
|
|
78
|
+
!keys(idea, ['claim', 'detail', 'kind', 'voice', 'quotes']) ||
|
|
79
|
+
!str(idea.claim) ||
|
|
80
|
+
!str(idea.detail) ||
|
|
81
|
+
!((v2 && idea.kind === undefined) || ideaKinds.includes(idea.kind)))
|
|
38
82
|
return bad(at);
|
|
39
83
|
const voice = idea.voice;
|
|
40
|
-
if (!(v2 && voice === undefined) &&
|
|
84
|
+
if (!(v2 && voice === undefined) &&
|
|
85
|
+
!(object(voice) &&
|
|
86
|
+
((voice.type === 'author' && keys(voice, ['type'])) ||
|
|
87
|
+
((voice.type === 'speaker' || voice.type === 'reported') &&
|
|
88
|
+
keys(voice, ['type', 'name']) &&
|
|
89
|
+
str(voice.name, 200)))))
|
|
41
90
|
bad(at + '.voice');
|
|
42
91
|
if (!Array.isArray(idea.quotes) || idea.quotes.length > 4)
|
|
43
92
|
return bad(at + '.quotes');
|
|
44
93
|
idea.quotes.forEach((quote, j) => {
|
|
45
|
-
if (!object(quote) ||
|
|
94
|
+
if (!object(quote) ||
|
|
95
|
+
!keys(quote, v2 ? ['text', 'speaker', 'attribution', 'locator'] : ['text', 'speaker', 'locator']) ||
|
|
96
|
+
!str(quote.text, 600) ||
|
|
97
|
+
(quote.speaker !== undefined && !str(quote.speaker, 200)) ||
|
|
98
|
+
(quote.attribution !== undefined && !str(quote.attribution, 600)))
|
|
46
99
|
return bad(`${at}.quotes.${j}`);
|
|
47
100
|
const loc = quote.locator;
|
|
48
|
-
if (loc !== undefined &&
|
|
101
|
+
if (loc !== undefined &&
|
|
102
|
+
!(object(loc) &&
|
|
103
|
+
((loc.type === 'timestamp' &&
|
|
104
|
+
keys(loc, ['type', 'ms']) &&
|
|
105
|
+
Number.isSafeInteger(loc.ms) &&
|
|
106
|
+
loc.ms >= 0) ||
|
|
107
|
+
(loc.type === 'page' &&
|
|
108
|
+
keys(loc, ['type', 'page']) &&
|
|
109
|
+
Number.isSafeInteger(loc.page) &&
|
|
110
|
+
loc.page >= 1))))
|
|
49
111
|
bad(`${at}.quotes.${j}.locator`);
|
|
50
112
|
});
|
|
51
113
|
});
|
|
52
|
-
if (!Array.isArray(value.tags) || value.tags.some(t => !str(t, 64)))
|
|
114
|
+
if (!Array.isArray(value.tags) || value.tags.some((t) => !str(t, 64)))
|
|
53
115
|
bad('tags');
|
|
54
116
|
return issues.length ? { issues } : { digest: value, issues };
|
|
55
117
|
}
|
|
56
118
|
/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
|
|
57
119
|
export function checkDigest(digest, source, options = {}) {
|
|
58
|
-
const issues = [], located = [], issue = (path, code) => {
|
|
120
|
+
const issues = [], located = [], issue = (path, code) => {
|
|
121
|
+
issues.push({ path, code });
|
|
122
|
+
}, v2 = digest.schemaVersion === 2, L = v2 ? digestLimits : digestLimitsV1, count = v2 ? sentences : sentencesV1;
|
|
59
123
|
const { lead, context } = digest.gist;
|
|
60
124
|
if (lead.length > L.leadChars)
|
|
61
125
|
issue('gist.lead', 'too-long');
|
|
@@ -103,7 +167,7 @@ export function checkDigest(digest, source, options = {}) {
|
|
|
103
167
|
if (!matches.length)
|
|
104
168
|
return issue(path, 'not-verbatim');
|
|
105
169
|
// Prefer the first occurrence not already claimed by an earlier quote.
|
|
106
|
-
const match = matches.find(m => !located.some(q => m.start < q.anchor.end && q.anchor.start < m.end));
|
|
170
|
+
const match = matches.find((m) => !located.some((q) => m.start < q.anchor.end && q.anchor.start < m.end));
|
|
107
171
|
if (!match)
|
|
108
172
|
return issue(path, 'overlap');
|
|
109
173
|
located.push({ idea: i, quote: j, anchor: textQuote(source, match.start, match.end) });
|
|
@@ -117,7 +181,7 @@ export function checkDigest(digest, source, options = {}) {
|
|
|
117
181
|
let previous = -1;
|
|
118
182
|
if (!v2)
|
|
119
183
|
digest.ideas.forEach((_, i) => {
|
|
120
|
-
const starts = located.filter(q => q.idea === i).map(q => q.anchor.start);
|
|
184
|
+
const starts = located.filter((q) => q.idea === i).map((q) => q.anchor.start);
|
|
121
185
|
if (!starts.length)
|
|
122
186
|
return;
|
|
123
187
|
const first = Math.min(...starts);
|
|
@@ -130,9 +194,13 @@ export function checkDigest(digest, source, options = {}) {
|
|
|
130
194
|
// The 10% share has a 25-word floor: an exemption so a short source can still carry one full quote.
|
|
131
195
|
if (quoteWords > Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)))
|
|
132
196
|
issue('ideas', 'quote-share');
|
|
133
|
-
if (digest.tags.length < (v2 ? 0 : 1) ||
|
|
197
|
+
if (digest.tags.length < (v2 ? 0 : 1) ||
|
|
198
|
+
digest.tags.length > L.maxTags ||
|
|
199
|
+
new Set(digest.tags).size !== digest.tags.length)
|
|
134
200
|
issue('tags', 'count');
|
|
135
|
-
digest.tags.forEach((tag, i) => {
|
|
136
|
-
|
|
201
|
+
digest.tags.forEach((tag, i) => {
|
|
202
|
+
if (options.tags ? !options.tags.includes(tag) : !/^[a-z0-9][a-z0-9-]{0,63}$/.test(tag))
|
|
203
|
+
issue(`tags.${i}`, 'vocabulary');
|
|
204
|
+
});
|
|
137
205
|
return { ok: issues.length === 0, issues, quotes: located };
|
|
138
206
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { ErrorCode } from '../contracts/index.js';
|
|
2
|
+
export type { ErrorCode };
|
|
3
|
+
/** Public codes: the frozen v1 ErrorEnvelope vocabulary. Exit statuses and hints are keyed by these. */
|
|
4
|
+
export declare const publicCodes: readonly ["grant-permission", "repair-auth", "unsupported", "unavailable", "rate-limited", "partial", "cleanup-required", "cancelled", "invalid-input", "unclassified", "integrity", "limit-exceeded", "runtime-missing", "schema-newer", "owner-busy"];
|
|
5
|
+
/** Detail codes narrow a public code. They are stored as job reasons, matched by callers, and printed as `reason`. */
|
|
6
|
+
declare const parents: {
|
|
7
|
+
readonly 'invalid-index': "invalid-input";
|
|
8
|
+
readonly 'stale-index': "unavailable";
|
|
9
|
+
readonly 'incomplete-index': "unavailable";
|
|
10
|
+
readonly 'cancelled-index': "unavailable";
|
|
11
|
+
readonly 'invalid-search-query': "invalid-input";
|
|
12
|
+
readonly 'invalid-search-vector': "invalid-input";
|
|
13
|
+
readonly 'invalid-search-index': "integrity";
|
|
14
|
+
readonly 'search-generation-missing': "unavailable";
|
|
15
|
+
readonly 'unsupported-vector-runtime': "unsupported";
|
|
16
|
+
readonly 'not-digest': "unsupported";
|
|
17
|
+
readonly malformed: "invalid-input";
|
|
18
|
+
readonly 'no-capture': "invalid-input";
|
|
19
|
+
};
|
|
20
|
+
export type DetailCode = keyof typeof parents;
|
|
21
|
+
export type SpongeCode = ErrorCode | DetailCode;
|
|
22
|
+
export declare const indexCodes: readonly DetailCode[];
|
|
23
|
+
export declare const isSpongeCode: (value: unknown) => value is SpongeCode;
|
|
24
|
+
/** The one code→exit-status table (contracts `exitCodes`: 0 success, 1 failure, 2 invalid input, 3 partial, 4 action required, 130 cancelled). */
|
|
25
|
+
export declare const exitStatus: Readonly<Record<ErrorCode, number>>;
|
|
26
|
+
/** The one code→user-facing hint table. Fixed text only: never paths, sources, credentials or upstream prose. */
|
|
27
|
+
export declare const hints: Readonly<Record<ErrorCode, string>>;
|
|
28
|
+
export interface SpongeErrorOptions {
|
|
29
|
+
detail?: Record<string, unknown>;
|
|
30
|
+
cause?: unknown;
|
|
31
|
+
issues?: readonly unknown[];
|
|
32
|
+
}
|
|
33
|
+
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
34
|
+
export declare class SpongeError extends Error {
|
|
35
|
+
readonly code: SpongeCode;
|
|
36
|
+
readonly detail?: Record<string, unknown>;
|
|
37
|
+
readonly issues?: readonly any[];
|
|
38
|
+
constructor(code: SpongeCode, options?: SpongeErrorOptions);
|
|
39
|
+
}
|
|
40
|
+
/** A thrown value's code: a SpongeError's, or any object's `code` when it is a Sponge code; otherwise the fallback. */
|
|
41
|
+
export declare function codeOf(error: unknown, fallback?: SpongeCode): SpongeCode;
|
|
42
|
+
/** The public code for a code or thrown value. */
|
|
43
|
+
export declare function publicCode(value: unknown, fallback?: SpongeCode): ErrorCode;
|
|
44
|
+
export declare const exitFor: (value: unknown) => number;
|
|
45
|
+
/** The cause chain for --debug only: names, codes, details, messages and the first stack. */
|
|
46
|
+
export declare function debugChain(error: unknown): string;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/** Public codes: the frozen v1 ErrorEnvelope vocabulary. Exit statuses and hints are keyed by these. */
|
|
2
|
+
export const publicCodes = [
|
|
3
|
+
'grant-permission',
|
|
4
|
+
'repair-auth',
|
|
5
|
+
'unsupported',
|
|
6
|
+
'unavailable',
|
|
7
|
+
'rate-limited',
|
|
8
|
+
'partial',
|
|
9
|
+
'cleanup-required',
|
|
10
|
+
'cancelled',
|
|
11
|
+
'invalid-input',
|
|
12
|
+
'unclassified',
|
|
13
|
+
'integrity',
|
|
14
|
+
'limit-exceeded',
|
|
15
|
+
'runtime-missing',
|
|
16
|
+
'schema-newer',
|
|
17
|
+
'owner-busy',
|
|
18
|
+
];
|
|
19
|
+
/** Detail codes narrow a public code. They are stored as job reasons, matched by callers, and printed as `reason`. */
|
|
20
|
+
const parents = {
|
|
21
|
+
'invalid-index': 'invalid-input',
|
|
22
|
+
'stale-index': 'unavailable',
|
|
23
|
+
'incomplete-index': 'unavailable',
|
|
24
|
+
'cancelled-index': 'unavailable',
|
|
25
|
+
'invalid-search-query': 'invalid-input',
|
|
26
|
+
'invalid-search-vector': 'invalid-input',
|
|
27
|
+
'invalid-search-index': 'integrity',
|
|
28
|
+
'search-generation-missing': 'unavailable',
|
|
29
|
+
'unsupported-vector-runtime': 'unsupported',
|
|
30
|
+
'not-digest': 'unsupported',
|
|
31
|
+
malformed: 'invalid-input',
|
|
32
|
+
'no-capture': 'invalid-input',
|
|
33
|
+
};
|
|
34
|
+
export const indexCodes = [
|
|
35
|
+
'invalid-index',
|
|
36
|
+
'stale-index',
|
|
37
|
+
'incomplete-index',
|
|
38
|
+
'cancelled-index',
|
|
39
|
+
];
|
|
40
|
+
const known = new Set([...publicCodes, ...Object.keys(parents)]);
|
|
41
|
+
export const isSpongeCode = (value) => typeof value === 'string' && known.has(value);
|
|
42
|
+
/** The one code→exit-status table (contracts `exitCodes`: 0 success, 1 failure, 2 invalid input, 3 partial, 4 action required, 130 cancelled). */
|
|
43
|
+
export const exitStatus = {
|
|
44
|
+
'grant-permission': 4,
|
|
45
|
+
'repair-auth': 4,
|
|
46
|
+
'cleanup-required': 4,
|
|
47
|
+
'runtime-missing': 4,
|
|
48
|
+
'owner-busy': 4,
|
|
49
|
+
'invalid-input': 2,
|
|
50
|
+
partial: 3,
|
|
51
|
+
cancelled: 130,
|
|
52
|
+
unsupported: 1,
|
|
53
|
+
unavailable: 1,
|
|
54
|
+
'rate-limited': 1,
|
|
55
|
+
unclassified: 1,
|
|
56
|
+
integrity: 1,
|
|
57
|
+
'limit-exceeded': 1,
|
|
58
|
+
'schema-newer': 1,
|
|
59
|
+
};
|
|
60
|
+
/** The one code→user-facing hint table. Fixed text only: never paths, sources, credentials or upstream prose. */
|
|
61
|
+
export const hints = {
|
|
62
|
+
'grant-permission': 'Grant the requested permission, then retry.',
|
|
63
|
+
'repair-auth': 'Sign in again or repair the saved connection, then retry.',
|
|
64
|
+
unsupported: 'This source or platform is not supported.',
|
|
65
|
+
unavailable: 'Not found or not reachable now. Check the identifier, or retry later.',
|
|
66
|
+
'rate-limited': 'The source is rate limiting requests. Retry later.',
|
|
67
|
+
partial: 'Saved with missing parts. Refresh later to try again.',
|
|
68
|
+
'cleanup-required': 'A previous run left work to clean up. Run `sponge queue recover`.',
|
|
69
|
+
cancelled: 'Cancelled.',
|
|
70
|
+
'invalid-input': 'Check the command and its arguments. Run `sponge help COMMAND`.',
|
|
71
|
+
unclassified: 'Unexpected failure. Run again with --debug for details.',
|
|
72
|
+
integrity: 'A verification check failed. Nothing was changed. Run `sponge doctor`.',
|
|
73
|
+
'limit-exceeded': 'A size, count or time limit was exceeded.',
|
|
74
|
+
'runtime-missing': 'A required runtime is not installed. Run `sponge setup`.',
|
|
75
|
+
'schema-newer': 'This library was written by a newer Sponge. Upgrade Sponge.',
|
|
76
|
+
'owner-busy': 'Another Sponge process owns the library. Wait for it, or run `sponge queue recover` after it stops.',
|
|
77
|
+
};
|
|
78
|
+
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
79
|
+
export class SpongeError extends Error {
|
|
80
|
+
code;
|
|
81
|
+
detail;
|
|
82
|
+
issues;
|
|
83
|
+
constructor(code, options = {}) {
|
|
84
|
+
super(code, options.cause === undefined ? undefined : { cause: options.cause });
|
|
85
|
+
this.code = code;
|
|
86
|
+
this.name = 'SpongeError';
|
|
87
|
+
if (options.detail)
|
|
88
|
+
this.detail = options.detail;
|
|
89
|
+
if (options.issues)
|
|
90
|
+
this.issues = options.issues;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** A thrown value's code: a SpongeError's, or any object's `code` when it is a Sponge code; otherwise the fallback. */
|
|
94
|
+
export function codeOf(error, fallback = 'unclassified') {
|
|
95
|
+
const code = error?.code;
|
|
96
|
+
return isSpongeCode(code) ? code : fallback;
|
|
97
|
+
}
|
|
98
|
+
/** The public code for a code or thrown value. */
|
|
99
|
+
export function publicCode(value, fallback = 'unclassified') {
|
|
100
|
+
const code = isSpongeCode(value) ? value : codeOf(value, fallback);
|
|
101
|
+
return Object.hasOwn(parents, code) ? parents[code] : code;
|
|
102
|
+
}
|
|
103
|
+
export const exitFor = (value) => exitStatus[publicCode(value)];
|
|
104
|
+
/** The cause chain for --debug only: names, codes, details, messages and the first stack. */
|
|
105
|
+
export function debugChain(error) {
|
|
106
|
+
const lines = [];
|
|
107
|
+
let current = error, depth = 0;
|
|
108
|
+
while (current !== undefined && current !== null && depth < 8) {
|
|
109
|
+
const e = current;
|
|
110
|
+
lines.push(`${depth ? 'caused by' : 'error'}: ${String(e.name ?? typeof current)}${e.code !== undefined ? ` [${String(e.code)}]` : ''}${e.message !== undefined && e.message !== e.code ? ` ${String(e.message)}` : ''}${e.detail !== undefined ? ` ${JSON.stringify(e.detail)}` : ''}`);
|
|
111
|
+
if (!depth && typeof e.stack === 'string')
|
|
112
|
+
lines.push(...e.stack.split('\n').slice(1, 8));
|
|
113
|
+
current = current instanceof Error || typeof current === 'object' ? e.cause : undefined;
|
|
114
|
+
depth++;
|
|
115
|
+
}
|
|
116
|
+
return lines.map((line) => 'debug: ' + line).join('\n') + '\n';
|
|
117
|
+
}
|
|
@@ -6,11 +6,6 @@
|
|
|
6
6
|
*/
|
|
7
7
|
export declare const campaignFields: ReadonlySet<string>;
|
|
8
8
|
export declare const clickTrackers: ReadonlySet<string>;
|
|
9
|
-
/** Credential-like query keys, compared case-insensitively. Presence makes a capture private by default. */
|
|
10
|
-
export declare const secretFields: ReadonlySet<string>;
|
|
11
|
-
export declare const secretPrefixes: readonly string[];
|
|
12
|
-
/** The decoded, lowercased key of one raw `a=b` field, or undefined when it does not decode. */
|
|
13
|
-
export declare function fieldKey(field: string): string | undefined;
|
|
14
9
|
export declare function isSecretField(key: string): boolean;
|
|
15
10
|
/** Filter raw fields: URLSearchParams mutation rewrites unknown signed query bytes. */
|
|
16
11
|
export declare function filterQuery(search: string, drop: (key: string) => boolean): string;
|
|
@@ -4,13 +4,46 @@
|
|
|
4
4
|
* Shareable URIs also drop `clickTrackers` and every credential-like field, so a published URI
|
|
5
5
|
* never carries a capability. See docs/decisions/sync-and-publishing.md.
|
|
6
6
|
*/
|
|
7
|
-
export const campaignFields = new Set([
|
|
8
|
-
|
|
7
|
+
export const campaignFields = new Set([
|
|
8
|
+
'utm_source',
|
|
9
|
+
'utm_medium',
|
|
10
|
+
'utm_campaign',
|
|
11
|
+
'utm_term',
|
|
12
|
+
'utm_content',
|
|
13
|
+
'utm_id',
|
|
14
|
+
]);
|
|
15
|
+
export const clickTrackers = new Set([
|
|
16
|
+
'fbclid',
|
|
17
|
+
'gclid',
|
|
18
|
+
'dclid',
|
|
19
|
+
'msclkid',
|
|
20
|
+
'mc_cid',
|
|
21
|
+
'mc_eid',
|
|
22
|
+
'igshid',
|
|
23
|
+
]);
|
|
9
24
|
/** Credential-like query keys, compared case-insensitively. Presence makes a capture private by default. */
|
|
10
|
-
|
|
11
|
-
|
|
25
|
+
const secretFields = new Set([
|
|
26
|
+
'token',
|
|
27
|
+
'key',
|
|
28
|
+
'sig',
|
|
29
|
+
'signature',
|
|
30
|
+
'auth',
|
|
31
|
+
'access_token',
|
|
32
|
+
'code',
|
|
33
|
+
'rlkey',
|
|
34
|
+
'session',
|
|
35
|
+
'password',
|
|
36
|
+
'secret',
|
|
37
|
+
'api_key',
|
|
38
|
+
'apikey',
|
|
39
|
+
'auth_token',
|
|
40
|
+
'id_token',
|
|
41
|
+
'refresh_token',
|
|
42
|
+
'client_secret',
|
|
43
|
+
]);
|
|
44
|
+
const secretPrefixes = ['x-amz-', 'x-goog-'];
|
|
12
45
|
/** The decoded, lowercased key of one raw `a=b` field, or undefined when it does not decode. */
|
|
13
|
-
|
|
46
|
+
function fieldKey(field) {
|
|
14
47
|
try {
|
|
15
48
|
return decodeURIComponent(field.split('=', 1)[0].replaceAll('+', ' ')).toLowerCase();
|
|
16
49
|
}
|
|
@@ -18,12 +51,20 @@ export function fieldKey(field) {
|
|
|
18
51
|
return undefined;
|
|
19
52
|
}
|
|
20
53
|
}
|
|
21
|
-
export function isSecretField(key) {
|
|
54
|
+
export function isSecretField(key) {
|
|
55
|
+
return secretFields.has(key) || secretPrefixes.some((p) => key.startsWith(p));
|
|
56
|
+
}
|
|
22
57
|
/** Filter raw fields: URLSearchParams mutation rewrites unknown signed query bytes. */
|
|
23
58
|
export function filterQuery(search, drop) {
|
|
24
59
|
if (!search)
|
|
25
60
|
return '';
|
|
26
|
-
const fields = search
|
|
61
|
+
const fields = search
|
|
62
|
+
.slice(1)
|
|
63
|
+
.split('&')
|
|
64
|
+
.filter((field) => {
|
|
65
|
+
const key = fieldKey(field);
|
|
66
|
+
return key === undefined || !drop(key);
|
|
67
|
+
});
|
|
27
68
|
return fields.length ? '?' + fields.join('&') : '';
|
|
28
69
|
}
|
|
29
70
|
/**
|
|
@@ -42,11 +83,22 @@ export function isCapabilityUrl(locator) {
|
|
|
42
83
|
return false;
|
|
43
84
|
if (url.username || url.password)
|
|
44
85
|
return true;
|
|
45
|
-
if (url.search &&
|
|
86
|
+
if (url.search &&
|
|
87
|
+
url.search
|
|
88
|
+
.slice(1)
|
|
89
|
+
.split('&')
|
|
90
|
+
.some((field) => {
|
|
91
|
+
const key = fieldKey(field);
|
|
92
|
+
return key !== undefined && isSecretField(key);
|
|
93
|
+
}))
|
|
46
94
|
return true;
|
|
47
95
|
const host = url.hostname.replace(/^www\./, ''), path = url.pathname;
|
|
48
96
|
// Google Docs, Sheets, Slides, Forms, Drawings and Drive files shared "anyone with the link".
|
|
49
|
-
if ((host === 'docs.google.com' &&
|
|
97
|
+
if (((host === 'docs.google.com' &&
|
|
98
|
+
/^\/(document|spreadsheets|presentation|forms|drawings)\/d\/[^/]+/.test(path) &&
|
|
99
|
+
!/^\/[^/]+\/d\/e\//.test(path)) ||
|
|
100
|
+
(host === 'drive.google.com' && /^\/(file\/d|drive\/folders)\/[^/]+/.test(path))) &&
|
|
101
|
+
url.searchParams.get('usp') === 'sharing')
|
|
50
102
|
return true;
|
|
51
103
|
// Dropbox shared links are capabilities with or without rlkey.
|
|
52
104
|
if ((host === 'dropbox.com' || host === 'dl.dropboxusercontent.com') && /^\/(s|sh|scl)\//.test(path))
|
|
@@ -3,15 +3,13 @@ export interface ItemUri {
|
|
|
3
3
|
uri: string;
|
|
4
4
|
role: 'primary' | 'alias';
|
|
5
5
|
resolvable: boolean;
|
|
6
|
-
origin: 'locator' | 'revision';
|
|
6
|
+
origin: 'locator' | 'revision' | 'metadata' | 'user';
|
|
7
7
|
}
|
|
8
8
|
/** `capability`: a public-realm or imported link that grants access by itself (see query-policy.ts). */
|
|
9
9
|
export type AccessClass = 'public' | 'account' | 'local' | 'imported' | 'capability';
|
|
10
10
|
export declare const maxUriBytes = 8192;
|
|
11
11
|
/** Same-work identity drops fragments, click trackers and credential-like fields that the source key keeps for capture fidelity. */
|
|
12
12
|
export declare function shareableWebUri(locator: string): string;
|
|
13
|
-
/** Provider identifiers name the same work across URL spellings, hosts and mirrors. */
|
|
14
|
-
export declare function providerUris(href: string): string[];
|
|
15
13
|
export declare function isResolvable(uri: string): boolean;
|
|
16
14
|
/**
|
|
17
15
|
* Derive an item's URIs from its first source and, once captured, its source PDF. Total: a
|
|
@@ -24,7 +22,10 @@ export declare function deriveUris(input: {
|
|
|
24
22
|
locator: string;
|
|
25
23
|
sourcePdfSha256?: string;
|
|
26
24
|
}): ItemUri[];
|
|
27
|
-
/**
|
|
28
|
-
|
|
25
|
+
/**
|
|
26
|
+
* How the first source was reached decides the default visibility; see docs/decisions/sync-and-publishing.md.
|
|
27
|
+
* An imported local file with a recorded web address (`webUri`, origin 'metadata') is classed by that address instead.
|
|
28
|
+
*/
|
|
29
|
+
export declare function accessClass(realm: string, locator: string, webUri?: string): AccessClass;
|
|
29
30
|
/** Public only when the source was public. Imports keep their source's publication decision. */
|
|
30
31
|
export declare function defaultVisibility(access: AccessClass, importedPrivacy?: string): 'public' | 'private';
|