@hraness/wordcell 0.26.3 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -21
- package/dist/capture.js +15 -7
- package/dist/cli-bin.js +2 -2
- package/dist/cli.js +21 -19
- package/dist/clip/acquire.js +4 -3
- package/dist/clip/bundle-reader.js +3 -2
- package/dist/clip/cli.js +8 -7
- package/dist/clip/defuddle-worker.js +4 -1
- package/dist/clip/doctor.js +5 -4
- package/dist/clip/jobs.js +2 -1
- package/dist/clip/persist.js +2 -1
- package/dist/clip/refresh.js +4 -3
- package/dist/evaluation-builder-bin.js +2 -2
- package/dist/evaluation-builder.js +3 -3
- package/dist/evaluation-kb.js +3 -3
- package/dist/{index-mp5y3ty3.js → index-30v9w4cy.js} +36 -20
- package/dist/{index-9paxge84.js → index-3hts6yy9.js} +1 -1
- package/dist/{index-h9vyd03z.js → index-58azscg5.js} +7 -6
- package/dist/{index-9zkba8hr.js → index-7bqwzjba.js} +7 -6
- package/dist/{index-j4zgmzjr.js → index-8djdwjjd.js} +1 -1
- package/dist/{index-5h5awwh4.js → index-b40e2dkx.js} +2 -2
- package/dist/index-e2wstc7t.js +323 -0
- package/dist/{index-hacpnaew.js → index-fd8j527z.js} +1 -1
- package/dist/{index-mxxxytys.js → index-g63d3937.js} +4 -0
- package/dist/{index-k86wepd8.js → index-h2w7syy2.js} +5 -1
- package/dist/{index-9fywek7g.js → index-ht14xne3.js} +1 -1
- package/dist/{index-82r6g4py.js → index-k9yzc4vn.js} +1 -1
- package/dist/{index-m9p2a4mj.js → index-my50xsa3.js} +221 -36
- package/dist/{index-ncwzmsge.js → index-nwdzhj1h.js} +2 -2
- package/dist/{index-9rf81m0p.js → index-pbz3qw1s.js} +35 -20
- package/dist/index-pn14a2m5.js +59 -0
- package/dist/{index-nd8v7r0z.js → index-sp43012q.js} +1 -1
- package/dist/{index-8pabwzqg.js → index-t4hjm1kk.js} +2 -1
- package/dist/{index-npg9z1a4.js → index-xn5q4kne.js} +5 -1
- package/dist/{index-j2jt49gr.js → index-yatn62qq.js} +1 -1
- package/dist/index.js +14 -4
- package/dist/pdf.js +438 -4
- package/dist/portfolio.js +3 -3
- package/dist/rerank-clef.js +16 -0
- package/dist/rerank-typesafe.js +3 -1
- package/dist/sdk.js +2 -2
- package/dist/{support-9qdr4hb8.js → support-s059eytv.js} +20 -18
- package/dist/url-intelligence.js +4 -3
- package/dist/workflows/decision-context.js +3 -3
- package/dist/workflows/index.js +3 -3
- package/package.json +23 -9
- package/skills/wordcell/SKILL.md +1 -1
- package/skills/wordcell/references/query.md +11 -6
- package/src/capture.ts +5 -0
- package/src/cli-help.ts +2 -1
- package/src/cli-program.ts +23 -8
- package/src/clip/bundle-reader.ts +3 -0
- package/src/clip/capture.ts +1 -0
- package/src/clip/extract.ts +2 -0
- package/src/clip/lib.ts +3 -0
- package/src/clip/metadata-search-tool/Cargo.lock +6 -6
- package/src/clip/metadata.ts +232 -0
- package/src/clip/persist.ts +3 -0
- package/src/index.ts +2 -1
- package/src/pdf/bundle-reader.ts +457 -0
- package/src/pdf/capture.ts +6 -2
- package/src/pdf/cli.ts +2 -1
- package/src/pdf/errors.ts +11 -0
- package/src/pdf/extract.ts +16 -13
- package/src/pdf/layout.ts +136 -3
- package/src/pdf/markdown.ts +7 -1
- package/src/pdf/model.ts +9 -0
- package/src/pdf/persist.ts +8 -2
- package/src/pdf/source.ts +5 -4
- package/src/pdf/tools.ts +8 -6
- package/src/pdf.ts +10 -0
- package/src/rerank-clef.ts +52 -0
- package/src/rerank-credentials.ts +11 -2
- package/src/rerank-typesafe.ts +44 -20
- package/src/rerank.ts +1 -1
- package/src/sdk.ts +4 -2
- /package/dist/{index-cd75vky9.js → index-jbtyp6q5.js} +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
createHostedReranker
|
|
4
|
+
} from "./index-pbz3qw1s.js";
|
|
5
|
+
|
|
6
|
+
// src/rerank-clef.ts
|
|
7
|
+
var CLEF_MODELS = ["clef", "clef-flash"];
|
|
8
|
+
var DEFAULT_CLEF_MODEL = "clef";
|
|
9
|
+
function clefEndpoint(accountId, model = DEFAULT_CLEF_MODEL) {
|
|
10
|
+
if (typeof accountId !== "string" || !/^[a-f0-9]{32}$/u.test(accountId) || !CLEF_MODELS.includes(model)) {
|
|
11
|
+
throw new Error("Invalid Cloudflare Clef account or model.");
|
|
12
|
+
}
|
|
13
|
+
return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/@cf/cloudflare/${model}`;
|
|
14
|
+
}
|
|
15
|
+
function unwrapClefResponse(value) {
|
|
16
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
17
|
+
return null;
|
|
18
|
+
const envelope = value;
|
|
19
|
+
if (envelope["success"] !== true || !Object.hasOwn(envelope, "result") || envelope["errors"] !== undefined && (!Array.isArray(envelope["errors"]) || envelope["errors"].length !== 0))
|
|
20
|
+
return null;
|
|
21
|
+
return envelope["result"];
|
|
22
|
+
}
|
|
23
|
+
function createClefReranker(options = {}) {
|
|
24
|
+
const environment = options.environment ?? process.env;
|
|
25
|
+
const accountId = Object.hasOwn(environment, "CLOUDFLARE_ACCOUNT_ID") ? environment["CLOUDFLARE_ACCOUNT_ID"] : undefined;
|
|
26
|
+
const apiKey = Object.hasOwn(environment, "CLOUDFLARE_API_TOKEN") ? environment["CLOUDFLARE_API_TOKEN"] : Object.hasOwn(environment, "CLOUDFLARE_AUTH_TOKEN") ? environment["CLOUDFLARE_AUTH_TOKEN"] : undefined;
|
|
27
|
+
const model = options.model === undefined ? DEFAULT_CLEF_MODEL : options.model;
|
|
28
|
+
let endpoint;
|
|
29
|
+
try {
|
|
30
|
+
if (accountId !== undefined)
|
|
31
|
+
endpoint = clefEndpoint(accountId, model);
|
|
32
|
+
} catch {
|
|
33
|
+
endpoint = undefined;
|
|
34
|
+
}
|
|
35
|
+
const reranker = createHostedReranker(options, {
|
|
36
|
+
id: "clef",
|
|
37
|
+
label: "Cloudflare Clef",
|
|
38
|
+
model,
|
|
39
|
+
endpoint,
|
|
40
|
+
apiKey,
|
|
41
|
+
unavailableMessage: "Cloudflare Clef credentials or model are missing or invalid; set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN.",
|
|
42
|
+
unwrap: unwrapClefResponse
|
|
43
|
+
});
|
|
44
|
+
return {
|
|
45
|
+
id: reranker.id,
|
|
46
|
+
rerank: async (request) => {
|
|
47
|
+
try {
|
|
48
|
+
if (typeof request === "object" && request !== null && "images" in request) {
|
|
49
|
+
return { status: "failed", message: "Wordcell reranking accepts text only; image evidence is not supported." };
|
|
50
|
+
}
|
|
51
|
+
return await reranker.rerank(request);
|
|
52
|
+
} catch {
|
|
53
|
+
return { status: "failed", message: "Cloudflare Clef rerank request was malformed or failed." };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export { CLEF_MODELS, DEFAULT_CLEF_MODEL, clefEndpoint, createClefReranker };
|
|
@@ -76,7 +76,8 @@ var commandHelp = [
|
|
|
76
76
|
PRIORITY,
|
|
77
77
|
["--candidate-limit <count>", "How many candidates to rank before trimming"],
|
|
78
78
|
["--min-score <score>", "Drop results scoring below this (0 to 1)"],
|
|
79
|
-
["--rerank <typesafe>", "
|
|
79
|
+
["--rerank <clef|typesafe>", "Send query and snippets to Clef (typesafe is deprecated)"],
|
|
80
|
+
["--rerank-model <clef|clef-flash>", "Clef model to use (default: clef)"],
|
|
80
81
|
["--rerank-limit <2..25>", "How many results the reranker sees"],
|
|
81
82
|
["--selected-passage", "Add a local source excerpt to each result"],
|
|
82
83
|
JSON_OPTION
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
import {
|
|
3
3
|
CAPTURE_MANIFEST_FILENAME,
|
|
4
4
|
CAPTURE_SOURCE_EVIDENCE_PATH
|
|
5
|
-
} from "./index-
|
|
5
|
+
} from "./index-g63d3937.js";
|
|
6
|
+
import {
|
|
7
|
+
parseArticleMetadata
|
|
8
|
+
} from "./index-e2wstc7t.js";
|
|
6
9
|
|
|
7
10
|
// src/clip/bundle-reader.ts
|
|
8
11
|
import { createHash } from "crypto";
|
|
@@ -384,6 +387,7 @@ async function readCaptureBundle(path, options = {}) {
|
|
|
384
387
|
}
|
|
385
388
|
await assertBundleRoot(root);
|
|
386
389
|
return Object.freeze({
|
|
390
|
+
...manifest.metadata === undefined ? {} : { metadata: parseArticleMetadata(manifest.metadata) },
|
|
387
391
|
root: root.path,
|
|
388
392
|
schemaVersion,
|
|
389
393
|
sourceUrl: manifest.sourceUrl,
|
package/dist/index.js
CHANGED
|
@@ -19,11 +19,17 @@ import {
|
|
|
19
19
|
import {
|
|
20
20
|
initVault
|
|
21
21
|
} from "./index-23z4zxgg.js";
|
|
22
|
+
import {
|
|
23
|
+
CLEF_MODELS,
|
|
24
|
+
DEFAULT_CLEF_MODEL,
|
|
25
|
+
clefEndpoint,
|
|
26
|
+
createClefReranker
|
|
27
|
+
} from "./index-pn14a2m5.js";
|
|
22
28
|
import {
|
|
23
29
|
DEFAULT_SYSTEMONE_ENDPOINT,
|
|
24
30
|
DEFAULT_SYSTEMONE_MODEL,
|
|
25
31
|
createTypeSafeReranker
|
|
26
|
-
} from "./index-
|
|
32
|
+
} from "./index-pbz3qw1s.js";
|
|
27
33
|
import {
|
|
28
34
|
MAX_SOURCE_DISPOSITION_EVIDENCE,
|
|
29
35
|
MAX_SOURCE_INBOX_CONNECTIONS,
|
|
@@ -37,7 +43,7 @@ import {
|
|
|
37
43
|
knowledgeBaseEvaluationRetrieverIds,
|
|
38
44
|
openKnowledgeBaseEvaluation,
|
|
39
45
|
verifyFrozenEvaluationSnapshot
|
|
40
|
-
} from "./index-
|
|
46
|
+
} from "./index-yatn62qq.js";
|
|
41
47
|
import {
|
|
42
48
|
DEFAULT_SEARCH_RESULTS,
|
|
43
49
|
MAX_SEARCH_CANDIDATES,
|
|
@@ -50,7 +56,8 @@ import {
|
|
|
50
56
|
packUntrustedSearchContext,
|
|
51
57
|
searchEvidenceRank,
|
|
52
58
|
validateKnowledgeBaseSearchHistory
|
|
53
|
-
} from "./index-
|
|
59
|
+
} from "./index-7bqwzjba.js";
|
|
60
|
+
import"./index-4j3tt0c3.js";
|
|
54
61
|
import"./index-adx6khj5.js";
|
|
55
62
|
import {
|
|
56
63
|
MAX_EMBEDDING_MODEL_BYTES,
|
|
@@ -68,7 +75,6 @@ import {
|
|
|
68
75
|
semanticDatabasePath,
|
|
69
76
|
sha256EmbeddingModelFile
|
|
70
77
|
} from "./index-fp732bgg.js";
|
|
71
|
-
import"./index-4j3tt0c3.js";
|
|
72
78
|
import {
|
|
73
79
|
percolateWithGraph
|
|
74
80
|
} from "./index-wqx1x32f.js";
|
|
@@ -866,7 +872,9 @@ export {
|
|
|
866
872
|
createNote,
|
|
867
873
|
createGraphSnapshot,
|
|
868
874
|
createConceptNote,
|
|
875
|
+
createClefReranker,
|
|
869
876
|
compareAgentGuideAudits,
|
|
877
|
+
clefEndpoint,
|
|
870
878
|
classifyRepositoryMemoryRecord,
|
|
871
879
|
checkpointSemanticWarmCache,
|
|
872
880
|
catalogStart,
|
|
@@ -1010,5 +1018,7 @@ export {
|
|
|
1010
1018
|
DEFAULT_PUBLISH_LIST_LIMIT,
|
|
1011
1019
|
DEFAULT_PERCOLATION_MIN_SUPPORT,
|
|
1012
1020
|
DEFAULT_PERCOLATION_LIMIT,
|
|
1021
|
+
DEFAULT_CLEF_MODEL,
|
|
1022
|
+
CLEF_MODELS,
|
|
1013
1023
|
AgentContextRepositoryPathError
|
|
1014
1024
|
};
|
package/dist/pdf.js
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
PDF_CAPTURE_MANIFEST_FILENAME,
|
|
5
5
|
PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
|
|
6
6
|
PDF_CAPTURE_SOURCE_FILENAME,
|
|
7
|
+
PdfCaptureError,
|
|
7
8
|
buildPdfMarkdown,
|
|
8
9
|
decodePopplerText,
|
|
9
10
|
inspectPdf,
|
|
@@ -24,15 +25,442 @@ import {
|
|
|
24
25
|
resolvePdfTools,
|
|
25
26
|
runPdfCapture,
|
|
26
27
|
runPdfToolCommand
|
|
27
|
-
} from "./index-
|
|
28
|
-
import"./index-
|
|
29
|
-
import"./index-
|
|
30
|
-
import"./index-
|
|
28
|
+
} from "./index-my50xsa3.js";
|
|
29
|
+
import"./index-h2w7syy2.js";
|
|
30
|
+
import"./index-jbtyp6q5.js";
|
|
31
|
+
import"./index-g63d3937.js";
|
|
32
|
+
import"./index-e2wstc7t.js";
|
|
31
33
|
import"./index-e5fbsywq.js";
|
|
32
34
|
import"./index-gh719d91.js";
|
|
33
35
|
import"./index-1xxnjn0d.js";
|
|
34
36
|
import"./index-z1w83f81.js";
|
|
35
37
|
|
|
38
|
+
// src/pdf/bundle-reader.ts
|
|
39
|
+
import { createHash } from "crypto";
|
|
40
|
+
import { constants as fsConstants } from "fs";
|
|
41
|
+
import { lstat, open, opendir, realpath } from "fs/promises";
|
|
42
|
+
import { dirname, isAbsolute, relative, resolve, sep } from "path";
|
|
43
|
+
function boundedInteger(value, fallback, maximum, label) {
|
|
44
|
+
const selected = value ?? fallback;
|
|
45
|
+
if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) {
|
|
46
|
+
throw new PdfCaptureError("invalid-input", `${label} must be an integer from 1 through ${maximum}`);
|
|
47
|
+
}
|
|
48
|
+
return selected;
|
|
49
|
+
}
|
|
50
|
+
function confinedRelativePath(value, label) {
|
|
51
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 4096 || value.includes("\x00")) {
|
|
52
|
+
throw new PdfCaptureError("integrity", `${label} must be a bounded non-empty path`);
|
|
53
|
+
}
|
|
54
|
+
if (isAbsolute(value) || value.includes("\\"))
|
|
55
|
+
throw new PdfCaptureError("integrity", `${label} must be a relative POSIX path`);
|
|
56
|
+
const segments = value.split("/");
|
|
57
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
|
|
58
|
+
throw new PdfCaptureError("integrity", `${label} contains an unsafe segment`);
|
|
59
|
+
}
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
class MissingCaptureAssetError extends PdfCaptureError {
|
|
64
|
+
constructor(message) {
|
|
65
|
+
super("integrity", message);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function isMissingPathError(error) {
|
|
69
|
+
return error instanceof Error && "code" in error && error.code === "ENOENT";
|
|
70
|
+
}
|
|
71
|
+
function assertAssetVerificationDeadline(deadline) {
|
|
72
|
+
if (deadline !== undefined && performance.now() >= deadline) {
|
|
73
|
+
throw new PdfCaptureError("timeout", "capture asset verification exceeded its time budget");
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function confined(root, relativePath, label) {
|
|
77
|
+
const path = resolve(root.path, relativePath);
|
|
78
|
+
const lexical = relative(root.path, path);
|
|
79
|
+
if (lexical === "" || lexical === ".." || lexical.startsWith(`..${sep}`) || isAbsolute(lexical)) {
|
|
80
|
+
throw new PdfCaptureError("integrity", `${label} escapes the capture bundle`);
|
|
81
|
+
}
|
|
82
|
+
return path;
|
|
83
|
+
}
|
|
84
|
+
async function safeDirectory(pathInput) {
|
|
85
|
+
if (pathInput.trim() === "")
|
|
86
|
+
throw new PdfCaptureError("integrity", "capture bundle path is required");
|
|
87
|
+
const path = resolve(pathInput);
|
|
88
|
+
const canonical = await realpath(path);
|
|
89
|
+
if (canonical !== path)
|
|
90
|
+
throw new PdfCaptureError("integrity", "capture bundle path must be canonical and cannot traverse a symbolic link");
|
|
91
|
+
const stats = await lstat(path, { bigint: true });
|
|
92
|
+
if (!stats.isDirectory() || stats.isSymbolicLink())
|
|
93
|
+
throw new PdfCaptureError("integrity", "capture bundle path must be a real directory");
|
|
94
|
+
if (dirname(path) === path)
|
|
95
|
+
throw new PdfCaptureError("integrity", "refusing a filesystem root as a capture bundle");
|
|
96
|
+
const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
97
|
+
try {
|
|
98
|
+
const opened = await handle.stat({ bigint: true });
|
|
99
|
+
if (!opened.isDirectory() || opened.dev !== stats.dev || opened.ino !== stats.ino) {
|
|
100
|
+
throw new PdfCaptureError("integrity", "capture bundle root changed while it was opened");
|
|
101
|
+
}
|
|
102
|
+
return { path, handle, device: opened.dev, inode: opened.ino };
|
|
103
|
+
} catch (error) {
|
|
104
|
+
await handle.close();
|
|
105
|
+
throw error;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async function assertBundleRoot(root, deadline) {
|
|
109
|
+
assertAssetVerificationDeadline(deadline);
|
|
110
|
+
const [opened, named, canonical] = await Promise.all([
|
|
111
|
+
root.handle.stat({ bigint: true }),
|
|
112
|
+
lstat(root.path, { bigint: true }),
|
|
113
|
+
realpath(root.path)
|
|
114
|
+
]);
|
|
115
|
+
assertAssetVerificationDeadline(deadline);
|
|
116
|
+
if (canonical !== root.path || !opened.isDirectory() || !named.isDirectory() || named.isSymbolicLink() || opened.dev !== root.device || opened.ino !== root.inode || named.dev !== root.device || named.ino !== root.inode)
|
|
117
|
+
throw new PdfCaptureError("integrity", "capture bundle root changed during inspection");
|
|
118
|
+
}
|
|
119
|
+
async function assertAncestorChain(root, relativePath, label, deadline) {
|
|
120
|
+
await assertBundleRoot(root, deadline);
|
|
121
|
+
const segments = confinedRelativePath(relativePath, label).split("/");
|
|
122
|
+
let current = root.path;
|
|
123
|
+
for (const segment of segments.slice(0, -1)) {
|
|
124
|
+
current = resolve(current, segment);
|
|
125
|
+
assertAssetVerificationDeadline(deadline);
|
|
126
|
+
const metadata = await lstat(current, { bigint: true });
|
|
127
|
+
assertAssetVerificationDeadline(deadline);
|
|
128
|
+
if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
|
|
129
|
+
throw new PdfCaptureError("integrity", `${label} must not traverse a linked or non-directory ancestor`);
|
|
130
|
+
}
|
|
131
|
+
const canonical = await realpath(current);
|
|
132
|
+
assertAssetVerificationDeadline(deadline);
|
|
133
|
+
if (canonical !== current) {
|
|
134
|
+
throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
await assertBundleRoot(root, deadline);
|
|
138
|
+
}
|
|
139
|
+
async function withRegularFile(root, relativePath, maximumBytes, label, read, deadline, missingError) {
|
|
140
|
+
await assertAncestorChain(root, relativePath, label, deadline);
|
|
141
|
+
const path = confined(root, relativePath, label);
|
|
142
|
+
assertAssetVerificationDeadline(deadline);
|
|
143
|
+
let before;
|
|
144
|
+
try {
|
|
145
|
+
before = await lstat(path, { bigint: true });
|
|
146
|
+
} catch (error) {
|
|
147
|
+
assertAssetVerificationDeadline(deadline);
|
|
148
|
+
if (missingError !== undefined && isMissingPathError(error))
|
|
149
|
+
throw missingError;
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
assertAssetVerificationDeadline(deadline);
|
|
153
|
+
if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n) {
|
|
154
|
+
throw new PdfCaptureError("integrity", `${label} must be a regular single-link file`);
|
|
155
|
+
}
|
|
156
|
+
if (before.size > BigInt(maximumBytes))
|
|
157
|
+
throw new PdfCaptureError("integrity", `${label} exceeds ${maximumBytes} bytes`);
|
|
158
|
+
const canonical = await realpath(path);
|
|
159
|
+
assertAssetVerificationDeadline(deadline);
|
|
160
|
+
if (canonical !== path)
|
|
161
|
+
throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`);
|
|
162
|
+
const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
163
|
+
try {
|
|
164
|
+
assertAssetVerificationDeadline(deadline);
|
|
165
|
+
const opened = await handle.stat({ bigint: true });
|
|
166
|
+
assertAssetVerificationDeadline(deadline);
|
|
167
|
+
if (!opened.isFile() || opened.nlink !== 1n || opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size)
|
|
168
|
+
throw new PdfCaptureError("integrity", `${label} changed while it was opened`);
|
|
169
|
+
await assertAncestorChain(root, relativePath, label, deadline);
|
|
170
|
+
const result = await read(handle, Number(opened.size));
|
|
171
|
+
assertAssetVerificationDeadline(deadline);
|
|
172
|
+
const after = await handle.stat({ bigint: true });
|
|
173
|
+
assertAssetVerificationDeadline(deadline);
|
|
174
|
+
const named = await lstat(path, { bigint: true });
|
|
175
|
+
assertAssetVerificationDeadline(deadline);
|
|
176
|
+
await assertAncestorChain(root, relativePath, label, deadline);
|
|
177
|
+
const finalCanonical = await realpath(path);
|
|
178
|
+
assertAssetVerificationDeadline(deadline);
|
|
179
|
+
if (after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.nlink !== 1n || named.dev !== before.dev || named.ino !== before.ino || named.size !== before.size || named.mtimeNs !== before.mtimeNs || named.nlink !== 1n || named.isSymbolicLink() || finalCanonical !== path)
|
|
180
|
+
throw new PdfCaptureError("integrity", `${label} changed while it was read`);
|
|
181
|
+
return result;
|
|
182
|
+
} finally {
|
|
183
|
+
await handle.close();
|
|
184
|
+
assertAssetVerificationDeadline(deadline);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
async function readRegularFile(root, relativePath, maximumBytes, label, deadline) {
|
|
188
|
+
return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => {
|
|
189
|
+
const bytes = Buffer.alloc(size);
|
|
190
|
+
let offset = 0;
|
|
191
|
+
while (offset < bytes.byteLength) {
|
|
192
|
+
assertAssetVerificationDeadline(deadline);
|
|
193
|
+
const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset);
|
|
194
|
+
if (result.bytesRead === 0)
|
|
195
|
+
throw new PdfCaptureError("integrity", `${label} changed while it was read`);
|
|
196
|
+
offset += result.bytesRead;
|
|
197
|
+
}
|
|
198
|
+
return bytes;
|
|
199
|
+
}, deadline);
|
|
200
|
+
}
|
|
201
|
+
async function hashRegularFile(root, relativePath, maximumBytes, deadline, label) {
|
|
202
|
+
assertAssetVerificationDeadline(deadline);
|
|
203
|
+
await assertBundleRoot(root, deadline);
|
|
204
|
+
assertAssetVerificationDeadline(deadline);
|
|
205
|
+
await assertAncestorChain(root, relativePath, label, deadline);
|
|
206
|
+
const path = confined(root, relativePath, label);
|
|
207
|
+
assertAssetVerificationDeadline(deadline);
|
|
208
|
+
try {
|
|
209
|
+
await lstat(path, { bigint: true });
|
|
210
|
+
} catch (error) {
|
|
211
|
+
assertAssetVerificationDeadline(deadline);
|
|
212
|
+
if (isMissingPathError(error)) {
|
|
213
|
+
await assertBundleRoot(root, deadline);
|
|
214
|
+
throw new MissingCaptureAssetError(`capture asset ${relativePath} is missing`);
|
|
215
|
+
}
|
|
216
|
+
throw error;
|
|
217
|
+
}
|
|
218
|
+
assertAssetVerificationDeadline(deadline);
|
|
219
|
+
return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => {
|
|
220
|
+
assertAssetVerificationDeadline(deadline);
|
|
221
|
+
const hash = createHash("sha256");
|
|
222
|
+
const chunk = Buffer.allocUnsafe(Math.min(1 * 1024 * 1024, Math.max(1, size)));
|
|
223
|
+
let offset = 0;
|
|
224
|
+
while (offset < size) {
|
|
225
|
+
assertAssetVerificationDeadline(deadline);
|
|
226
|
+
const result = await handle.read(chunk, 0, Math.min(chunk.byteLength, size - offset), offset);
|
|
227
|
+
assertAssetVerificationDeadline(deadline);
|
|
228
|
+
if (result.bytesRead === 0)
|
|
229
|
+
throw new PdfCaptureError("integrity", `${label} changed while it was read`);
|
|
230
|
+
hash.update(chunk.subarray(0, result.bytesRead));
|
|
231
|
+
offset += result.bytesRead;
|
|
232
|
+
}
|
|
233
|
+
assertAssetVerificationDeadline(deadline);
|
|
234
|
+
return { bytes: size, sha256: hash.digest("hex") };
|
|
235
|
+
}, deadline, new MissingCaptureAssetError(`capture asset ${relativePath} is missing`));
|
|
236
|
+
}
|
|
237
|
+
function digest(bytes) {
|
|
238
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
239
|
+
}
|
|
240
|
+
function invalid(message) {
|
|
241
|
+
throw new PdfCaptureError("invalid-input", message);
|
|
242
|
+
}
|
|
243
|
+
function object(value) {
|
|
244
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
245
|
+
invalid("PDF bundle requires an object");
|
|
246
|
+
return value;
|
|
247
|
+
}
|
|
248
|
+
function integer(value, maximum, minimum = 0) {
|
|
249
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum)
|
|
250
|
+
invalid("PDF bundle has an invalid count or page reference");
|
|
251
|
+
return value;
|
|
252
|
+
}
|
|
253
|
+
function descriptor(value) {
|
|
254
|
+
const entry = object(value);
|
|
255
|
+
if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(entry.sha256))
|
|
256
|
+
invalid("PDF bundle has an invalid digest");
|
|
257
|
+
return { path: confinedRelativePath(entry.path, "PDF artifact"), bytes: integer(entry.bytes, 8 * 1024 ** 3), sha256: entry.sha256 };
|
|
258
|
+
}
|
|
259
|
+
function decode(bytes) {
|
|
260
|
+
try {
|
|
261
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
262
|
+
} catch {
|
|
263
|
+
return invalid("PDF bundle text is not UTF-8");
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function parse(bytes) {
|
|
267
|
+
try {
|
|
268
|
+
return JSON.parse(decode(bytes));
|
|
269
|
+
} catch (error) {
|
|
270
|
+
if (error instanceof PdfCaptureError)
|
|
271
|
+
throw error;
|
|
272
|
+
return invalid("PDF bundle JSON is invalid");
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async function inspectPdfCaptureBundle(path, options = {}) {
|
|
276
|
+
const maxDocument = boundedInteger(options.maxDocumentBytes, 64 * 1024 ** 2, 256 * 1024 ** 2, "maxDocumentBytes");
|
|
277
|
+
const maxSource = boundedInteger(options.maxSourceBytes, 512 * 1024 ** 2, 8 * 1024 ** 3, "maxSourceBytes");
|
|
278
|
+
const maxAsset = boundedInteger(options.maxAssetBytes, 100 * 1024 ** 2, 8 * 1024 ** 3, "maxAssetBytes");
|
|
279
|
+
const maxTotal = boundedInteger(options.maxTotalAssetBytes, 500 * 1024 ** 2, 8 * 1024 ** 3, "maxTotalAssetBytes");
|
|
280
|
+
const maxAssets = boundedInteger(options.maxVerifiedAssets, 1000, 1e4, "maxVerifiedAssets");
|
|
281
|
+
const deadline = performance.now() + boundedInteger(options.maxVerificationMs, 30000, 300000, "maxVerificationMs");
|
|
282
|
+
const root = await safeDirectory(path);
|
|
283
|
+
try {
|
|
284
|
+
const manifest = object(parse(await readRegularFile(root, "capture.json", 4 * 1024 ** 2, "PDF manifest", deadline)));
|
|
285
|
+
if (manifest.schemaVersion !== 1)
|
|
286
|
+
throw new PdfCaptureError("unsupported-schema", "Unsupported PDF manifest schema");
|
|
287
|
+
if (manifest.kind !== "pdf" || manifest.status !== "complete" && manifest.status !== "partial")
|
|
288
|
+
invalid("Invalid PDF manifest kind or status");
|
|
289
|
+
const document = object(manifest.document);
|
|
290
|
+
const pageCount = integer(document.pageCount, 1e5, 1);
|
|
291
|
+
const processedPages = integer(document.processedPages, pageCount, 1);
|
|
292
|
+
if (processedPages < pageCount && manifest.status !== "partial")
|
|
293
|
+
invalid("Truncated PDF cannot claim complete");
|
|
294
|
+
if (manifest.quality !== undefined) {
|
|
295
|
+
const quality = object(manifest.quality);
|
|
296
|
+
if (quality.schemaVersion !== 1)
|
|
297
|
+
throw new PdfCaptureError("unsupported-schema", "Unsupported PDF quality schema");
|
|
298
|
+
if (!Array.isArray(quality.issues) || quality.issues.length > 1e5)
|
|
299
|
+
invalid("Invalid PDF quality issues");
|
|
300
|
+
for (const raw of quality.issues) {
|
|
301
|
+
const issue = object(raw);
|
|
302
|
+
if (!["reading-order-uncertain", "table-structure-uncertain", "math-structure-uncertain", "ocr-needed"].includes(String(issue.code)))
|
|
303
|
+
invalid("Unknown PDF quality issue");
|
|
304
|
+
integer(issue.page, processedPages, 1);
|
|
305
|
+
}
|
|
306
|
+
if (quality.issues.length > 0 && manifest.status !== "partial")
|
|
307
|
+
invalid("Uncertain PDF cannot claim complete");
|
|
308
|
+
}
|
|
309
|
+
const sourceDescriptor = descriptor(manifest.source);
|
|
310
|
+
if (sourceDescriptor.path !== "source.pdf" || object(manifest.source).mimeType !== "application/pdf")
|
|
311
|
+
invalid("Invalid retained PDF source");
|
|
312
|
+
const check = async (expected, maximum) => {
|
|
313
|
+
if (expected.bytes > maximum)
|
|
314
|
+
throw new PdfCaptureError("limit-exceeded", "PDF artifact exceeds verification byte limit");
|
|
315
|
+
const actual = await hashRegularFile(root, expected.path, maximum, deadline, "PDF artifact");
|
|
316
|
+
return { ...actual, path: expected.path, integrity: actual.bytes === expected.bytes && actual.sha256 === expected.sha256 ? "verified" : "mismatch" };
|
|
317
|
+
};
|
|
318
|
+
const source = await check(sourceDescriptor, maxSource);
|
|
319
|
+
let expectedMarkdown = null;
|
|
320
|
+
let markdownPath;
|
|
321
|
+
if (manifest.markdown !== undefined) {
|
|
322
|
+
expectedMarkdown = descriptor(manifest.markdown);
|
|
323
|
+
markdownPath = expectedMarkdown.path;
|
|
324
|
+
if (markdownPath.includes("/") || !markdownPath.endsWith(".md"))
|
|
325
|
+
invalid("Invalid PDF Markdown path");
|
|
326
|
+
} else {
|
|
327
|
+
const directory = await opendir(root.path);
|
|
328
|
+
const candidates = [];
|
|
329
|
+
let entries = 0;
|
|
330
|
+
try {
|
|
331
|
+
for await (const entry of directory) {
|
|
332
|
+
assertAssetVerificationDeadline(deadline);
|
|
333
|
+
if (++entries > 1000)
|
|
334
|
+
throw new PdfCaptureError("limit-exceeded", "Legacy PDF directory exceeds entry limit");
|
|
335
|
+
if (entry.name.endsWith(".md"))
|
|
336
|
+
candidates.push(entry.name);
|
|
337
|
+
}
|
|
338
|
+
} finally {
|
|
339
|
+
try {
|
|
340
|
+
await directory.close();
|
|
341
|
+
} catch {}
|
|
342
|
+
}
|
|
343
|
+
if (candidates.length !== 1)
|
|
344
|
+
invalid("Legacy PDF bundle must contain exactly one Markdown document");
|
|
345
|
+
markdownPath = candidates[0];
|
|
346
|
+
}
|
|
347
|
+
const markdownBytes = await readRegularFile(root, markdownPath, maxDocument, "PDF Markdown", deadline);
|
|
348
|
+
const markdownText = decode(markdownBytes);
|
|
349
|
+
const markdownDigest = digest(markdownBytes);
|
|
350
|
+
const markdown = {
|
|
351
|
+
path: markdownPath,
|
|
352
|
+
bytes: markdownBytes.length,
|
|
353
|
+
sha256: markdownDigest,
|
|
354
|
+
markdown: markdownText,
|
|
355
|
+
integrity: expectedMarkdown === null ? "unavailable" : expectedMarkdown.bytes === markdownBytes.length && expectedMarkdown.sha256 === markdownDigest ? "verified" : "mismatch"
|
|
356
|
+
};
|
|
357
|
+
const assetReferences = new Map;
|
|
358
|
+
let currentPage = 0;
|
|
359
|
+
for (const match of markdownText.matchAll(/<!-- pdf-page: (\d+) -->|!\[[^\]\n]*\]\((assets\/[^)\r\n]+)\)/gu)) {
|
|
360
|
+
if (match[1] !== undefined)
|
|
361
|
+
currentPage = integer(Number(match[1]), processedPages, 1);
|
|
362
|
+
else if (match[2] !== undefined) {
|
|
363
|
+
const pages = assetReferences.get(match[2]) ?? new Set;
|
|
364
|
+
pages.add(currentPage);
|
|
365
|
+
assetReferences.set(match[2], pages);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
const anchors = new Set;
|
|
369
|
+
let anchorCount = 0;
|
|
370
|
+
for (const match of markdownText.matchAll(/<!-- pdf-page: (\d+) -->/gu)) {
|
|
371
|
+
if (++anchorCount > 1e5)
|
|
372
|
+
invalid("Too many PDF page anchors");
|
|
373
|
+
anchors.add(integer(Number(match[1]), processedPages, 1));
|
|
374
|
+
}
|
|
375
|
+
if (!Array.isArray(manifest.images) || manifest.images.length > maxAssets)
|
|
376
|
+
invalid("Invalid PDF image count");
|
|
377
|
+
const imageIds = new Map;
|
|
378
|
+
const assetsByPath = new Map;
|
|
379
|
+
const declaredPages = new Map;
|
|
380
|
+
let totalBytes = 0;
|
|
381
|
+
for (const raw of manifest.images) {
|
|
382
|
+
const image = object(raw);
|
|
383
|
+
if (typeof image.id !== "string" || !/^[a-z0-9][a-z0-9._:-]{0,255}$/u.test(image.id) || imageIds.has(image.id))
|
|
384
|
+
invalid("Invalid or duplicate PDF image ID");
|
|
385
|
+
const page = integer(image.page, processedPages, 1);
|
|
386
|
+
if (!anchors.has(page))
|
|
387
|
+
invalid("PDF image has no corresponding Markdown page anchor");
|
|
388
|
+
const asset = descriptor(image.asset);
|
|
389
|
+
if (!/^assets\/[^/]+$/u.test(asset.path))
|
|
390
|
+
invalid("PDF image must be a direct assets child");
|
|
391
|
+
if (!assetReferences.get(asset.path)?.has(page))
|
|
392
|
+
invalid("PDF image has no corresponding Markdown asset reference");
|
|
393
|
+
imageIds.set(image.id, asset.sha256);
|
|
394
|
+
const pages = declaredPages.get(asset.path) ?? new Set;
|
|
395
|
+
pages.add(page);
|
|
396
|
+
declaredPages.set(asset.path, pages);
|
|
397
|
+
const existing = assetsByPath.get(asset.path);
|
|
398
|
+
if (existing !== undefined) {
|
|
399
|
+
if (existing.bytes !== asset.bytes || existing.sha256 !== asset.sha256)
|
|
400
|
+
invalid("Conflicting PDF asset descriptors");
|
|
401
|
+
} else {
|
|
402
|
+
totalBytes += asset.bytes;
|
|
403
|
+
if (totalBytes > maxTotal)
|
|
404
|
+
throw new PdfCaptureError("limit-exceeded", "PDF assets exceed total byte limit");
|
|
405
|
+
const checked = await check(asset, Math.min(maxAsset, maxTotal - (totalBytes - asset.bytes)));
|
|
406
|
+
totalBytes += checked.bytes - asset.bytes;
|
|
407
|
+
assetsByPath.set(asset.path, checked);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
for (const [path2, pages] of assetReferences) {
|
|
411
|
+
if (!assetsByPath.has(path2))
|
|
412
|
+
invalid("Markdown references an undeclared PDF asset");
|
|
413
|
+
if ([...pages].some((page) => !declaredPages.get(path2)?.has(page)))
|
|
414
|
+
invalid("Markdown asset page disagrees with PDF manifest");
|
|
415
|
+
}
|
|
416
|
+
const extraction = object(manifest.extraction);
|
|
417
|
+
if (integer(extraction.imageCount, maxAssets) !== imageIds.size)
|
|
418
|
+
invalid("PDF image count disagrees with manifest");
|
|
419
|
+
let annotations = null;
|
|
420
|
+
if (manifest.annotations !== null) {
|
|
421
|
+
const annotationDescriptor = descriptor(manifest.annotations);
|
|
422
|
+
if (annotationDescriptor.path !== "annotations.json")
|
|
423
|
+
invalid("Invalid PDF annotations path");
|
|
424
|
+
const annotationBytes = await readRegularFile(root, "annotations.json", 16 * 1024 ** 2, "PDF annotations", deadline);
|
|
425
|
+
const annotationHash = digest(annotationBytes);
|
|
426
|
+
annotations = { path: annotationDescriptor.path, bytes: annotationBytes.length, sha256: annotationHash, integrity: annotationDescriptor.bytes === annotationBytes.length && annotationDescriptor.sha256 === annotationHash ? "verified" : "mismatch" };
|
|
427
|
+
let retained;
|
|
428
|
+
try {
|
|
429
|
+
retained = parsePdfImageInterpretations(parse(annotationBytes));
|
|
430
|
+
} catch (error) {
|
|
431
|
+
if (error instanceof PdfCaptureError)
|
|
432
|
+
throw error;
|
|
433
|
+
throw new PdfCaptureError("invalid-input", "Invalid PDF image annotations", { cause: error });
|
|
434
|
+
}
|
|
435
|
+
if (retained.length !== integer(object(manifest.annotations).count, 1e4))
|
|
436
|
+
invalid("PDF annotation count disagrees with manifest");
|
|
437
|
+
for (const annotation of retained) {
|
|
438
|
+
if (imageIds.get(annotation.id) !== annotation.sha256)
|
|
439
|
+
invalid("PDF annotation references an unknown image or digest");
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
await assertBundleRoot(root, deadline);
|
|
443
|
+
return { root: root.path, schemaVersion: 1, status: manifest.status, pageCount, processedPages, markdown, source, assets: [...assetsByPath.values()], annotations };
|
|
444
|
+
} finally {
|
|
445
|
+
await root.handle.close();
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
async function verifyPdfCaptureBundle(path, options = {}) {
|
|
449
|
+
const inspection = await inspectPdfCaptureBundle(path, options);
|
|
450
|
+
const issues = [];
|
|
451
|
+
const add = (file, kind) => {
|
|
452
|
+
if (file.integrity !== "verified")
|
|
453
|
+
issues.push({ kind, path: file.path, integrity: file.integrity });
|
|
454
|
+
};
|
|
455
|
+
add(inspection.markdown, "document-integrity");
|
|
456
|
+
add(inspection.source, "source-integrity");
|
|
457
|
+
for (const asset of inspection.assets)
|
|
458
|
+
add(asset, "asset-integrity");
|
|
459
|
+
if (inspection.annotations !== null)
|
|
460
|
+
add(inspection.annotations, "annotation-integrity");
|
|
461
|
+
return { ok: issues.length === 0, inspection, issues };
|
|
462
|
+
}
|
|
463
|
+
|
|
36
464
|
// src/pdf.ts
|
|
37
465
|
var PDF_CAPTURE_MANIFEST_FILENAME2 = PDF_CAPTURE_MANIFEST_FILENAME;
|
|
38
466
|
var PDF_CAPTURE_MANIFEST_SCHEMA_VERSION2 = PDF_CAPTURE_MANIFEST_SCHEMA_VERSION;
|
|
@@ -58,7 +486,11 @@ var resolvePdfTools2 = resolvePdfTools;
|
|
|
58
486
|
var runPdfCapture2 = runPdfCapture;
|
|
59
487
|
var runPdfCommand = main;
|
|
60
488
|
var runPdfToolCommand2 = runPdfToolCommand;
|
|
489
|
+
var inspectPdfCaptureBundle2 = (...arguments_) => inspectPdfCaptureBundle(...arguments_);
|
|
490
|
+
var verifyPdfCaptureBundle2 = (...arguments_) => verifyPdfCaptureBundle(...arguments_);
|
|
491
|
+
var PdfCaptureError2 = PdfCaptureError;
|
|
61
492
|
export {
|
|
493
|
+
verifyPdfCaptureBundle2 as verifyPdfCaptureBundle,
|
|
62
494
|
runPdfToolCommand2 as runPdfToolCommand,
|
|
63
495
|
runPdfCommand,
|
|
64
496
|
runPdfCapture2 as runPdfCapture,
|
|
@@ -76,9 +508,11 @@ export {
|
|
|
76
508
|
parsePdfArguments2 as parsePdfArguments,
|
|
77
509
|
ocrPdfImage2 as ocrPdfImage,
|
|
78
510
|
layoutBlocks2 as layoutBlocks,
|
|
511
|
+
inspectPdfCaptureBundle2 as inspectPdfCaptureBundle,
|
|
79
512
|
inspectPdf2 as inspectPdf,
|
|
80
513
|
decodePopplerText2 as decodePopplerText,
|
|
81
514
|
buildPdfMarkdown2 as buildPdfMarkdown,
|
|
515
|
+
PdfCaptureError2 as PdfCaptureError,
|
|
82
516
|
PDF_CAPTURE_SOURCE_FILENAME2 as PDF_CAPTURE_SOURCE_FILENAME,
|
|
83
517
|
PDF_CAPTURE_MANIFEST_SCHEMA_VERSION2 as PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
|
|
84
518
|
PDF_CAPTURE_MANIFEST_FILENAME2 as PDF_CAPTURE_MANIFEST_FILENAME,
|
package/dist/portfolio.js
CHANGED
|
@@ -20,11 +20,11 @@ import {
|
|
|
20
20
|
selectAuthorizedVaults,
|
|
21
21
|
snapshotPortfolioRegistry,
|
|
22
22
|
validateResolvedPortfolioVaults
|
|
23
|
-
} from "./index-
|
|
24
|
-
import"./index-
|
|
23
|
+
} from "./index-3hts6yy9.js";
|
|
24
|
+
import"./index-7bqwzjba.js";
|
|
25
|
+
import"./index-4j3tt0c3.js";
|
|
25
26
|
import"./index-adx6khj5.js";
|
|
26
27
|
import"./index-fp732bgg.js";
|
|
27
|
-
import"./index-4j3tt0c3.js";
|
|
28
28
|
import"./index-wqx1x32f.js";
|
|
29
29
|
import"./index-3gc2yk4k.js";
|
|
30
30
|
import"./index-pgtm2nhf.js";
|