@arnilo/prism-tools 0.0.0-stage → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +418 -0
- package/LICENSE +21 -0
- package/README.md +44 -2
- package/browser/README.md +98 -0
- package/browser/skills/browser-automation/SKILL.md +14 -0
- package/browser/src/actions/cdp.ts +131 -0
- package/browser/src/actions/dialog.ts +25 -0
- package/browser/src/actions/download_release.ts +22 -0
- package/browser/src/actions/index.ts +37 -0
- package/browser/src/actions/locator.ts +70 -0
- package/browser/src/actions/navigate.ts +23 -0
- package/browser/src/actions/screenshot.ts +27 -0
- package/browser/src/actions/scroll.ts +24 -0
- package/browser/src/actions/select_page.ts +21 -0
- package/browser/src/actions/types.ts +114 -0
- package/browser/src/actions/upload.ts +44 -0
- package/browser/src/actions/wait.ts +45 -0
- package/browser/src/browser.ts +197 -0
- package/browser/src/cdp.ts +178 -0
- package/browser/src/checkpoint.ts +189 -0
- package/browser/src/config.ts +72 -0
- package/browser/src/downloads.ts +253 -0
- package/browser/src/errors.ts +31 -0
- package/browser/src/evaluate.ts +101 -0
- package/browser/src/index.ts +129 -0
- package/browser/src/limits.ts +271 -0
- package/browser/src/manager.ts +730 -0
- package/browser/src/network.ts +263 -0
- package/browser/src/observe.ts +178 -0
- package/browser/src/policy.ts +133 -0
- package/browser/src/screenshot.ts +107 -0
- package/browser/src/shared-sandbox.ts +53 -0
- package/browser/src/snapshot.ts +99 -0
- package/browser/src/targets.ts +165 -0
- package/browser/src/tools.ts +480 -0
- package/browser/src/types.ts +316 -0
- package/browser/src/uploads.ts +174 -0
- package/coding/README.md +337 -0
- package/coding/src/agent/acp-operations.ts +146 -0
- package/coding/src/agent/artifacts.ts +38 -0
- package/coding/src/agent/ask-user-decision.ts +623 -0
- package/coding/src/agent/atomic-write.ts +23 -0
- package/coding/src/agent/bounded-file.ts +21 -0
- package/coding/src/agent/checks.ts +301 -0
- package/coding/src/agent/coding-checkpoint.ts +828 -0
- package/coding/src/agent/delete.ts +237 -0
- package/coding/src/agent/diagnostics-tool.ts +163 -0
- package/coding/src/agent/diagnostics.ts +262 -0
- package/coding/src/agent/edit-diff.ts +382 -0
- package/coding/src/agent/edit.ts +322 -0
- package/coding/src/agent/effects.ts +102 -0
- package/coding/src/agent/env.ts +28 -0
- package/coding/src/agent/execution-policy.ts +89 -0
- package/coding/src/agent/file-mutation-queue.ts +72 -0
- package/coding/src/agent/forge/github.ts +603 -0
- package/coding/src/agent/forge/index.ts +16 -0
- package/coding/src/agent/forge/types.ts +185 -0
- package/coding/src/agent/git-aware-repository.ts +305 -0
- package/coding/src/agent/git-exec.ts +326 -0
- package/coding/src/agent/git-status.ts +177 -0
- package/coding/src/agent/git-tools.ts +360 -0
- package/coding/src/agent/git.ts +1074 -0
- package/coding/src/agent/glob-match.ts +146 -0
- package/coding/src/agent/glob.ts +182 -0
- package/coding/src/agent/goal-verify.ts +385 -0
- package/coding/src/agent/grep-worker.ts +74 -0
- package/coding/src/agent/grep.ts +459 -0
- package/coding/src/agent/identity.ts +48 -0
- package/coding/src/agent/index.ts +676 -0
- package/coding/src/agent/language/client.ts +429 -0
- package/coding/src/agent/language/framing.ts +190 -0
- package/coding/src/agent/language/index.ts +23 -0
- package/coding/src/agent/language/intelligence.ts +682 -0
- package/coding/src/agent/language/types.ts +187 -0
- package/coding/src/agent/lifecycle-events.ts +121 -0
- package/coding/src/agent/lifecycle.ts +166 -0
- package/coding/src/agent/limits.ts +214 -0
- package/coding/src/agent/move.ts +178 -0
- package/coding/src/agent/mutation-path.ts +52 -0
- package/coding/src/agent/output-accumulator.ts +304 -0
- package/coding/src/agent/path-utils.ts +120 -0
- package/coding/src/agent/process/index.ts +58 -0
- package/coding/src/agent/process/recovery.ts +508 -0
- package/coding/src/agent/process/sessions-host.ts +222 -0
- package/coding/src/agent/process/sessions-monitor.ts +141 -0
- package/coding/src/agent/process/sessions-recovery.ts +241 -0
- package/coding/src/agent/process/sessions-spawn.ts +613 -0
- package/coding/src/agent/process/sessions-teardown.ts +281 -0
- package/coding/src/agent/process/sessions.ts +30 -0
- package/coding/src/agent/process/types.ts +358 -0
- package/coding/src/agent/read-path-set.ts +126 -0
- package/coding/src/agent/read.ts +713 -0
- package/coding/src/agent/repository/glob.ts +147 -0
- package/coding/src/agent/repository/indexed-search.ts +480 -0
- package/coding/src/agent/repository/list.ts +131 -0
- package/coding/src/agent/repository/operations.ts +22 -0
- package/coding/src/agent/repository/path.ts +98 -0
- package/coding/src/agent/repository/search.ts +277 -0
- package/coding/src/agent/repository/types.ts +243 -0
- package/coding/src/agent/repository/walk.ts +126 -0
- package/coding/src/agent/repository.ts +12 -0
- package/coding/src/agent/shell-background.ts +264 -0
- package/coding/src/agent/shell-output.ts +75 -0
- package/coding/src/agent/shell.ts +526 -0
- package/coding/src/agent/todo.ts +399 -0
- package/coding/src/agent/truncate.ts +251 -0
- package/coding/src/agent/write.ts +170 -0
- package/coding/src/ask-user-host.ts +48 -0
- package/coding/src/config.ts +278 -0
- package/coding/src/hard-deny.ts +97 -0
- package/coding/src/index.ts +318 -0
- package/coding/src/principal-guard.ts +84 -0
- package/coding/src/sandbox.ts +92 -0
- package/coding/src/security/approval.ts +144 -0
- package/coding/src/security/command-rules.ts +57 -0
- package/coding/src/security/docker-cli.ts +222 -0
- package/coding/src/security/docker-sandbox.ts +1205 -0
- package/coding/src/security/e2b-sandbox.ts +853 -0
- package/coding/src/security/egress/dns-pin.ts +88 -0
- package/coding/src/security/egress/index.ts +32 -0
- package/coding/src/security/egress/limits.ts +57 -0
- package/coding/src/security/egress/policy.ts +137 -0
- package/coding/src/security/egress/proxy.ts +599 -0
- package/coding/src/security/egress/types.ts +30 -0
- package/coding/src/security/index.ts +199 -0
- package/coding/src/security/native-sandbox.ts +639 -0
- package/coding/src/security/path-containment.ts +55 -0
- package/coding/src/security/sandbox-coding-operations.ts +392 -0
- package/coding/src/security/sandbox-fs-operations.ts +644 -0
- package/coding/src/security/sandbox-limits.ts +147 -0
- package/coding/src/security/sandbox-tar.ts +308 -0
- package/coding/src/security/sandbox.ts +179 -0
- package/coding/src/security/semaphore.ts +55 -0
- package/coding/src/targets.ts +110 -0
- package/coding/src/workspace/diff.ts +78 -0
- package/coding/src/workspace/errors.ts +29 -0
- package/coding/src/workspace/index.ts +5 -0
- package/coding/src/workspace/merge.ts +151 -0
- package/coding/src/workspace/routing.ts +76 -0
- package/coding/src/workspace/service.ts +584 -0
- package/computer-use/README.md +105 -0
- package/computer-use/skills/computer-use-linux/SKILL.md +17 -0
- package/computer-use/src/classify.ts +53 -0
- package/computer-use/src/command.ts +29 -0
- package/computer-use/src/config.ts +69 -0
- package/computer-use/src/create.ts +170 -0
- package/computer-use/src/devices.ts +247 -0
- package/computer-use/src/env.ts +28 -0
- package/computer-use/src/index.ts +157 -0
- package/computer-use/src/skill.ts +21 -0
- package/openapi/README.md +97 -0
- package/openapi/src/compile.ts +386 -0
- package/openapi/src/config.ts +73 -0
- package/openapi/src/errors.ts +16 -0
- package/openapi/src/execute.ts +241 -0
- package/openapi/src/index.ts +124 -0
- package/openapi/src/limits.ts +63 -0
- package/openapi/src/specs.ts +86 -0
- package/openapi/src/tools.ts +60 -0
- package/openapi/src/types.ts +61 -0
- package/package.json +171 -4
- package/research/README.md +72 -0
- package/research/skills/research-report/SKILL.md +12 -0
- package/research/skills/research-scoping/SKILL.md +10 -0
- package/research/skills/research-synthesis/SKILL.md +10 -0
- package/research/skills/source-triage/SKILL.md +10 -0
- package/research/src/check.ts +71 -0
- package/research/src/config.ts +41 -0
- package/research/src/index.ts +147 -0
- package/research/src/ledger.ts +157 -0
- package/research/src/urls.ts +21 -0
- package/web/README.md +287 -0
- package/web/skills/web-research/SKILL.md +17 -0
- package/web/src/brave.ts +66 -0
- package/web/src/cache.ts +151 -0
- package/web/src/config.ts +210 -0
- package/web/src/controls.ts +128 -0
- package/web/src/domains.ts +114 -0
- package/web/src/exa.ts +60 -0
- package/web/src/firecrawl.ts +160 -0
- package/web/src/index.ts +387 -0
- package/web/src/kagi.ts +73 -0
- package/web/src/limits.ts +55 -0
- package/web/src/native/convert.ts +84 -0
- package/web/src/native/fetch.ts +226 -0
- package/web/src/normalize.ts +131 -0
- package/web/src/obscura/cdp.ts +201 -0
- package/web/src/obscura/classify.ts +35 -0
- package/web/src/obscura/cli.ts +254 -0
- package/web/src/obscura/errors.ts +8 -0
- package/web/src/obscura/index.ts +59 -0
- package/web/src/obscura/limits.ts +92 -0
- package/web/src/obscura/mcp.ts +104 -0
- package/web/src/obscura/playwright.ts +133 -0
- package/web/src/obscura/process.ts +235 -0
- package/web/src/obscura/search-profile.ts +73 -0
- package/web/src/obscura/types.ts +48 -0
- package/web/src/obscura/web.ts +419 -0
- package/web/src/prompt-fetch.ts +88 -0
- package/web/src/render.ts +75 -0
- package/web/src/searxng.ts +78 -0
- package/web/src/slice.ts +139 -0
- package/web/src/tavily.ts +77 -0
- package/web/src/tools.ts +182 -0
- package/web/src/transport.ts +310 -0
- package/web/src/types.ts +199 -0
- package/work/README.md +77 -0
- package/work/THIRD_PARTY_NOTICES +24 -0
- package/work/docling/ocr.py +150 -0
- package/work/sandbox/Dockerfile +36 -0
- package/work/sandbox/soffice.sh +12 -0
- package/work/src/command.ts +29 -0
- package/work/src/config.ts +138 -0
- package/work/src/connectors/cli.ts +343 -0
- package/work/src/connectors/drafts.ts +647 -0
- package/work/src/connectors/errors.ts +8 -0
- package/work/src/connectors/file-bytes.ts +149 -0
- package/work/src/connectors/google-workspace-http.ts +399 -0
- package/work/src/connectors/google-workspace.ts +465 -0
- package/work/src/connectors/gws-tools.ts +378 -0
- package/work/src/connectors/http.ts +123 -0
- package/work/src/connectors/idempotency.ts +235 -0
- package/work/src/connectors/identity.ts +34 -0
- package/work/src/connectors/index.ts +93 -0
- package/work/src/connectors/limits.ts +50 -0
- package/work/src/connectors/m365-tools.ts +308 -0
- package/work/src/connectors/microsoft365-http.ts +428 -0
- package/work/src/connectors/microsoft365.ts +402 -0
- package/work/src/connectors/normalize.ts +120 -0
- package/work/src/connectors/tool-helpers.ts +398 -0
- package/work/src/connectors/tools.ts +42 -0
- package/work/src/connectors/types.ts +399 -0
- package/work/src/diagrams/canonicalize.ts +166 -0
- package/work/src/diagrams/caps.ts +52 -0
- package/work/src/diagrams/embed.ts +340 -0
- package/work/src/diagrams/errors.ts +76 -0
- package/work/src/diagrams/index.ts +81 -0
- package/work/src/diagrams/messages.ts +150 -0
- package/work/src/diagrams/origin.ts +52 -0
- package/work/src/diagrams/telemetry.ts +34 -0
- package/work/src/diagrams/xml.ts +267 -0
- package/work/src/document-extraction/index.ts +563 -0
- package/work/src/document-reader/ambient.d.ts +2 -0
- package/work/src/document-reader/errors.ts +7 -0
- package/work/src/document-reader/index.ts +181 -0
- package/work/src/document-reader/mistral-ocr.ts +318 -0
- package/work/src/document-reader/parsers.ts +171 -0
- package/work/src/documents/caps.ts +135 -0
- package/work/src/documents/diff.ts +269 -0
- package/work/src/documents/errors.ts +44 -0
- package/work/src/documents/fidelity.ts +98 -0
- package/work/src/documents/generate.ts +96 -0
- package/work/src/documents/hash.ts +8 -0
- package/work/src/documents/index.ts +14 -0
- package/work/src/documents/model-schema.ts +639 -0
- package/work/src/documents/parse.ts +252 -0
- package/work/src/documents/patch-history.ts +81 -0
- package/work/src/documents/patch.ts +476 -0
- package/work/src/documents/preview-html.ts +245 -0
- package/work/src/documents/preview.ts +227 -0
- package/work/src/documents/telemetry.ts +36 -0
- package/work/src/documents/translate/docx.ts +292 -0
- package/work/src/documents/translate/pptx.ts +304 -0
- package/work/src/documents/translate/xlsx.ts +137 -0
- package/work/src/documents/types.ts +188 -0
- package/work/src/index.ts +311 -0
- package/work/src/runtime/worker-pool-entry.ts +102 -0
- package/work/src/runtime/worker-pool.ts +267 -0
- package/work/src/sandbox/composition.ts +212 -0
- package/work/src/sandbox/image.ts +6 -0
- package/work/src/sandbox/index.ts +14 -0
- package/work/src/sheets/caps.ts +68 -0
- package/work/src/sheets/csv.ts +351 -0
- package/work/src/sheets/decimal.ts +388 -0
- package/work/src/sheets/errors.ts +52 -0
- package/work/src/sheets/index.ts +62 -0
- package/work/src/sheets/inference.ts +378 -0
- package/work/src/sheets/telemetry.ts +34 -0
- package/work/src/sheets/types.ts +162 -0
- package/work/src/sheets/xlsx.ts +388 -0
- package/work/src/skills/index.ts +8 -0
- package/work/src/skills/load.ts +43 -0
- package/work/src/tools/filesystem.ts +21 -0
- package/work/src/tools/index.ts +1 -0
- package/work/src/tools/local-filesystem.ts +33 -0
- package/work/src/tools/office.ts +311 -0
- package/work/vendor/hermes-agent/LICENSE +21 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/LICENSE +21 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/SKILL.md +196 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_comments.py +289 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_common.py +94 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_create.py +177 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_edit.py +250 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_read.py +149 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_revisions.py +147 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_template.py +70 -0
- package/work/vendor/hermes-agent/skills/productivity/docx/scripts/docx_validate.py +156 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/LICENSE +21 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/SKILL.md +125 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/_raster.py +76 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/extract_marker.py +88 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/extract_pymupdf.py +101 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_create.py +130 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_fill_form.py +97 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_form_layout.py +168 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_make_form.py +145 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_merge.py +50 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_meta.py +115 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_page_image.py +99 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_read.py +153 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_secure.py +71 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_split.py +84 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_stamp.py +143 -0
- package/work/vendor/hermes-agent/skills/productivity/pdf/scripts/pdf_watermark.py +51 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/LICENSE +21 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/SKILL.md +220 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/scripts/pptx_create.py +214 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/scripts/pptx_edit.py +436 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/scripts/pptx_from_template.py +88 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/scripts/pptx_read.py +131 -0
- package/work/vendor/hermes-agent/skills/productivity/powerpoint/scripts/pptx_render.py +93 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/LICENSE +21 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/SKILL.md +196 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/csv_to_xlsx.py +104 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_create.py +259 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_edit.py +263 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_read.py +160 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_recalc.py +111 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_restructure.py +337 -0
- package/work/vendor/hermes-agent/skills/productivity/xlsx/scripts/xlsx_to_csv.py +69 -0
- package/work/vendor-lock.json +11 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded PDF/Office literal-text extraction adapter for the Prism coding
|
|
3
|
+
* read tool (plan 018 closeout `doc-reader`).
|
|
4
|
+
*
|
|
5
|
+
* Explicit activation only: the host wires the returned {@link DocumentReader}
|
|
6
|
+
* into `createReadTool({ documentReader })`; no file-extension sniffing ever
|
|
7
|
+
* enables parsing. Bounds live here (input bytes, pages, output text bytes);
|
|
8
|
+
* parsing is delegated to optional peer libraries (`pdf-parse`, `mammoth`)
|
|
9
|
+
* that fail closed at creation when absent. Literal text only — no embedded
|
|
10
|
+
* script execution, no macro evaluation, no external resource fetching.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { SecretRedactor } from "@arnilo/prism";
|
|
14
|
+
import { reviveWorkerTaskError, runInPool, shouldOffloadToPool } from "../runtime/worker-pool.js";
|
|
15
|
+
import { DocumentReaderError } from "./errors.js";
|
|
16
|
+
import { createDocxParser, createOoxmlParser, createPdfParser } from "./parsers.js";
|
|
17
|
+
|
|
18
|
+
export { createDocxParser, createOoxmlParser, createPdfParser } from "./parsers.js";
|
|
19
|
+
export { DocumentReaderError };
|
|
20
|
+
|
|
21
|
+
/** Structural reader contract accepted by coding-tools' host injection seam. */
|
|
22
|
+
export interface DocumentReader {
|
|
23
|
+
readonly maxInputBytes: number;
|
|
24
|
+
readonly maxTextBytes: number;
|
|
25
|
+
extract(input: {
|
|
26
|
+
readonly buffer: Uint8Array;
|
|
27
|
+
readonly path: string;
|
|
28
|
+
readonly signal?: AbortSignal;
|
|
29
|
+
}): Promise<DocumentReaderResult | null>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface DocumentReaderResult {
|
|
33
|
+
readonly text: string;
|
|
34
|
+
readonly format: string;
|
|
35
|
+
readonly pages: number;
|
|
36
|
+
readonly truncatedBy: "pages" | "bytes" | null;
|
|
37
|
+
readonly pageSpans?: readonly { readonly page: number; readonly start: number; readonly end: number }[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// --- caps (module-local; additive package, no shared limits surface) ---
|
|
41
|
+
|
|
42
|
+
export const DEFAULT_MAX_DOCUMENT_BYTES = 32 * 1024 * 1024;
|
|
43
|
+
export const HARD_MAX_DOCUMENT_BYTES = 512 * 1024 * 1024;
|
|
44
|
+
export const DEFAULT_MAX_DOCUMENT_PAGES = 1000;
|
|
45
|
+
export const HARD_MAX_DOCUMENT_PAGES = 10_000;
|
|
46
|
+
export const DEFAULT_MAX_DOCUMENT_TEXT_BYTES = 2 * 1024 * 1024;
|
|
47
|
+
export const HARD_MAX_DOCUMENT_TEXT_BYTES = 64 * 1024 * 1024;
|
|
48
|
+
|
|
49
|
+
function validateCap(name: string, value: number, hard: number): number {
|
|
50
|
+
if (!Number.isInteger(value) || value <= 0 || value > hard) {
|
|
51
|
+
throw new RangeError(`document-reader ${name} must be an integer in (0, ${hard}], got ${value}`);
|
|
52
|
+
}
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Host-selected per-format parser implementation. */
|
|
57
|
+
export interface DocumentParser {
|
|
58
|
+
readonly format: string;
|
|
59
|
+
/**
|
|
60
|
+
* Cheap magic-byte gate; false means "this buffer is not my format" (never reaches the parser).
|
|
61
|
+
* Receives `Uint8Array` so a worker-shaped payload works without a `Buffer.from` wrap; every
|
|
62
|
+
* `Buffer`-only helper (`toString`, `includes`) must go through a view first.
|
|
63
|
+
*/
|
|
64
|
+
detect(bytes: Uint8Array): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Extract literal text. Must never execute embedded content or fetch
|
|
67
|
+
* external resources. Over-page documents must refuse; over-text results
|
|
68
|
+
* must be truncated with `truncatedBy: "bytes"`.
|
|
69
|
+
*/
|
|
70
|
+
extract(
|
|
71
|
+
bytes: Uint8Array,
|
|
72
|
+
options: { readonly maxPages: number; readonly maxTextBytes: number; readonly signal?: AbortSignal },
|
|
73
|
+
): Promise<Omit<DocumentReaderResult, "format">>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface CreateDocumentReaderOptions {
|
|
77
|
+
/** Hard input size cap in bytes (default 32 MiB, hard ceiling 512 MiB). */
|
|
78
|
+
readonly maxBytes?: number;
|
|
79
|
+
/** Hard page/sheet cap for formats that report pages (default 1000, ceiling 10000). */
|
|
80
|
+
readonly maxPages?: number;
|
|
81
|
+
/** Hard extracted-literal-text cap in bytes (default 2 MiB, ceiling 64 MiB). */
|
|
82
|
+
readonly maxTextBytes?: number;
|
|
83
|
+
/** Host-selected parsers; default wiring uses the pdf-parse and mammoth optional peers. */
|
|
84
|
+
readonly parsers?: readonly DocumentParser[];
|
|
85
|
+
/** Optional redactor applied to extracted text at the adapter boundary. */
|
|
86
|
+
readonly redactor?: SecretRedactor;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Create a bounded document reader for `createReadTool({ documentReader })`.
|
|
91
|
+
* Throws {@link DocumentReaderError} when a selected format's peer parser is
|
|
92
|
+
* absent. Unsupported buffers return `null` from `extract` (the read tool
|
|
93
|
+
* falls through to its 0.1.5 text path).
|
|
94
|
+
*/
|
|
95
|
+
export async function createDocumentReader(options: CreateDocumentReaderOptions = {}): Promise<DocumentReader> {
|
|
96
|
+
const maxBytes = validateCap("maxBytes", options.maxBytes ?? DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES);
|
|
97
|
+
const maxPages = validateCap("maxPages", options.maxPages ?? DEFAULT_MAX_DOCUMENT_PAGES, HARD_MAX_DOCUMENT_PAGES);
|
|
98
|
+
const maxTextBytes = validateCap("maxTextBytes", options.maxTextBytes ?? DEFAULT_MAX_DOCUMENT_TEXT_BYTES, HARD_MAX_DOCUMENT_TEXT_BYTES);
|
|
99
|
+
const usesDefaultParsers = options.parsers === undefined;
|
|
100
|
+
const parsers = options.parsers ?? [
|
|
101
|
+
// Default wiring: the optional peers, probed once at creation (fail closed).
|
|
102
|
+
await createPdfParser(),
|
|
103
|
+
await createDocxParser(),
|
|
104
|
+
createOoxmlParser("xlsx", "sheet", "xl/workbook.xml"),
|
|
105
|
+
createOoxmlParser("pptx", "deck", "ppt/presentation.xml"),
|
|
106
|
+
];
|
|
107
|
+
if (parsers.length === 0) {
|
|
108
|
+
throw new DocumentReaderError("createDocumentReader requires at least one parser (or the optional peers installed)");
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return {
|
|
112
|
+
maxInputBytes: maxBytes,
|
|
113
|
+
maxTextBytes,
|
|
114
|
+
extract: async (input) => {
|
|
115
|
+
input.signal?.throwIfAborted();
|
|
116
|
+
if (input.buffer.byteLength > maxBytes) {
|
|
117
|
+
throw new DocumentReaderError(`document exceeds maxBytes cap (${maxBytes}); refusing to extract`);
|
|
118
|
+
}
|
|
119
|
+
const parser = parsers.find((candidate) => candidate.detect(input.buffer));
|
|
120
|
+
if (!parser) return null;
|
|
121
|
+
// Plan 127 Task 3 + further action #1: the 250-378 ms default-parser extract runs on the pool
|
|
122
|
+
// for large buffers, priced in both directions — bytes in at the flat clone rate, and one text
|
|
123
|
+
// value plus up to `maxPages` page spans back (a cloned string is shared rather than copied:
|
|
124
|
+
// 27 KiB and 2 MB text payloads both clone in ~0.06 ms, evidence §12). The inline estimate is
|
|
125
|
+
// the measured ~1 ms per KiB of PDF (§3: 250-378 ms for the 281 KiB fixture).
|
|
126
|
+
// Host-supplied parsers are functions and stay in-process; caps, redaction and the result
|
|
127
|
+
// shape stay here. The parser itself never honors a mid-parse abort either, so checking the
|
|
128
|
+
// signal before and after preserves the existing observable behavior.
|
|
129
|
+
let result: Omit<DocumentReaderResult, "format">;
|
|
130
|
+
if (
|
|
131
|
+
usesDefaultParsers &&
|
|
132
|
+
shouldOffloadToPool({
|
|
133
|
+
inputBytes: input.buffer.byteLength,
|
|
134
|
+
resultValues: 1 + maxPages * 4,
|
|
135
|
+
computeMs: input.buffer.byteLength / 1024,
|
|
136
|
+
})
|
|
137
|
+
) {
|
|
138
|
+
try {
|
|
139
|
+
result = await runInPool({
|
|
140
|
+
kind: "document.extract",
|
|
141
|
+
payload: { format: parser.format, bytes: input.buffer, maxPages, maxTextBytes },
|
|
142
|
+
});
|
|
143
|
+
} catch (error) {
|
|
144
|
+
throw reviveWorkerTaskError(error, [DocumentReaderError]) ?? error;
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
147
|
+
result = await parser.extract(input.buffer, { maxPages, maxTextBytes, signal: input.signal });
|
|
148
|
+
}
|
|
149
|
+
input.signal?.throwIfAborted();
|
|
150
|
+
if (Buffer.byteLength(result.text, "utf8") > maxTextBytes) {
|
|
151
|
+
throw new DocumentReaderError(`parser returned text beyond the maxTextBytes cap (${maxTextBytes})`);
|
|
152
|
+
}
|
|
153
|
+
const redacted = options.redactor ? options.redactor.redact(result.text) : result.text;
|
|
154
|
+
return {
|
|
155
|
+
text: redacted,
|
|
156
|
+
format: parser.format,
|
|
157
|
+
pages: result.pages,
|
|
158
|
+
truncatedBy: result.truncatedBy,
|
|
159
|
+
...(!options.redactor && result.pageSpans ? { pageSpans: result.pageSpans } : {}),
|
|
160
|
+
} satisfies DocumentReaderResult;
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export type { CreateMistralOcrParserOptions, OcrPageSpan, OcrUsage } from "./mistral-ocr.js";
|
|
166
|
+
|
|
167
|
+
export {
|
|
168
|
+
createMistralOcrParser,
|
|
169
|
+
DEFAULT_OCR_BASE_URL,
|
|
170
|
+
DEFAULT_OCR_MAX_BYTES,
|
|
171
|
+
DEFAULT_OCR_MAX_CONCURRENT,
|
|
172
|
+
DEFAULT_OCR_MAX_PAGES,
|
|
173
|
+
DEFAULT_OCR_MAX_RESPONSE_BYTES,
|
|
174
|
+
DEFAULT_OCR_MODEL,
|
|
175
|
+
DEFAULT_OCR_TIMEOUT_MS,
|
|
176
|
+
HARD_OCR_MAX_BYTES,
|
|
177
|
+
HARD_OCR_MAX_CONCURRENT,
|
|
178
|
+
HARD_OCR_MAX_PAGES,
|
|
179
|
+
HARD_OCR_MAX_RESPONSE_BYTES,
|
|
180
|
+
HARD_OCR_TIMEOUT_MS,
|
|
181
|
+
} from "./mistral-ocr.js";
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional Mistral Document AI OCR parser. Host-selected only — never wired
|
|
3
|
+
* into default `createDocumentReader()`. Native fetch (no SDK peer). Inline
|
|
4
|
+
* base64 by default so nothing is uploaded to the Files API (no cleanup).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { assertSsrfAllowedUrl, type SsrfPolicy } from "@arnilo/prism";
|
|
8
|
+
import { pinnedFetch } from "@arnilo/prism/net";
|
|
9
|
+
import { DocumentReaderError } from "./errors.js";
|
|
10
|
+
import type { DocumentParser } from "./index.js";
|
|
11
|
+
|
|
12
|
+
export const DEFAULT_OCR_MODEL = "mistral-ocr-latest";
|
|
13
|
+
export const DEFAULT_OCR_BASE_URL = "https://api.mistral.ai";
|
|
14
|
+
export const DEFAULT_OCR_TIMEOUT_MS = 60_000;
|
|
15
|
+
export const HARD_OCR_TIMEOUT_MS = 180_000;
|
|
16
|
+
export const DEFAULT_OCR_MAX_BYTES = 8 * 1024 * 1024;
|
|
17
|
+
export const HARD_OCR_MAX_BYTES = 50 * 1024 * 1024;
|
|
18
|
+
export const DEFAULT_OCR_MAX_PAGES = 32;
|
|
19
|
+
export const HARD_OCR_MAX_PAGES = 10_000;
|
|
20
|
+
export const DEFAULT_OCR_MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
|
|
21
|
+
export const HARD_OCR_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
|
22
|
+
export const DEFAULT_OCR_MAX_CONCURRENT = 1;
|
|
23
|
+
export const HARD_OCR_MAX_CONCURRENT = 4;
|
|
24
|
+
|
|
25
|
+
const PDF_MAGIC = "%PDF-";
|
|
26
|
+
|
|
27
|
+
export interface OcrUsage {
|
|
28
|
+
readonly model: string;
|
|
29
|
+
readonly pagesProcessed: number;
|
|
30
|
+
readonly bytes: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface OcrPageSpan {
|
|
34
|
+
readonly page: number;
|
|
35
|
+
readonly start: number;
|
|
36
|
+
readonly end: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface CreateMistralOcrParserOptions {
|
|
40
|
+
/** Mistral API key. Required. Never read from the environment. */
|
|
41
|
+
readonly apiKey: string;
|
|
42
|
+
/** OCR model alias. Default `mistral-ocr-latest`. */
|
|
43
|
+
readonly model?: string;
|
|
44
|
+
/** API origin (residency). Default `https://api.mistral.ai`. */
|
|
45
|
+
readonly baseUrl?: string;
|
|
46
|
+
/**
|
|
47
|
+
* Host-authorized document URL Mistral should fetch. SSRF-checked.
|
|
48
|
+
* When omitted, the buffer is sent as a data URL (no Files API upload).
|
|
49
|
+
*/
|
|
50
|
+
readonly documentUrl?: string;
|
|
51
|
+
readonly timeoutMs?: number;
|
|
52
|
+
readonly maxBytes?: number;
|
|
53
|
+
readonly maxPages?: number;
|
|
54
|
+
readonly maxResponseBytes?: number;
|
|
55
|
+
readonly maxConcurrent?: number;
|
|
56
|
+
readonly ssrf?: SsrfPolicy;
|
|
57
|
+
/** Host transport; host owns DNS/SSRF when set. */
|
|
58
|
+
readonly fetch?: typeof globalThis.fetch;
|
|
59
|
+
/** Task 7 admission hook. Called after a successful HTTP 200, before over-page refusal. */
|
|
60
|
+
readonly recordUsage?: (usage: OcrUsage) => void | Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function cap(name: string, value: number, hard: number): number {
|
|
64
|
+
if (!Number.isInteger(value) || value <= 0 || value > hard) {
|
|
65
|
+
throw new RangeError(`mistral-ocr ${name} must be an integer in (0, ${hard}], got ${value}`);
|
|
66
|
+
}
|
|
67
|
+
return value;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Zero-copy `Buffer` view: `toString`/`base64` need it once the parser accepts plain `Uint8Array`. */
|
|
71
|
+
function byteView(bytes: Uint8Array): Buffer {
|
|
72
|
+
return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function detectKind(bytes: Uint8Array): "pdf" | "image" | null {
|
|
76
|
+
const view = byteView(bytes);
|
|
77
|
+
if (bytes.length >= PDF_MAGIC.length && view.toString("latin1", 0, PDF_MAGIC.length) === PDF_MAGIC) return "pdf";
|
|
78
|
+
if (bytes.length >= 8 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) return "image";
|
|
79
|
+
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return "image";
|
|
80
|
+
if (bytes.length >= 6 && view.toString("ascii", 0, 3) === "GIF") return "image";
|
|
81
|
+
if (bytes.length >= 12 && view.toString("ascii", 0, 4) === "RIFF" && view.toString("ascii", 8, 12) === "WEBP") {
|
|
82
|
+
return "image";
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function imageMediaType(bytes: Uint8Array): string {
|
|
88
|
+
if (bytes[0] === 0x89) return "image/png";
|
|
89
|
+
if (bytes[0] === 0xff) return "image/jpeg";
|
|
90
|
+
if (byteView(bytes).toString("ascii", 0, 3) === "GIF") return "image/gif";
|
|
91
|
+
return "image/webp";
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function originUrl(baseUrl: string): URL {
|
|
95
|
+
let url: URL;
|
|
96
|
+
try {
|
|
97
|
+
url = new URL(baseUrl);
|
|
98
|
+
} catch {
|
|
99
|
+
throw new DocumentReaderError("mistral-ocr baseUrl must be an absolute URL");
|
|
100
|
+
}
|
|
101
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
102
|
+
throw new DocumentReaderError("mistral-ocr baseUrl must use http(s)");
|
|
103
|
+
}
|
|
104
|
+
if (url.username || url.password) throw new DocumentReaderError("mistral-ocr baseUrl must not embed credentials");
|
|
105
|
+
if (url.hash) throw new DocumentReaderError("mistral-ocr baseUrl must not contain a fragment");
|
|
106
|
+
try {
|
|
107
|
+
assertSsrfAllowedUrl(url.href);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
throw new DocumentReaderError("mistral-ocr baseUrl is not allowed", { cause: error });
|
|
110
|
+
}
|
|
111
|
+
return url;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function limiter(max: number) {
|
|
115
|
+
let active = 0;
|
|
116
|
+
const wait: Array<() => void> = [];
|
|
117
|
+
return {
|
|
118
|
+
async acquire(signal?: AbortSignal): Promise<void> {
|
|
119
|
+
signal?.throwIfAborted();
|
|
120
|
+
while (active >= max) {
|
|
121
|
+
await new Promise<void>((resolve, reject) => {
|
|
122
|
+
const resume = () => {
|
|
123
|
+
signal?.removeEventListener("abort", onAbort);
|
|
124
|
+
resolve();
|
|
125
|
+
};
|
|
126
|
+
const onAbort = () => {
|
|
127
|
+
const i = wait.indexOf(resume);
|
|
128
|
+
if (i >= 0) wait.splice(i, 1);
|
|
129
|
+
reject(signal?.reason ?? new DOMException("Aborted", "AbortError"));
|
|
130
|
+
};
|
|
131
|
+
wait.push(resume);
|
|
132
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
133
|
+
});
|
|
134
|
+
signal?.throwIfAborted();
|
|
135
|
+
}
|
|
136
|
+
active += 1;
|
|
137
|
+
},
|
|
138
|
+
release(): void {
|
|
139
|
+
active -= 1;
|
|
140
|
+
wait.shift()?.();
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): { signal: AbortSignal; dispose: () => void } {
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
148
|
+
const onAbort = () => controller.abort();
|
|
149
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
150
|
+
if (signal?.aborted) controller.abort();
|
|
151
|
+
return {
|
|
152
|
+
signal: controller.signal,
|
|
153
|
+
dispose() {
|
|
154
|
+
clearTimeout(timer);
|
|
155
|
+
signal?.removeEventListener("abort", onAbort);
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
interface OcrPage {
|
|
161
|
+
readonly index?: number;
|
|
162
|
+
readonly markdown?: string;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function parseOcrBody(body: unknown): { pages: OcrPage[]; model: string; pagesProcessed: number } {
|
|
166
|
+
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
167
|
+
throw new DocumentReaderError("mistral-ocr: invalid response");
|
|
168
|
+
}
|
|
169
|
+
const record = body as Record<string, unknown>;
|
|
170
|
+
if (!Array.isArray(record.pages) || record.pages.length === 0) throw new DocumentReaderError("mistral-ocr: invalid response");
|
|
171
|
+
const pages: OcrPage[] = [];
|
|
172
|
+
for (const page of record.pages) {
|
|
173
|
+
if (typeof page !== "object" || page === null) throw new DocumentReaderError("mistral-ocr: invalid response");
|
|
174
|
+
const row = page as Record<string, unknown>;
|
|
175
|
+
pages.push({
|
|
176
|
+
index: typeof row.index === "number" ? row.index : undefined,
|
|
177
|
+
markdown: typeof row.markdown === "string" ? row.markdown : undefined,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const usage = record.usage_info;
|
|
181
|
+
const pagesProcessed =
|
|
182
|
+
typeof usage === "object" && usage !== null && typeof (usage as { pages_processed?: unknown }).pages_processed === "number"
|
|
183
|
+
? (usage as { pages_processed: number }).pages_processed
|
|
184
|
+
: pages.length;
|
|
185
|
+
return {
|
|
186
|
+
pages,
|
|
187
|
+
model: typeof record.model === "string" ? record.model : DEFAULT_OCR_MODEL,
|
|
188
|
+
pagesProcessed,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function assemblePages(
|
|
193
|
+
pages: readonly OcrPage[],
|
|
194
|
+
maxTextBytes: number,
|
|
195
|
+
): { text: string; pageSpans: OcrPageSpan[]; truncatedBy: "bytes" | null } {
|
|
196
|
+
const spans: OcrPageSpan[] = [];
|
|
197
|
+
let text = "";
|
|
198
|
+
for (let i = 0; i < pages.length; i += 1) {
|
|
199
|
+
const markdown = pages[i]?.markdown ?? "";
|
|
200
|
+
const page = pages[i]?.index ?? i + 1;
|
|
201
|
+
const chunk = `${i === 0 ? "" : "\n\n"}<!-- prism-page ${page} -->\n${markdown}`;
|
|
202
|
+
const next = text + chunk;
|
|
203
|
+
if (Buffer.byteLength(next, "utf8") > maxTextBytes) {
|
|
204
|
+
const buf = Buffer.from(next, "utf8").subarray(0, maxTextBytes).toString("utf8");
|
|
205
|
+
spans.push({ page, start: text.length, end: buf.length });
|
|
206
|
+
return { text: buf, pageSpans: spans, truncatedBy: "bytes" };
|
|
207
|
+
}
|
|
208
|
+
spans.push({ page, start: text.length, end: next.length });
|
|
209
|
+
text = next;
|
|
210
|
+
}
|
|
211
|
+
return { text, pageSpans: spans, truncatedBy: null };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Host-selected OCR/layout parser. Not a default `createDocumentReader` parser.
|
|
216
|
+
* Extracted content is untrusted inert text.
|
|
217
|
+
*/
|
|
218
|
+
export function createMistralOcrParser(options: CreateMistralOcrParserOptions): DocumentParser {
|
|
219
|
+
const apiKey = options.apiKey.trim();
|
|
220
|
+
if (!apiKey) throw new DocumentReaderError("mistral-ocr requires apiKey");
|
|
221
|
+
const model = options.model?.trim() || DEFAULT_OCR_MODEL;
|
|
222
|
+
const origin = originUrl(options.baseUrl?.trim() || DEFAULT_OCR_BASE_URL);
|
|
223
|
+
const timeoutMs = cap("timeoutMs", options.timeoutMs ?? DEFAULT_OCR_TIMEOUT_MS, HARD_OCR_TIMEOUT_MS);
|
|
224
|
+
const maxBytes = cap("maxBytes", options.maxBytes ?? DEFAULT_OCR_MAX_BYTES, HARD_OCR_MAX_BYTES);
|
|
225
|
+
const parserMaxPages = cap("maxPages", options.maxPages ?? DEFAULT_OCR_MAX_PAGES, HARD_OCR_MAX_PAGES);
|
|
226
|
+
const maxResponseBytes = cap("maxResponseBytes", options.maxResponseBytes ?? DEFAULT_OCR_MAX_RESPONSE_BYTES, HARD_OCR_MAX_RESPONSE_BYTES);
|
|
227
|
+
const maxConcurrent = cap("maxConcurrent", options.maxConcurrent ?? DEFAULT_OCR_MAX_CONCURRENT, HARD_OCR_MAX_CONCURRENT);
|
|
228
|
+
const documentUrl = options.documentUrl?.trim();
|
|
229
|
+
if (documentUrl) {
|
|
230
|
+
try {
|
|
231
|
+
assertSsrfAllowedUrl(documentUrl, options.ssrf);
|
|
232
|
+
} catch (error) {
|
|
233
|
+
throw new DocumentReaderError("mistral-ocr documentUrl is not allowed", { cause: error });
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const gate = limiter(maxConcurrent);
|
|
237
|
+
const transport = options.fetch;
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
format: "ocr",
|
|
241
|
+
detect: (bytes) => detectKind(bytes) !== null,
|
|
242
|
+
extract: async (bytes, { maxPages, maxTextBytes, signal }) => {
|
|
243
|
+
signal?.throwIfAborted();
|
|
244
|
+
if (bytes.byteLength > maxBytes) {
|
|
245
|
+
throw new DocumentReaderError(`mistral-ocr document exceeds maxBytes cap (${maxBytes})`);
|
|
246
|
+
}
|
|
247
|
+
const kind = detectKind(bytes);
|
|
248
|
+
if (!kind) throw new DocumentReaderError("mistral-ocr: buffer is not a PDF or image");
|
|
249
|
+
const pageCap = Math.min(maxPages, parserMaxPages);
|
|
250
|
+
const endpoint = new URL("/v1/ocr", origin);
|
|
251
|
+
const document = documentUrl
|
|
252
|
+
? kind === "pdf"
|
|
253
|
+
? { type: "document_url", document_url: documentUrl }
|
|
254
|
+
: { type: "image_url", image_url: documentUrl }
|
|
255
|
+
: kind === "pdf"
|
|
256
|
+
? { type: "document_url", document_url: `data:application/pdf;base64,${byteView(bytes).toString("base64")}` }
|
|
257
|
+
: { type: "image_url", image_url: `data:${imageMediaType(bytes)};base64,${byteView(bytes).toString("base64")}` };
|
|
258
|
+
const payload = JSON.stringify({
|
|
259
|
+
model,
|
|
260
|
+
document,
|
|
261
|
+
include_image_base64: false,
|
|
262
|
+
});
|
|
263
|
+
await gate.acquire(signal);
|
|
264
|
+
const timed = withTimeout(signal, timeoutMs);
|
|
265
|
+
try {
|
|
266
|
+
const init: RequestInit = {
|
|
267
|
+
method: "POST",
|
|
268
|
+
headers: {
|
|
269
|
+
authorization: `Bearer ${apiKey}`,
|
|
270
|
+
"content-type": "application/json",
|
|
271
|
+
},
|
|
272
|
+
body: payload,
|
|
273
|
+
signal: timed.signal,
|
|
274
|
+
};
|
|
275
|
+
const response = transport
|
|
276
|
+
? await transport(endpoint, init)
|
|
277
|
+
: await pinnedFetch(endpoint, init, {
|
|
278
|
+
errorPrefix: "mistral-ocr",
|
|
279
|
+
ssrf: options.ssrf,
|
|
280
|
+
maxResponseBytes,
|
|
281
|
+
});
|
|
282
|
+
if (!response.ok) {
|
|
283
|
+
await response.body?.cancel().catch(() => undefined);
|
|
284
|
+
throw new DocumentReaderError(`mistral-ocr HTTP ${response.status}`);
|
|
285
|
+
}
|
|
286
|
+
const raw = Buffer.from(await response.arrayBuffer());
|
|
287
|
+
if (raw.byteLength > maxResponseBytes) {
|
|
288
|
+
throw new DocumentReaderError(`mistral-ocr response exceeds maxResponseBytes cap (${maxResponseBytes})`);
|
|
289
|
+
}
|
|
290
|
+
let parsed: unknown;
|
|
291
|
+
try {
|
|
292
|
+
parsed = JSON.parse(raw.toString("utf8"));
|
|
293
|
+
} catch {
|
|
294
|
+
throw new DocumentReaderError("mistral-ocr: invalid response");
|
|
295
|
+
}
|
|
296
|
+
const body = parseOcrBody(parsed);
|
|
297
|
+
await options.recordUsage?.({
|
|
298
|
+
model: body.model,
|
|
299
|
+
pagesProcessed: body.pagesProcessed,
|
|
300
|
+
bytes: bytes.byteLength,
|
|
301
|
+
});
|
|
302
|
+
if (body.pages.length > pageCap) {
|
|
303
|
+
throw new DocumentReaderError(`document has ${body.pages.length} pages, exceeds maxPages cap (${pageCap}); refusing to extract`);
|
|
304
|
+
}
|
|
305
|
+
const assembled = assemblePages(body.pages, maxTextBytes);
|
|
306
|
+
return {
|
|
307
|
+
text: assembled.text,
|
|
308
|
+
pages: body.pages.length,
|
|
309
|
+
truncatedBy: assembled.truncatedBy,
|
|
310
|
+
pageSpans: assembled.pageSpans,
|
|
311
|
+
};
|
|
312
|
+
} finally {
|
|
313
|
+
timed.dispose();
|
|
314
|
+
gate.release();
|
|
315
|
+
}
|
|
316
|
+
},
|
|
317
|
+
};
|
|
318
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import type { CellValue, DocumentKind, DocumentModel } from "../documents/types.js";
|
|
2
|
+
import { DocumentReaderError } from "./errors.js";
|
|
3
|
+
import type { DocumentParser } from "./index.js";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Default parser wiring for the document reader, split out of `index.ts` so the worker entry
|
|
7
|
+
* (`runtime/worker-pool-entry.ts`) can build the same parsers on the pool thread (plan 127 Task 3).
|
|
8
|
+
* Non-public module: the package export map points `./document-reader` at `index.js`, which
|
|
9
|
+
* re-exports the two historical public factories. The documents stack is imported lazily inside
|
|
10
|
+
* the OOXML parser so a PDF-only worker does not load it.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
let parseDocumentModule: Promise<typeof import("../documents/parse.js")> | undefined;
|
|
14
|
+
|
|
15
|
+
/** `%PDF-` as bytes: a plain `Uint8Array` stringifies to "37,80,68,70,45", so the gate compares bytes. */
|
|
16
|
+
const PDF_MAGIC = [0x25, 0x50, 0x44, 0x46, 0x2d] as const;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Zero-copy `Buffer` view over any byte input. `Buffer`-only helpers (`includes`, `toString`) answer
|
|
20
|
+
* wrongly for a plain `Uint8Array` — the shape a worker payload arrives as (plan 127 further action
|
|
21
|
+
* #2) — so the byte checks that need a string view go through this.
|
|
22
|
+
*/
|
|
23
|
+
function byteView(bytes: Uint8Array): Buffer {
|
|
24
|
+
return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Byte-safe truncation to at most `maxBytes` UTF-8 bytes. */
|
|
28
|
+
function truncateToBytes(text: string, maxBytes: number): string {
|
|
29
|
+
const buffer = Buffer.from(text, "utf8");
|
|
30
|
+
if (buffer.byteLength <= maxBytes) return text;
|
|
31
|
+
return buffer.subarray(0, maxBytes).toString("utf8");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function loadPeer(name: string): Promise<unknown> {
|
|
35
|
+
try {
|
|
36
|
+
const mod = await import(name);
|
|
37
|
+
// CJS peers expose their export as `default` under Node ESM interop.
|
|
38
|
+
return (mod as { default?: unknown }).default ?? mod;
|
|
39
|
+
} catch {
|
|
40
|
+
throw new DocumentReaderError(
|
|
41
|
+
`document-reader: optional peer parser "${name}" is not installed. ` +
|
|
42
|
+
`Install it (npm i ${name}) or supply a host-selected parser via createDocumentReader({ parsers }); ` +
|
|
43
|
+
`refusing to create a reader that cannot extract this format.`,
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** v2 `PDFParse` surface used by the adapter (typed locally; the package is ambient). */
|
|
49
|
+
type PdfParseResult = { readonly total: number; readonly text: string };
|
|
50
|
+
type PdfParseCtor = new (options: {
|
|
51
|
+
readonly data: Uint8Array;
|
|
52
|
+
readonly isEvalSupported?: boolean;
|
|
53
|
+
}) => {
|
|
54
|
+
getText(options?: { readonly pageJoiner?: string }): Promise<PdfParseResult>;
|
|
55
|
+
destroy(): Promise<void>;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** Default PDF parser backed by the optional `pdf-parse` peer (v2 `PDFParse` class). */
|
|
59
|
+
export async function createPdfParser(): Promise<DocumentParser> {
|
|
60
|
+
const ctor = (await loadPeer("pdf-parse")) as { PDFParse?: unknown };
|
|
61
|
+
if (typeof ctor.PDFParse !== "function") {
|
|
62
|
+
throw new DocumentReaderError('document-reader: optional peer "pdf-parse" (v2+) does not export the PDFParse class');
|
|
63
|
+
}
|
|
64
|
+
const PDFParse = ctor.PDFParse as PdfParseCtor;
|
|
65
|
+
return {
|
|
66
|
+
format: "pdf",
|
|
67
|
+
detect: (bytes) => bytes.length >= PDF_MAGIC.length && PDF_MAGIC.every((byte, index) => bytes[index] === byte),
|
|
68
|
+
extract: async (bytes, { maxPages, maxTextBytes, signal }) => {
|
|
69
|
+
signal?.throwIfAborted();
|
|
70
|
+
// v2 runs pdf.js in a worker thread and claims the TypedArray (transfer);
|
|
71
|
+
// the adapter never reuses the buffer after parse, so passing a fresh
|
|
72
|
+
// Uint8Array view is safe. isEvalSupported:false keeps PDF functions from
|
|
73
|
+
// executing embedded scripts — the document-reader contract forbids it.
|
|
74
|
+
const parser = new PDFParse({ data: new Uint8Array(bytes), isEvalSupported: false });
|
|
75
|
+
let data: { total: number; text: string };
|
|
76
|
+
try {
|
|
77
|
+
// pageJoiner:'' keeps v2's page-boundary markers out of extracted text
|
|
78
|
+
// (v1 emitted none; goldens assert on raw page text).
|
|
79
|
+
data = await parser.getText({ pageJoiner: "" });
|
|
80
|
+
} finally {
|
|
81
|
+
await parser.destroy();
|
|
82
|
+
}
|
|
83
|
+
if (data.total > maxPages) {
|
|
84
|
+
throw new DocumentReaderError(`document has ${data.total} pages, exceeds maxPages cap (${maxPages}); refusing to extract`);
|
|
85
|
+
}
|
|
86
|
+
const text = data.text ?? "";
|
|
87
|
+
if (Buffer.byteLength(text, "utf8") > maxTextBytes) {
|
|
88
|
+
return { text: truncateToBytes(text, maxTextBytes), pages: data.total, truncatedBy: "bytes" };
|
|
89
|
+
}
|
|
90
|
+
return { text, pages: data.total, truncatedBy: null };
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Default DOCX parser backed by the optional `mammoth` peer (raw text only). */
|
|
96
|
+
export async function createDocxParser(): Promise<DocumentParser> {
|
|
97
|
+
const mammoth = (await loadPeer("mammoth")) as {
|
|
98
|
+
extractRawText(input: { buffer: Buffer }): Promise<{ value: string }>;
|
|
99
|
+
};
|
|
100
|
+
return {
|
|
101
|
+
format: "docx",
|
|
102
|
+
// zip container + main document part marker; entry names are stored
|
|
103
|
+
// uncompressed, so the literal name appears verbatim in the headers.
|
|
104
|
+
detect: (bytes) =>
|
|
105
|
+
bytes.length >= 4 &&
|
|
106
|
+
bytes[0] === 0x50 &&
|
|
107
|
+
bytes[1] === 0x4b &&
|
|
108
|
+
bytes[2] === 0x03 &&
|
|
109
|
+
bytes[3] === 0x04 &&
|
|
110
|
+
byteView(bytes).includes("word/document.xml"),
|
|
111
|
+
extract: async (bytes, { maxTextBytes, signal }) => {
|
|
112
|
+
signal?.throwIfAborted();
|
|
113
|
+
const { value } = await mammoth.extractRawText({ buffer: byteView(bytes) });
|
|
114
|
+
const text = value ?? "";
|
|
115
|
+
if (Buffer.byteLength(text, "utf8") > maxTextBytes) {
|
|
116
|
+
return { text: truncateToBytes(text, maxTextBytes), pages: 1, truncatedBy: "bytes" };
|
|
117
|
+
}
|
|
118
|
+
return { text, pages: 1, truncatedBy: null };
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function cellText(value: CellValue): string {
|
|
124
|
+
if (value === null) return "";
|
|
125
|
+
if (typeof value !== "object") return String(value);
|
|
126
|
+
if ("formula" in value) return value.cachedValue === undefined || value.cachedValue === null ? value.formula : String(value.cachedValue);
|
|
127
|
+
return value.value;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function modelText(model: DocumentModel): string {
|
|
131
|
+
if (model.kind === "sheet") {
|
|
132
|
+
return model.sheets.map((sheet) => [sheet.name, ...sheet.cells.map((row) => row.map(cellText).join("\t"))].join("\n")).join("\n\n");
|
|
133
|
+
}
|
|
134
|
+
if (model.kind === "deck") {
|
|
135
|
+
return model.slides
|
|
136
|
+
.map((slide, index) => [`Slide ${index + 1}`, slide.title, slide.subtitle, ...(slide.bullets ?? [])].filter(Boolean).join("\n"))
|
|
137
|
+
.join("\n\n");
|
|
138
|
+
}
|
|
139
|
+
return "";
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** OOXML (xlsx/pptx) parser over the documents model — the same shared core the tools use. */
|
|
143
|
+
export function createOoxmlParser(format: "xlsx" | "pptx", kind: DocumentKind, part: string): DocumentParser {
|
|
144
|
+
return {
|
|
145
|
+
format,
|
|
146
|
+
detect: (bytes) =>
|
|
147
|
+
bytes.length >= 4 &&
|
|
148
|
+
bytes[0] === 0x50 &&
|
|
149
|
+
bytes[1] === 0x4b &&
|
|
150
|
+
bytes[2] === 0x03 &&
|
|
151
|
+
bytes[3] === 0x04 &&
|
|
152
|
+
byteView(bytes).includes(part),
|
|
153
|
+
extract: async (bytes, { maxPages, maxTextBytes, signal }) => {
|
|
154
|
+
signal?.throwIfAborted();
|
|
155
|
+
parseDocumentModule ??= import("../documents/parse.js");
|
|
156
|
+
const { parseDocument } = await parseDocumentModule;
|
|
157
|
+
const model = await parseDocument(bytes, {
|
|
158
|
+
kind,
|
|
159
|
+
caps:
|
|
160
|
+
kind === "sheet"
|
|
161
|
+
? { maxBytes: bytes.byteLength, maxSheets: Math.min(maxPages, 1_000) }
|
|
162
|
+
: { maxBytes: bytes.byteLength, maxSlides: Math.min(maxPages, 2_000) },
|
|
163
|
+
});
|
|
164
|
+
const pages = model.kind === "sheet" ? model.sheets.length : model.kind === "deck" ? model.slides.length : 0;
|
|
165
|
+
const text = modelText(model);
|
|
166
|
+
return Buffer.byteLength(text, "utf8") > maxTextBytes
|
|
167
|
+
? { text: truncateToBytes(text, maxTextBytes), pages, truncatedBy: "bytes" as const }
|
|
168
|
+
: { text, pages, truncatedBy: null };
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
}
|