@echofiles/echo-pdf 0.11.0 → 0.11.4

package/README.md

@@ -187,12 +187,12 @@ Published docs site:
 ## Development
 
 ```bash
-npm ci
-npm run build
-npm run typecheck
-npm run test:unit
-npm run test:acceptance
-npm run test:integration
+bun install --frozen-lockfile
+bun run build
+bun run typecheck
+bun run test:unit
+bun run test:acceptance
+bun run test:integration
 ```
 
 For source-checkout CLI development and repo-local workflows, see [docs/DEVELOPMENT.md](./docs/DEVELOPMENT.md).

package/bin/echo-pdf.js

@@ -203,13 +203,13 @@ const loadLocalDocumentApi = async () => {
     }
     throw new Error(
       "Internal source-checkout CLI dev mode requires Bun and src/local/index.ts. " +
-      "Use `npm run cli:dev -- <primitive> ...` only from a source checkout."
+      "Use `bun run cli:dev -- <primitive> ...` only from a source checkout."
     )
   }
   if (!fs.existsSync(LOCAL_DOCUMENT_DIST_PATH)) {
     throw new Error(
       "Local primitive commands require built artifacts in a source checkout. " +
-      "Run `npm run build` first, use the internal `npm run cli:dev -- <primitive> ...` path in this repo, or install the published package."
+      "Run `bun run build` first, use the internal `bun run cli:dev -- <primitive> ...` path in this repo, or install the published package."
     )
   }
   return import(LOCAL_DOCUMENT_DIST_ENTRY.href)

package/dist/local/semantic.js

@@ -5,7 +5,7 @@ import { resolveModelForProvider, resolveProviderAlias } from "../provider-defau
 import { toDataUrl } from "../file-utils.js";
 import { generateText, visionRecognize } from "../provider-client.js";
 import { ensureRenderArtifact, indexDocumentInternal } from "./document.js";
-import { fileExists, matchesSourceSnapshot, matchesStrategyKey, pageLabel, parseJsonObject, readJson, resolveConfig, resolveEnv, writeJson, } from "./shared.js";
+import { fileExists, matchesSourceSnapshot, matchesStrategyKey, pageLabel, parseJsonObject, parseJsonObjectWithRepair, readJson, resolveConfig, resolveEnv, writeJson, } from "./shared.js";
 import { normalizeFigureItems, normalizeUnderstandingFormulas, normalizeUnderstandingTables } from "./understanding.js";
 const resolveSemanticExtractionBudget = (input) => ({
     pageSelection: "all",
@@ -134,6 +134,15 @@ const resolveSemanticAgentContext = (config, request) => {
     }
     return { provider, model };
 };
+class SemanticAggregationModelOutputError extends Error {
+    detail;
+    code = "SEMANTIC_AGGREGATION_INVALID_JSON";
+    constructor(message, detail) {
+        super(message);
+        this.detail = detail;
+        this.name = "SemanticAggregationModelOutputError";
+    }
+}
 const extractCombinedPageData = async (input) => {
     const renderArtifact = await ensureRenderArtifact({
         pdfPath: input.request.pdfPath,
@@ -170,6 +179,55 @@ const extractCombinedPageData = async (input) => {
         },
     };
 };
+const buildSemanticAggregationRetryPrompt = (record, candidates) => {
+    return [
+        buildSemanticAggregationPrompt(record, candidates),
+        "",
+        "Your previous response was not strict JSON.",
+        "Return the same semantic structure again, but this time produce strict RFC 8259 JSON only.",
+        "Do not wrap in markdown fences.",
+        "Do not use invalid backslash escapes such as \\(, \\), \\_, or \\- inside JSON strings.",
+    ].join("\n");
+};
+const parseSemanticAggregationResponse = async (input) => {
+    try {
+        const parsed = parseJsonObjectWithRepair(input.aggregated);
+        return {
+            sections: parsed.parsed?.sections,
+            repaired: parsed.repaired,
+            retried: false,
+        };
+    }
+    catch (firstError) {
+        const causeMessage = firstError instanceof Error ? firstError.message : String(firstError);
+        const retried = await generateText({
+            config: input.config,
+            env: input.env,
+            providerAlias: input.provider,
+            model: input.model,
+            prompt: buildSemanticAggregationRetryPrompt(input.record, input.candidates),
+            runtimeApiKeys: input.runtimeApiKeys,
+        });
+        try {
+            const parsed = parseJsonObjectWithRepair(retried);
+            return {
+                sections: parsed.parsed?.sections,
+                repaired: parsed.repaired,
+                retried: true,
+            };
+        }
+        catch (retryError) {
+            const retryCauseMessage = retryError instanceof Error ? retryError.message : String(retryError);
+            throw new SemanticAggregationModelOutputError("semantic aggregation returned invalid JSON after repair and retry", {
+                provider: input.provider,
+                model: input.model,
+                repaired: false,
+                retried: true,
+                causeMessage: `${causeMessage}; retry=${retryCauseMessage}`,
+            });
+        }
+    }
+};
 const mergeCrossPageTables = (understandings) => {
     const merged = [];
     let nextId = 1;
@@ -304,8 +362,17 @@ const ensureSemanticStructureArtifact = async (request) => {
         prompt: buildSemanticAggregationPrompt(record, [...candidateMap.values()]),
         runtimeApiKeys: request.providerApiKeys,
     });
-    const parsed = parseJsonObject(aggregated);
-    const sections = toSemanticTree(parsed?.sections, pageArtifactPaths);
+    const parsed = await parseSemanticAggregationResponse({
+        aggregated,
+        record,
+        candidates: [...candidateMap.values()],
+        config,
+        env,
+        provider,
+        model,
+        runtimeApiKeys: request.providerApiKeys,
+    });
+    const sections = toSemanticTree(parsed.sections, pageArtifactPaths);
     const mergedTables = mergeCrossPageTables(pageElements);
     const mergedFormulas = mergeCrossPageFormulas(pageElements);
     const mergedFigures = mergeCrossPageFigures(pageElements);

package/dist/local/shared.d.ts

@@ -19,6 +19,10 @@ export declare const createPreview: (text: string) => string;
 export declare const createPageTitle: (pageNumber: number, text: string) => string;
 export declare const stripCodeFences: (value: string) => string;
 export declare const parseJsonObject: (value: string) => unknown;
+export declare const parseJsonObjectWithRepair: (value: string) => {
+    parsed: unknown;
+    repaired: boolean;
+};
 export declare const normalizeTableItems: (value: unknown) => LocalTableArtifactItem[];
 export declare const normalizeFormulaItems: (value: unknown) => LocalFormulaArtifactItem[];
 export declare const resolveEnv: (env?: Env) => Env;

package/dist/local/shared.js

@@ -77,6 +77,96 @@ export const parseJsonObject = (value) => {
         throw new Error("model output was not valid JSON");
     }
 };
+const validJsonEscape = (value) => /["\\/bfnrt]/.test(value);
+const repairInvalidJsonEscapes = (value) => {
+    let repaired = false;
+    let inString = false;
+    let escaping = false;
+    let unicodeDigitsRemaining = 0;
+    let output = "";
+    for (let index = 0; index < value.length; index += 1) {
+        const char = value[index] ?? "";
+        if (!inString) {
+            output += char;
+            if (char === "\"")
+                inString = true;
+            continue;
+        }
+        if (unicodeDigitsRemaining > 0) {
+            output += char;
+            if (/^[0-9a-fA-F]$/.test(char)) {
+                unicodeDigitsRemaining -= 1;
+            }
+            else {
+                repaired = true;
+                unicodeDigitsRemaining = 0;
+            }
+            continue;
+        }
+        if (escaping) {
+            if (validJsonEscape(char)) {
+                output += char;
+            }
+            else if (char === "u") {
+                output += char;
+                unicodeDigitsRemaining = 4;
+            }
+            else {
+                output += `\\${char}`;
+                repaired = true;
+            }
+            escaping = false;
+            continue;
+        }
+        if (char === "\\") {
+            output += char;
+            escaping = true;
+            continue;
+        }
+        output += char;
+        if (char === "\"")
+            inString = false;
+    }
+    if (escaping) {
+        output += "\\";
+        repaired = true;
+    }
+    return { repairedText: output, repaired };
+};
+export const parseJsonObjectWithRepair = (value) => {
+    const trimmed = stripCodeFences(value).trim();
+    if (!trimmed)
+        return { parsed: null, repaired: false };
+    const candidates = [trimmed];
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+        const sliced = trimmed.slice(start, end + 1);
+        if (sliced !== trimmed)
+            candidates.push(sliced);
+    }
+    let lastError = null;
+    for (const candidate of candidates) {
+        try {
+            return { parsed: JSON.parse(candidate), repaired: false };
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+        }
+        const repairedCandidate = repairInvalidJsonEscapes(candidate);
+        if (!repairedCandidate.repaired)
+            continue;
+        try {
+            return { parsed: JSON.parse(repairedCandidate.repairedText), repaired: true };
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+        }
+    }
+    if (lastError)
+        throw lastError;
+    throw new Error("model output was not valid JSON");
+};
 export const normalizeTableItems = (value) => {
     if (!Array.isArray(value))
         return [];

package/dist/provider-client.js

@@ -1,4 +1,28 @@
 import { resolveProviderApiKey } from "./provider-keys.js";
+class ProviderRequestError extends Error {
+    code;
+    detail;
+    constructor(code, detail) {
+        const parts = [
+            `${detail.operation} failed`,
+            `code=${code}`,
+            `url=${detail.url}`,
+            `attempt=${detail.attempt}/${detail.maxAttempts}`,
+        ];
+        if (typeof detail.status === "number")
+            parts.push(`http=${detail.status}`);
+        if (detail.contentType)
+            parts.push(`contentType=${detail.contentType}`);
+        if (detail.causeMessage)
+            parts.push(`cause=${detail.causeMessage}`);
+        if (detail.responsePreview)
+            parts.push(`preview=${JSON.stringify(detail.responsePreview)}`);
+        super(parts.join(" "));
+        this.code = code;
+        this.detail = detail;
+        this.name = "ProviderRequestError";
+    }
+}
 const defaultBaseUrl = (provider) => {
     if (provider.baseUrl)
         return provider.baseUrl;
@@ -32,33 +56,197 @@ const toAuthHeader = (config, providerAlias, provider, env, runtimeApiKeys) => {
     });
     return token ? { Authorization: `Bearer ${token}` } : {};
 };
-const withTimeout = async (url, init, timeoutMs) => {
+const isPrivateIpv4 = (hostname) => /^10\./.test(hostname) ||
+    /^127\./.test(hostname) ||
+    /^192\.168\./.test(hostname) ||
+    /^172\.(1[6-9]|2\d|3[0-1])\./.test(hostname);
+const isLocalOpenAiCompatible = (provider, url) => {
+    try {
+        const parsed = new URL(url);
+        const hostname = parsed.hostname.toLowerCase();
+        return (provider.type === "openai-compatible" &&
+            (hostname === "localhost" || hostname === "::1" || isPrivateIpv4(hostname)));
+    }
+    catch {
+        return false;
+    }
+};
+const maxAttemptsFor = (provider, url, method, operation) => {
+    if (method !== "POST")
+        return 1;
+    if (operation === "Model list request")
+        return 1;
+    return isLocalOpenAiCompatible(provider, url) ? 2 : 1;
+};
+const truncate = (value, length = 800) => value.slice(0, length);
+const withTimeout = async (input) => {
     const ctrl = new AbortController();
-    const timer = setTimeout(() => ctrl.abort("timeout"), timeoutMs);
+    const timer = setTimeout(() => ctrl.abort("timeout"), input.timeoutMs);
     try {
-        return await fetch(url, { ...init, signal: ctrl.signal });
+        return await fetch(input.url, { ...input.init, signal: ctrl.signal });
     }
     catch (error) {
         if (error instanceof Error && error.name === "AbortError") {
-            throw new Error(`Request timeout after ${timeoutMs}ms for ${url}`);
+            throw new ProviderRequestError("PROVIDER_REQUEST_TIMEOUT", {
+                operation: input.operation,
+                providerAlias: input.providerAlias,
+                url: input.url,
+                attempt: input.attempt,
+                maxAttempts: input.maxAttempts,
+                causeMessage: `timeout after ${input.timeoutMs}ms`,
+            });
         }
-        throw error;
+        throw new ProviderRequestError("PROVIDER_REQUEST_SEND_FAILED", {
+            operation: input.operation,
+            providerAlias: input.providerAlias,
+            url: input.url,
+            attempt: input.attempt,
+            maxAttempts: input.maxAttempts,
+            causeMessage: error instanceof Error ? error.message : String(error),
+        });
     }
     finally {
         clearTimeout(timer);
     }
 };
-const responseDetail = async (response) => {
-    const contentType = response.headers.get("content-type") ?? "";
+const readResponseText = async (input) => {
+    const reader = input.response.body?.getReader();
+    if (!reader) {
+        try {
+            return await input.response.text();
+        }
+        catch (error) {
+            throw new ProviderRequestError("PROVIDER_RESPONSE_BODY_READ_FAILED", {
+                operation: input.operation,
+                providerAlias: input.providerAlias,
+                url: input.url,
+                status: input.response.status,
+                contentType: input.response.headers.get("content-type") ?? "",
+                attempt: input.attempt,
+                maxAttempts: input.maxAttempts,
+                causeMessage: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    const chunks = [];
+    let total = 0;
     try {
-        if (contentType.includes("application/json")) {
-            return JSON.stringify(await response.json()).slice(0, 800);
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (!value || value.length === 0)
+                continue;
+            chunks.push(value);
+            total += value.length;
         }
-        return (await response.text()).slice(0, 800);
     }
-    catch {
-        return "<unable to parse response payload>";
+    catch (error) {
+        const combined = new Uint8Array(total);
+        let offset = 0;
+        for (const chunk of chunks) {
+            combined.set(chunk, offset);
+            offset += chunk.length;
+        }
+        throw new ProviderRequestError("PROVIDER_RESPONSE_BODY_READ_FAILED", {
+            operation: input.operation,
+            providerAlias: input.providerAlias,
+            url: input.url,
+            status: input.response.status,
+            contentType: input.response.headers.get("content-type") ?? "",
+            attempt: input.attempt,
+            maxAttempts: input.maxAttempts,
+            causeMessage: error instanceof Error ? error.message : String(error),
+            responsePreview: truncate(new TextDecoder().decode(combined)),
+        });
+    }
+    const combined = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        combined.set(chunk, offset);
+        offset += chunk.length;
     }
+    return new TextDecoder().decode(combined);
+};
+const parseJsonResponse = (input) => {
+    try {
+        return JSON.parse(input.rawText);
+    }
+    catch (error) {
+        throw new ProviderRequestError("PROVIDER_RESPONSE_JSON_PARSE_FAILED", {
+            operation: input.operation,
+            providerAlias: input.providerAlias,
+            url: input.url,
+            status: input.response.status,
+            contentType: input.response.headers.get("content-type") ?? "",
+            attempt: input.attempt,
+            maxAttempts: input.maxAttempts,
+            causeMessage: error instanceof Error ? error.message : String(error),
+            responsePreview: truncate(input.rawText),
+        });
+    }
+};
+const requestJson = async (input) => {
+    const maxAttempts = maxAttemptsFor(input.provider, input.url, input.method, input.operation);
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+        try {
+            const response = await withTimeout({
+                operation: input.operation,
+                providerAlias: input.providerAlias,
+                url: input.url,
+                init: {
+                    method: input.method,
+                    headers: input.headers,
+                    ...(typeof input.body === "string" ? { body: input.body } : {}),
+                },
+                timeoutMs: input.provider.timeoutMs ?? 30000,
+                attempt,
+                maxAttempts,
+            });
+            const rawText = await readResponseText({
+                response,
+                operation: input.operation,
+                providerAlias: input.providerAlias,
+                url: input.url,
+                attempt,
+                maxAttempts,
+            });
+            if (!response.ok) {
+                throw new ProviderRequestError("PROVIDER_HTTP_ERROR", {
+                    operation: input.operation,
+                    providerAlias: input.providerAlias,
+                    url: input.url,
+                    status: response.status,
+                    contentType: response.headers.get("content-type") ?? "",
+                    attempt,
+                    maxAttempts,
+                    responsePreview: truncate(rawText),
+                });
+            }
+            return parseJsonResponse({
+                rawText,
+                response,
+                operation: input.operation,
+                providerAlias: input.providerAlias,
+                url: input.url,
+                attempt,
+                maxAttempts,
+            });
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            if (attempt < maxAttempts &&
+                lastError instanceof ProviderRequestError &&
+                (lastError.code === "PROVIDER_REQUEST_SEND_FAILED" ||
+                    lastError.code === "PROVIDER_RESPONSE_BODY_READ_FAILED" ||
+                    lastError.code === "PROVIDER_RESPONSE_JSON_PARSE_FAILED")) {
+                continue;
+            }
+            throw lastError;
+        }
+    }
+    throw lastError ?? new Error(`${input.operation} failed`);
 };
 const getProvider = (config, alias) => {
     const provider = config.providers[alias];
@@ -70,18 +258,20 @@ const getProvider = (config, alias) => {
 export const listProviderModels = async (config, env, alias, runtimeApiKeys) => {
     const provider = getProvider(config, alias);
     const url = resolveEndpoint(provider, "modelsPath");
-    const response = await withTimeout(url, {
+    const payload = await requestJson({
+        operation: "Model list request",
+        config,
+        env,
+        providerAlias: alias,
+        provider,
+        url,
         method: "GET",
         headers: {
             Accept: "application/json",
             ...toAuthHeader(config, alias, provider, env, runtimeApiKeys),
             ...(provider.headers ?? {}),
         },
-    }, provider.timeoutMs ?? 30000);
-    if (!response.ok) {
-        throw new Error(`Model list request failed: HTTP ${response.status} url=${url} detail=${await responseDetail(response)}`);
-    }
-    const payload = await response.json();
+    });
     const data = payload.data;
     if (!Array.isArray(data))
         return [];
@@ -93,7 +283,13 @@ export const listProviderModels = async (config, env, alias, runtimeApiKeys) =>
 export const visionRecognize = async (input) => {
     const provider = getProvider(input.config, input.providerAlias);
     const url = resolveEndpoint(provider, "chatCompletionsPath");
-    const response = await withTimeout(url, {
+    const payload = await requestJson({
+        operation: "Vision request",
+        config: input.config,
+        env: input.env,
+        providerAlias: input.providerAlias,
+        provider,
+        url,
         method: "POST",
         headers: {
             "Content-Type": "application/json",
@@ -112,11 +308,7 @@ export const visionRecognize = async (input) => {
                 },
             ],
         }),
-    }, provider.timeoutMs ?? 30000);
-    if (!response.ok) {
-        throw new Error(`Vision request failed: HTTP ${response.status} url=${url} detail=${await responseDetail(response)}`);
-    }
-    const payload = await response.json();
+    });
     const message = payload.choices?.[0]?.message;
     if (!message)
         return "";
@@ -135,7 +327,13 @@ export const visionRecognize = async (input) => {
 export const generateText = async (input) => {
     const provider = getProvider(input.config, input.providerAlias);
     const url = resolveEndpoint(provider, "chatCompletionsPath");
-    const response = await withTimeout(url, {
+    const payload = await requestJson({
+        operation: "Text generation request",
+        config: input.config,
+        env: input.env,
+        providerAlias: input.providerAlias,
+        provider,
+        url,
         method: "POST",
         headers: {
             "Content-Type": "application/json",
@@ -151,11 +349,7 @@ export const generateText = async (input) => {
                 },
             ],
         }),
-    }, provider.timeoutMs ?? 30000);
-    if (!response.ok) {
-        throw new Error(`Text generation request failed: HTTP ${response.status} url=${url} detail=${await responseDetail(response)}`);
-    }
-    const payload = await response.json();
+    });
     const message = payload.choices?.[0]?.message;
     if (!message)
         return "";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@echofiles/echo-pdf",
   "description": "Local-first PDF document component core with CLI, workspace artifacts, and reusable page primitives.",
-  "version": "0.11.0",
+  "version": "0.11.4",
   "type": "module",
   "homepage": "https://pdf.echofile.ai/",
   "repository": {
@@ -18,6 +18,7 @@
     "cli",
     "vision-language"
   ],
+  "packageManager": "bun@1.3.5",
   "publishConfig": {
     "access": "public"
   },
@@ -53,14 +54,14 @@
     "eval:stress": "node ./eval/run-local.mjs --suite stress",
     "eval:known-bad": "node ./eval/run-local.mjs --suite known-bad",
     "eval:fetch-public-samples": "node ./eval/fetch-public-samples.mjs",
-    "typecheck": "npm run check:runtime && tsc --noEmit",
-    "test:unit": "npm run check:runtime && vitest run tests/unit",
-    "test:acceptance": "npm run check:runtime && npm run build && vitest run tests/acceptance",
-    "test:import-smoke": "npm run check:runtime && npm run build && vitest run tests/integration/npm-pack-import.integration.test.ts tests/integration/ts-nodenext-consumer.integration.test.ts",
-    "test:integration": "npm run check:runtime && npm run build && vitest run tests/integration/local-document-cli.integration.test.ts tests/integration/local-document.integration.test.ts tests/integration/local-semantic-structure.integration.test.ts tests/integration/npm-pack-import.integration.test.ts tests/integration/ts-nodenext-consumer.integration.test.ts",
-    "test": "npm run test:unit && npm run test:acceptance && npm run test:integration",
+    "typecheck": "bun run check:runtime && tsc --noEmit",
+    "test:unit": "bun run check:runtime && vitest run tests/unit",
+    "test:acceptance": "bun run check:runtime && bun run build && vitest run tests/acceptance",
+    "test:import-smoke": "bun run check:runtime && bun run build && vitest run tests/integration/npm-pack-import.integration.test.ts tests/integration/ts-nodenext-consumer.integration.test.ts",
+    "test:integration": "bun run check:runtime && bun run build && vitest run tests/integration/local-document-cli.integration.test.ts tests/integration/local-document.integration.test.ts tests/integration/local-provider-stability.integration.test.ts tests/integration/local-semantic-structure.integration.test.ts tests/integration/npm-pack-import.integration.test.ts tests/integration/ts-nodenext-consumer.integration.test.ts",
+    "test": "bun run test:unit && bun run test:acceptance && bun run test:integration",
     "smoke": "bash ./scripts/smoke.sh",
-    "prepublishOnly": "npm run build && npm run typecheck && npm run test"
+    "prepublishOnly": "bun run build && bun run typecheck && bun run test"
   },
   "engines": {
     "node": ">=20.0.0"

package/scripts/check-runtime.sh

@@ -9,7 +9,7 @@ if [[ -z "${current_node_major}" ]] || (( current_node_major < required_node_maj
   exit 1
 fi
 
-for cmd in npm curl grep sed; do
+for cmd in bun curl grep sed; do
   if ! command -v "${cmd}" >/dev/null 2>&1; then
     echo "Missing required command: ${cmd}"
     exit 1
@@ -23,4 +23,4 @@ if [[ "${CHECK_LLM_KEYS:-0}" == "1" ]]; then
   fi
 fi
 
-echo "runtime check passed: node=$(node -v), npm=$(npm -v)"
+echo "runtime check passed: node=$(node -v), bun=$(bun -v)"

package/scripts/smoke.sh

@@ -11,4 +11,4 @@ bash "${SCRIPT_DIR}/check-runtime.sh"
 # - SMOKE_LLM_PROVIDER
 # - SMOKE_LLM_MODEL
 # - TESTCASE_DIR
-npm run test:integration
+bun run test:integration