@webcrawl/mcp 1.4.3 → 1.4.4

package/dist/core/engine/chrome.js

@@ -1,131 +1,199 @@
 /**
- * Local Chrome dump-dom renderer (JS pages the native fetch ladder cannot hydrate).
+ * Local Chrome renderer (JS pages the native fetch ladder cannot hydrate).
  *
- * Uses a user-installed Chrome/Chromium binary — the same one `webcrawl doctor`
- * reports — with `--dump-dom` + virtual time so `document.write` / client-rendered
- * content appears in the HTML. Playwright is not required; this is the zero-key
- * local JS tier.
+ * Primary driver is Chrome DevTools Protocol:
+ * launch headless Chrome, navigate, wait until the DOM is content-ready
+ * (load + 200ms mutation-idle), then read `documentElement.outerHTML`.
+ *
+ * dump-dom is a last-resort fallback: Chrome 156 hangs on dump-dom with a
+ * fresh profile, which used to look like a successful scrape of the
+ * unrendered nav shell. Every attempt has a hard timeout and a process-group
+ * kill. Playwright's chrome-headless-shell is discovered automatically when
+ * present.
  */
 import { spawn } from "node:child_process";
-import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-const DUMP_MAX_BYTES = 2 * 1024 * 1024;
-const DEFAULT_TIMEOUT_MS = 20_000;
-const VIRTUAL_TIME_MS = 10_000;
-/**
- * Path to a real installed Chrome/Chromium, or null.
- * Priority: WEBCRAWL_CHROME_PATH, INTERACT_CHROME_PATH, CHROME_PATH, then per-OS
- * defaults (same locations `webcrawl doctor` checks).
- */
-export function installedChromePath() {
-    const fromEnv = process.env.WEBCRAWL_CHROME_PATH?.trim() ||
-        process.env.INTERACT_CHROME_PATH?.trim() ||
-        process.env.CHROME_PATH?.trim();
-    if (fromEnv)
-        return fromEnv;
-    const candidates = [];
-    if (process.platform === "darwin") {
-        candidates.push("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", `${process.env.HOME || ""}/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`);
-    }
-    else if (process.platform === "win32") {
-        candidates.push(`${process.env["ProgramFiles"] || "C:\\Program Files"}\\Google\\Chrome\\Application\\chrome.exe`, `${process.env["ProgramFiles(x86)"] || "C:\\Program Files (x86)"}\\Google\\Chrome\\Application\\chrome.exe`);
+import { renderUrlViaCdp } from "./chrome-cdp.js";
+import { DEFAULT_TIMEOUT_MS, DUMP_MAX_BYTES, chromeDumpDomArgs, discoverChromeBinaries, killProcessGroup, } from "./chrome-launch.js";
+export { DEFAULT_TIMEOUT_MS, VIRTUAL_TIME_MS, discoverChromeBinaries, findPlaywrightHeadlessShell, installedChromePath, isLocalChromeAvailable, parseDevToolsActivePort, chromeCdpArgs, chromeDumpDomArgs, killProcessGroup, } from "./chrome-launch.js";
+function attemptPlan(binary) {
+    const isShell = /headless[-_]?shell/i.test(binary);
+    if (isShell) {
+        return [
+            { driver: "cdp", headless: "shell" },
+            { driver: "dump-dom", headless: "shell" },
+        ];
     }
-    else {
-        candidates.push("/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/snap/bin/chromium");
-    }
-    for (const p of candidates) {
-        try {
-            if (p && existsSync(p))
-                return p;
-        }
-        catch {
-            /* not installed here */
-        }
-    }
-    return null;
+    return [
+        { driver: "cdp", headless: "new" },
+        { driver: "cdp", headless: "old" },
+        { driver: "dump-dom", headless: "old" },
+        { driver: "dump-dom", headless: "new" },
+    ];
 }
-export function isLocalChromeAvailable() {
-    return installedChromePath() !== null;
+function methodName(binary, kind) {
+    const base = binary.split(/[/\\]/).pop() || binary;
+    return `${kind.driver}/${kind.headless} (${base})`;
 }
 /**
  * Render `url` in local Chrome and return the hydrated HTML, or null when
  * Chrome is missing, times out, or produces nothing.
  */
 export async function renderViaLocalChrome(url, opts = {}) {
-    const chrome = installedChromePath();
-    if (!chrome)
-        return null;
-    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const result = await renderViaLocalChromeDetailed(url, opts);
+    return result.html;
+}
+/**
+ * Same as `renderViaLocalChrome` but with the attempt log and an actionable
+ * error when every driver fails. Callers that need to report honest JS-render
+ * failures should use this.
+ */
+export async function renderViaLocalChromeDetailed(url, opts = {}) {
+    const binaries = discoverChromeBinaries();
+    if (binaries.length === 0) {
+        return {
+            html: null,
+            error: "No local Chrome/Chromium binary found. Install Chrome, set WEBCRAWL_CHROME_PATH, " +
+                "or run `npx playwright install chromium-headless-shell`. See `webcrawl doctor`.",
+            attempts: [],
+        };
+    }
+    const budgetMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const started = Date.now();
+    const attempts = [];
+    let lastError = "local Chrome produced no content";
+    for (const binary of binaries) {
+        for (const kind of attemptPlan(binary)) {
+            const remaining = budgetMs - (Date.now() - started);
+            if (remaining < 1_200)
+                break;
+            const method = methodName(binary, kind);
+            const attemptBudget = kind.driver === "cdp"
+                ? Math.min(remaining, kind.headless === "new" ? 12_000 : 8_000)
+                : Math.min(remaining, 6_000);
+            const t0 = Date.now();
+            try {
+                const html = kind.driver === "cdp"
+                    ? (await renderUrlViaCdp(binary, url, {
+                        timeoutMs: attemptBudget,
+                        headless: kind.headless,
+                    })).html
+                    : await dumpDom(binary, url, kind.headless, attemptBudget);
+                const ms = Date.now() - t0;
+                if (html && html.length > 100) {
+                    attempts.push({ ok: true, method, ms });
+                    return { html, method, attempts };
+                }
+                lastError = `${method} returned empty HTML`;
+                attempts.push({ ok: false, method, ms, error: lastError });
+            }
+            catch (err) {
+                const ms = Date.now() - t0;
+                lastError = `${method}: ${err instanceof Error ? err.message : String(err)}`;
+                attempts.push({ ok: false, method, ms, error: lastError });
+            }
+        }
+    }
+    return {
+        html: null,
+        error: `JavaScript rendering failed (${lastError}). ` +
+            "Chrome hung, timed out, or produced no content. Install Chrome, point WEBCRAWL_CHROME_PATH " +
+            "at chrome-headless-shell (`npx playwright install chromium-headless-shell`), " +
+            "run `webcrawl login` and retry with `--mode cloud`, or set WEBCRAWL_RENDER_URL.",
+        attempts,
+    };
+}
+async function dumpDom(chrome, url, headless, timeoutMs) {
     let profile = "";
     try {
         profile = mkdtempSync(join(tmpdir(), "wc-chrome-"));
     }
     catch {
-        return null;
+        throw new Error("could not create Chrome profile directory");
     }
-    const args = [
-        "--headless=new",
-        "--no-sandbox",
-        "--disable-gpu",
-        "--disable-dev-shm-usage",
-        "--no-first-run",
-        "--no-default-browser-check",
-        "--disable-crash-reporter",
-        "--disable-breakpad",
-        `--user-data-dir=${profile}`,
-        `--virtual-time-budget=${VIRTUAL_TIME_MS}`,
-        `--timeout=${timeoutMs}`,
-        "--dump-dom",
+    const args = chromeDumpDomArgs({
+        profile,
+        headless,
         url,
-    ];
+        timeoutMs,
+        binaryPath: chrome,
+    });
     try {
-        return await new Promise((resolve) => {
+        return await new Promise((resolve, reject) => {
             let settled = false;
-            const done = (value) => {
+            const done = (err, value) => {
                 if (settled)
                     return;
                 settled = true;
-                resolve(value);
+                if (err)
+                    reject(err);
+                else
+                    resolve(value ?? "");
             };
             let child;
             try {
-                child = spawn(chrome, args, { stdio: ["ignore", "pipe", "pipe"] });
+                child = spawn(chrome, args, {
+                    stdio: ["ignore", "pipe", "pipe"],
+                    detached: true,
+                });
             }
-            catch {
-                done(null);
+            catch (err) {
+                done(err instanceof Error ? err : new Error(String(err)));
                 return;
             }
             const chunks = [];
             let bytes = 0;
-            const timer = setTimeout(() => {
-                try {
-                    child.kill("SIGKILL");
-                }
-                catch {
-                    /* already gone */
+            let idle = null;
+            const finish = (err) => {
+                clearTimeout(timer);
+                if (idle)
+                    clearTimeout(idle);
+                killProcessGroup(child.pid, child);
+                if (err) {
+                    done(err);
+                    return;
                 }
-                done(null);
-            }, timeoutMs + 2_000);
+                const text = Buffer.concat(chunks).toString("utf8");
+                done(null, text.trim());
+            };
+            const timer = setTimeout(() => {
+                if (bytes > 100)
+                    finish(null);
+                else
+                    finish(new Error(`dump-dom timed out after ${timeoutMs}ms`));
+            }, timeoutMs);
             child.stdout?.on("data", (c) => {
                 if (bytes >= DUMP_MAX_BYTES)
                     return;
                 const take = c.subarray(0, DUMP_MAX_BYTES - bytes);
                 chunks.push(take);
                 bytes += take.length;
+                // Chrome 156 often prints the DOM then never exits (`--dump-dom` hang).
+                if (idle)
+                    clearTimeout(idle);
+                idle = setTimeout(() => {
+                    if (bytes > 40)
+                        finish(null);
+                }, 400);
             });
-            child.on("error", () => {
+            child.on("error", (err) => {
                 clearTimeout(timer);
-                done(null);
+                if (idle)
+                    clearTimeout(idle);
+                done(err);
             });
             child.on("close", () => {
                 clearTimeout(timer);
+                if (idle)
+                    clearTimeout(idle);
                 if (bytes === 0) {
-                    done(null);
+                    done(new Error("dump-dom produced no stdout"));
                     return;
                 }
                 const text = Buffer.concat(chunks).toString("utf8");
-                done(text.trim().length > 0 ? text : null);
+                done(null, text.trim().length > 0 ? text : "");
             });
         });
     }

package/dist/core/extract/index.d.ts

@@ -6,4 +6,4 @@ export { dedupeHotelListings, extractHotelListingFromSerpResult, extractHotelLis
 export { dedupeBusinessListings, extractBusinessListingFromSerpResult, extractBusinessListingsFromHtml, type BusinessListing, } from "./business.js";
 export { cvss3BaseScore, cveSeverityLabel, extractCveIdsFromText, extractGhsaIdsFromText, looksLikeVulnId, mergeCveRecords, normalizeCveId, nvdUrl, parseNvdCve, parseNvdResponse, parseOsvVuln, severityFromScore, sortCveRecords, type CveAffected, type CveRecord, type CveSeverity, } from "./cve.js";
 export { extractBusinessListingsWithScaleDown, extractFlightOffersWithScaleDown, extractHotelListingsWithScaleDown, extractPropertyListingsWithScaleDown, } from "./scaledown.js";
-export { extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, shapeQuoteAuthors, wantsQuoteAuthorExtract, type QuoteAuthor, } from "./structured.js";
+export { coerceExtractSchema, extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, isEmptyExtractResult, isJsTemplateJunk, nestedQuoteAuthorFields, normalizeExtractSchema, shapeQuoteAuthors, stripNonContentHtml, wantsQuoteAuthorExtract, type QuoteAuthor, } from "./structured.js";

package/dist/core/extract/index.js

@@ -6,4 +6,4 @@ export { dedupeHotelListings, extractHotelListingFromSerpResult, extractHotelLis
 export { dedupeBusinessListings, extractBusinessListingFromSerpResult, extractBusinessListingsFromHtml, } from "./business.js";
 export { cvss3BaseScore, cveSeverityLabel, extractCveIdsFromText, extractGhsaIdsFromText, looksLikeVulnId, mergeCveRecords, normalizeCveId, nvdUrl, parseNvdCve, parseNvdResponse, parseOsvVuln, severityFromScore, sortCveRecords, } from "./cve.js";
 export { extractBusinessListingsWithScaleDown, extractFlightOffersWithScaleDown, extractHotelListingsWithScaleDown, extractPropertyListingsWithScaleDown, } from "./scaledown.js";
-export { extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, shapeQuoteAuthors, wantsQuoteAuthorExtract, } from "./structured.js";
+export { coerceExtractSchema, extractByPromptAndSchema, extractQuoteAuthorPairs, hasCustomExtractRequest, isEmptyExtractResult, isJsTemplateJunk, nestedQuoteAuthorFields, normalizeExtractSchema, shapeQuoteAuthors, stripNonContentHtml, wantsQuoteAuthorExtract, } from "./structured.js";

package/dist/core/extract/structured.d.ts

@@ -2,8 +2,27 @@ export interface QuoteAuthor {
     text: string;
     author: string;
 }
+/** A bare property map or shorthand array becomes a JSON Schema. */
+export declare function coerceExtractSchema(node: unknown): Record<string, unknown>;
+/**
+ * Root-array wrap: a top-level array becomes
+ * `{ type: object, properties: { items: <array> } }`.
+ */
+export declare function normalizeExtractSchema(schema?: Record<string, unknown>): Record<string, unknown> | undefined;
+export declare function schemaProperties(schema?: Record<string, unknown>): Record<string, unknown>;
 export declare function hasCustomExtractRequest(prompt?: string, schema?: Record<string, unknown>): boolean;
+/** Nested object/array field names that look like quote+author records. */
+export declare function nestedQuoteAuthorFields(schema?: Record<string, unknown>): {
+    arrayPath: string[];
+    textKey: string;
+    authorKey: string;
+} | null;
 export declare function wantsQuoteAuthorExtract(prompt?: string, schema?: Record<string, unknown>): boolean;
+/** Strip script/style/template (including unclosed tags) before reading visible DOM. */
+export declare function stripNonContentHtml(html: string): string;
+/** Inline `document.write("... " + d['text'] + ...)` leftovers, not real quotes. */
+export declare function isJsTemplateJunk(text: string): boolean;
+export declare function isEmptyExtractResult(data: Record<string, unknown> | null | undefined): boolean;
 /** Pull quote/author pairs from HTML (cards, microdata, `var data = [...]`) and markdown. */
 export declare function extractQuoteAuthorPairs(html: string, markdown?: string): QuoteAuthor[];
 export declare function shapeQuoteAuthors(pairs: QuoteAuthor[], schema?: Record<string, unknown>): Record<string, unknown>;

package/dist/core/extract/structured.js

@@ -1,16 +1,73 @@
 /**
  * Prompt/schema-aware structured extraction used by `extractWebsiteData`.
  * Deterministic (no LLM): quote/author cards, embedded JSON, and schema shaping.
+ *
+ * Nested objects/arrays follow a JSON-schema walk: shorthand
+ * `{items:[{quote,author}]}` is coerced to a real object/array schema,
+ * root arrays are wrapped as `{items: ...}`, and empty results are detectable.
  */
 import { decodeHtmlEntities } from "../engine/markdown.js";
-function schemaProperties(schema) {
+const CACHE_META_KEYS = new Set(["cached", "cacheLayer"]);
+function isPlainObject(v) {
+    return !!v && typeof v === "object" && !Array.isArray(v);
+}
+/** A bare property map or shorthand array becomes a JSON Schema. */
+export function coerceExtractSchema(node) {
+    if (Array.isArray(node)) {
+        const item = node.length > 0 ? coerceExtractSchema(node[0]) : { type: "object" };
+        return { type: "array", items: item };
+    }
+    if (typeof node === "string") {
+        const t = node.trim() || "string";
+        return { type: t };
+    }
+    if (!isPlainObject(node)) {
+        return { type: "string" };
+    }
+    if (typeof node.type === "string" || Array.isArray(node.type)) {
+        const out = { ...node };
+        if (isPlainObject(out.properties)) {
+            out.properties = Object.fromEntries(Object.entries(out.properties).map(([k, v]) => [k, coerceExtractSchema(v)]));
+        }
+        if (out.items !== undefined) {
+            out.items = coerceExtractSchema(out.items);
+        }
+        return out;
+    }
+    const properties = Object.fromEntries(Object.entries(node).map(([k, v]) => [k, coerceExtractSchema(v)]));
+    return {
+        type: "object",
+        properties,
+        required: Object.keys(properties),
+        additionalProperties: false,
+    };
+}
+/**
+ * Root-array wrap: a top-level array becomes
+ * `{ type: object, properties: { items: <array> } }`.
+ */
+export function normalizeExtractSchema(schema) {
     if (!schema || typeof schema !== "object")
+        return undefined;
+    let coerced = coerceExtractSchema(schema);
+    if (coerced.type === "array") {
+        coerced = {
+            type: "object",
+            properties: { items: coerced },
+            required: ["items"],
+            additionalProperties: false,
+        };
+    }
+    return coerced;
+}
+export function schemaProperties(schema) {
+    const normalized = normalizeExtractSchema(schema);
+    if (!normalized)
         return {};
-    const props = schema.properties;
-    if (props && typeof props === "object" && !Array.isArray(props)) {
+    const props = normalized.properties;
+    if (isPlainObject(props))
         return props;
-    }
-    return schema;
+    return normalized;
 }
 export function hasCustomExtractRequest(prompt, schema) {
     if (prompt && prompt.trim())
@@ -18,7 +75,58 @@ export function hasCustomExtractRequest(prompt, schema) {
     return Object.keys(schemaProperties(schema)).length > 0;
 }
 function blobForIntent(prompt, schema) {
-    return `${prompt ?? ""} ${Object.keys(schemaProperties(schema)).join(" ")}`.toLowerCase();
+    const keys = [];
+    const walk = (node) => {
+        if (Array.isArray(node)) {
+            for (const n of node)
+                walk(n);
+            return;
+        }
+        if (!isPlainObject(node))
+            return;
+        keys.push(...Object.keys(node));
+        if (isPlainObject(node.properties))
+            walk(node.properties);
+        if (node.items)
+            walk(node.items);
+    };
+    walk(schema);
+    walk(schemaProperties(schema));
+    return `${prompt ?? ""} ${keys.join(" ")}`.toLowerCase();
+}
+/** Nested object/array field names that look like quote+author records. */
+export function nestedQuoteAuthorFields(schema) {
+    const normalized = normalizeExtractSchema(schema);
+    if (!normalized)
+        return null;
+    const visit = (node, path) => {
+        if (node.type === "array" && node.items) {
+            const items = coerceExtractSchema(node.items);
+            const itemProps = isPlainObject(items.properties) ? items.properties : {};
+            const itemKeys = Object.keys(itemProps);
+            const textKey = itemKeys.find((k) => /quote|text|content|body/i.test(k));
+            const authorKey = itemKeys.find((k) => /author|by|name/i.test(k));
+            if (textKey && authorKey) {
+                return { arrayPath: path, textKey, authorKey };
+            }
+            if (items.type === "object" && isPlainObject(items.properties)) {
+                const nested = visit(items, path);
+                if (nested)
+                    return nested;
+            }
+        }
+        const props = isPlainObject(node.properties) ? node.properties : {};
+        for (const [key, value] of Object.entries(props)) {
+            if (!isPlainObject(value) && !Array.isArray(value))
+                continue;
+            const child = coerceExtractSchema(value);
+            const found = visit(child, [...path, key]);
+            if (found)
+                return found;
+        }
+        return null;
+    };
+    return visit(normalized, []);
 }
 export function wantsQuoteAuthorExtract(prompt, schema) {
     const blob = blobForIntent(prompt, schema);
@@ -27,8 +135,44 @@ export function wantsQuoteAuthorExtract(prompt, schema) {
     const props = schemaProperties(schema);
     if (props.quotes || props.quote || props.authors)
         return true;
+    if (nestedQuoteAuthorFields(schema))
+        return true;
     return false;
 }
+/** Strip script/style/template (including unclosed tags) before reading visible DOM. */
+export function stripNonContentHtml(html) {
+    return String(html ?? "")
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*/gi, "");
+}
+/** Inline `document.write("... " + d['text'] + ...)` leftovers, not real quotes. */
+export function isJsTemplateJunk(text) {
+    const t = String(text ?? "");
+    return (/\+\s*d\s*\[/i.test(t) ||
+        /['"]\s*\+\s*d/i.test(t) ||
+        /\bd\s*\[\s*['"]text['"]\s*\]/.test(t) ||
+        /\bd\s*\[\s*['"]author['"]\s*\]/.test(t) ||
+        /document\.write/i.test(t));
+}
+export function isEmptyExtractResult(data) {
+    if (!data || typeof data !== "object")
+        return true;
+    const keys = Object.keys(data).filter((k) => !CACHE_META_KEYS.has(k));
+    if (keys.length === 0)
+        return true;
+    return keys.every((k) => {
+        const v = data[k];
+        if (v == null || v === "")
+            return true;
+        if (Array.isArray(v))
+            return v.length === 0;
+        if (typeof v === "object" && !Array.isArray(v)) {
+            return isEmptyExtractResult(v);
+        }
+        return false;
+    });
+}
 function cleanText(raw) {
     return decodeHtmlEntities(raw.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim());
 }
@@ -37,6 +181,8 @@ function pushPair(out, text, author) {
     const a = cleanText(author);
     if (t.length < 8 || a.length < 2)
         return;
+    if (isJsTemplateJunk(t) || isJsTemplateJunk(a))
+        return;
     if (out.some((p) => p.text === t && p.author === a))
         return;
     out.push({ text: t, author: a });
@@ -44,7 +190,8 @@ function pushPair(out, text, author) {
 /** Pull quote/author pairs from HTML (cards, microdata, `var data = [...]`) and markdown. */
 export function extractQuoteAuthorPairs(html, markdown = "") {
     const out = [];
-    const dataAssign = html.match(/var\s+data\s*=\s*(\[[\s\S]*?\]);/);
+    const source = String(html ?? "");
+    const dataAssign = source.match(/var\s+data\s*=\s*(\[[\s\S]*?\]);/);
     if (dataAssign) {
         try {
             const parsed = JSON.parse(dataAssign[1]);
@@ -60,9 +207,10 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
             /* not JSON */
         }
     }
+    const visible = stripNonContentHtml(source);
     const cardRe = /<(?:div|figure|blockquote|li)[^>]*class=["'][^"']*\bquote\b[^"']*["'][^>]*>([\s\S]*?)<\/(?:div|figure|blockquote|li)>/gi;
     let card;
-    while ((card = cardRe.exec(html)) !== null) {
+    while ((card = cardRe.exec(visible)) !== null) {
         const block = card[1];
         const text = block.match(/<(?:span|div|p|q)[^>]*class=["'][^"']*\btext\b[^"']*["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
             block.match(/itemprop=["']text["'][^>]*>([\s\S]*?)<\//i)?.[1] ||
@@ -75,7 +223,7 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
     }
     const microRe = /itemprop=["']text["'][^>]*>([\s\S]*?)<\/[^>]+>[\s\S]{0,400}?itemprop=["']author["'][^>]*>([\s\S]*?)<\//gi;
     let micro;
-    while ((micro = microRe.exec(html)) !== null) {
+    while ((micro = microRe.exec(visible)) !== null) {
         pushPair(out, micro[1], micro[2]);
     }
     if (out.length === 0 && markdown) {
@@ -88,6 +236,14 @@ export function extractQuoteAuthorPairs(html, markdown = "") {
     return out;
 }
 function itemFieldNames(schema) {
+    const nested = nestedQuoteAuthorFields(schema);
+    if (nested && nested.arrayPath.length > 0) {
+        return {
+            arrayKey: nested.arrayPath[nested.arrayPath.length - 1] || "items",
+            textKey: nested.textKey,
+            authorKey: nested.authorKey,
+        };
+    }
     const props = schemaProperties(schema);
     const keys = Object.keys(props);
     const arrayKey = keys.find((k) => {
@@ -96,8 +252,7 @@ function itemFieldNames(schema) {
             return /quotes?|items|results|entries/i.test(k);
         const t = p.type;
         return t === "array" || /quotes?|items|results|entries/i.test(k);
-    }) ||
-        (keys.length === 0 ? "quotes" : undefined);
+    }) || (keys.length === 0 ? "quotes" : undefined);
     const itemSchema = arrayKey && props[arrayKey] && typeof props[arrayKey] === "object"
         ? props[arrayKey]
         : undefined;
@@ -107,6 +262,22 @@ function itemFieldNames(schema) {
     const authorKey = itemKeys.find((k) => /author|by|name/i.test(k)) ?? "author";
     return { arrayKey: arrayKey ?? "quotes", textKey, authorKey };
 }
+function assignAtPath(root, path, value) {
+    if (path.length === 0) {
+        return isPlainObject(value) ? { ...root, ...value } : root;
+    }
+    let cursor = root;
+    for (let i = 0; i < path.length - 1; i++) {
+        const key = path[i];
+        const next = cursor[key];
+        if (!isPlainObject(next)) {
+            cursor[key] = {};
+        }
+        cursor = cursor[key];
+    }
+    cursor[path[path.length - 1]] = value;
+    return root;
+}
 export function shapeQuoteAuthors(pairs, schema) {
     const props = schemaProperties(schema);
     const keys = Object.keys(props);
@@ -119,10 +290,13 @@ export function shapeQuoteAuthors(pairs, schema) {
             [authorArr]: pairs.map((p) => p.author),
         };
     }
+    const nested = nestedQuoteAuthorFields(schema);
     const { arrayKey, textKey, authorKey } = itemFieldNames(schema);
-    return {
-        [arrayKey]: pairs.map((p) => ({ [textKey]: p.text, [authorKey]: p.author })),
-    };
+    const rows = pairs.map((p) => ({ [textKey]: p.text, [authorKey]: p.author }));
+    if (nested && nested.arrayPath.length > 0) {
+        return assignAtPath({}, nested.arrayPath, rows);
+    }
+    return { [arrayKey]: rows };
 }
 /**
  * Deterministic extract when the caller passed a prompt and/or JSON schema.
@@ -131,13 +305,11 @@ export function shapeQuoteAuthors(pairs, schema) {
 export function extractByPromptAndSchema(html, markdown, prompt, schema) {
     if (!hasCustomExtractRequest(prompt, schema))
         return null;
-    if (wantsQuoteAuthorExtract(prompt, schema)) {
-        const pairs = extractQuoteAuthorPairs(html, markdown);
-        if (pairs.length > 0)
-            return shapeQuoteAuthors(pairs, schema);
-    }
     const pairs = extractQuoteAuthorPairs(html, markdown);
-    if (pairs.length > 0 && /\bquotes?\b/i.test(blobForIntent(prompt, schema))) {
+    if (wantsQuoteAuthorExtract(prompt, schema) && pairs.length > 0) {
+        return shapeQuoteAuthors(pairs, schema);
+    }
+    if (pairs.length > 0 && /\b(quotes?|items|results|entries)\b/i.test(blobForIntent(prompt, schema))) {
         return shapeQuoteAuthors(pairs, schema);
     }
     return null;

package/dist/core/local-engine.d.ts

@@ -38,6 +38,12 @@ export interface LocalEngineBackend {
  * for content. Exported for unit tests.
  */
 export declare function blockedScrapeResult(url: string, statusCode: number, tier: string, latencyMs: number, reason?: string): Record<string, unknown>;
+/**
+ * JS page whose renderer failed. `success: false` so callers never treat the
+ * unrendered nav shell as content (v1.4.3 smoke test: 114 chars of nav with
+ * success:true after Chrome hung on --dump-dom).
+ */
+export declare function jsRenderFailedResult(url: string, statusCode: number, tier: string, latencyMs: number, reason: string): Record<string, unknown>;
 /** Shells shorter than this are unrendered JS / empty pages and must not be cached as content. */
 export declare const MIN_CACHEABLE_MARKDOWN = 200;
 /** Attach source HTML when the caller asked for the `html` format. */