@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/tools/crawl.js

@@ -1,97 +1,180 @@
-import { Crawler } from '../crawl/crawler.js';
-import { deduplicatePages } from '../crawl/dedup.js';
-import { mapUrls } from '../crawl/mapper.js';
-import { handleFetch } from './fetch.js';
-import { createLogger } from '../logger.js';
-const log = createLogger('crawl');
-const DEFAULT_MAX_TOTAL_CHARS = 100000;
-export async function handleCrawl(input, router) {
-    try {
-        // Map strategy: lightweight URL-only discovery, skip full crawl pipeline
-        if (input.strategy === 'map') {
-            return handleMapStrategy(input, router);
-        }
-        const fetchFn = async (url) => handleFetch({ url, use_auth: input.use_auth }, router);
-        const rawFetchFn = async (url) => router.fetch(url, { renderJs: 'never' });
-        const crawler = new Crawler(fetchFn, rawFetchFn);
-        const result = await crawler.crawl(input);
-        // Deduplicate cross-page content (pass domain for SQLite boilerplate caching)
-        const domain = new URL(input.url).hostname;
-        const dedupedPages = deduplicatePages(result.pages.map((p) => ({ url: p.url, markdown: p.markdown })), domain);
-        // Apply deduped markdown back to pages
-        const pages = result.pages.map((page, i) => ({
-            ...page,
-            markdown: dedupedPages[i]?.markdown ?? page.markdown,
-        }));
-        // Enforce max_total_chars budget
-        const maxTotalChars = input.max_total_chars ?? DEFAULT_MAX_TOTAL_CHARS;
-        const budgetedPages = [];
-        let charCount = 0;
-        for (const page of pages) {
-            if (charCount + page.markdown.length > maxTotalChars && budgetedPages.length > 0) {
-                break;
-            }
-            budgetedPages.push(page);
-            charCount += page.markdown.length;
-        }
-        log.info('Crawl complete', {
-            url: input.url,
-            crawled: result.crawled,
-            returned: budgetedPages.length,
-            totalChars: charCount,
-        });
-        return {
-            pages: budgetedPages,
-            total_found: result.total_found,
-            crawled: result.crawled,
-            ...(result.links ? { links: result.links } : {}),
-        };
+import { Crawler } from "../crawl/crawler.js";
+import { deduplicatePages } from "../crawl/dedup.js";
+import { mapUrls } from "../crawl/mapper.js";
+import { handleFetch } from "./fetch.js";
+import {
+  buildEvidenceFromMarkdown,
+  applyAggregateMarkdownBudget
+} from "../search/evidence.js";
+import { countTokens } from "../search/tokens.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("crawl");
+const DEFAULT_MAX_TOTAL_CHARS = 1e5;
+const DEFAULT_MAX_TOKENS_OUT = 4e3;
+async function handleCrawl(input, router) {
+  try {
+    if (input.strategy === "map") {
+      return handleMapStrategy(input, router);
     }
-    catch (err) {
-        log.error('Crawl failed', { url: input.url, error: String(err) });
+    const fetchFn = async (url) => {
+      const r = await handleFetch({ url, use_auth: input.use_auth, include_full_markdown: true }, router);
+      if (!r.ok) {
         return {
-            pages: [],
-            total_found: 0,
-            crawled: 0,
-            error: err instanceof Error ? err.message : String(err),
+          url,
+          title: "",
+          markdown: "",
+          metadata: {},
+          links: [],
+          images: [],
+          cached: false,
+          error: r.error_reason
         };
+      }
+      return r.data;
+    };
+    const rawFetchFn = async (url) => router.fetch(url, { renderJs: "never" });
+    const crawler = new Crawler(fetchFn, rawFetchFn);
+    const result = await crawler.crawl(input);
+    const domain = new URL(input.url).hostname;
+    const dedupedPages = deduplicatePages(
+      result.pages.map((p) => ({ url: p.url, markdown: p.markdown })),
+      domain
+    );
+    const pages = result.pages.map((page, i) => ({
+      ...page,
+      markdown: dedupedPages[i]?.markdown ?? page.markdown
+    }));
+    const maxTotalChars = input.max_total_chars ?? DEFAULT_MAX_TOTAL_CHARS;
+    const budgetedPages = [];
+    let charCount = 0;
+    for (const page of pages) {
+      if (charCount + page.markdown.length > maxTotalChars && budgetedPages.length > 0) {
+        break;
+      }
+      budgetedPages.push(page);
+      charCount += page.markdown.length;
     }
-}
-async function handleMapStrategy(input, router) {
-    const httpFetchFn = async (url) => {
-        const raw = await router.fetch(url, { renderJs: 'never' });
-        return { html: raw.html, finalUrl: raw.finalUrl, statusCode: raw.statusCode };
+    const droppedOverBudget = result.crawled - budgetedPages.length;
+    log.info("Crawl complete", {
+      url: input.url,
+      crawled: result.crawled,
+      returned: budgetedPages.length,
+      totalChars: charCount,
+      droppedOverBudget
+    });
+    const out = {
+      pages: budgetedPages,
+      total_found: result.total_found,
+      crawled: budgetedPages.length,
+      ...droppedOverBudget > 0 ? { dropped_over_budget: droppedOverBudget } : {},
+      ...result.links ? { links: result.links } : {}
     };
-    try {
-        const mapResult = await mapUrls({
-            url: input.url,
-            max_depth: input.max_depth,
-            max_pages: input.max_pages,
-            include_patterns: input.include_patterns,
-            exclude_patterns: input.exclude_patterns,
-        }, httpFetchFn);
-        log.info('Map complete', {
-            url: input.url,
-            total_found: mapResult.total_found,
-            sitemap_found: mapResult.sitemap_found,
-        });
-        return {
-            urls: mapResult.urls,
-            total_found: mapResult.total_found,
-            sitemap_found: mapResult.sitemap_found,
-            crawled: 0,
-            ...(mapResult.error ? { error: mapResult.error } : {}),
-        };
+    await attachEvidence(out, input);
+    return out;
+  } catch (err) {
+    log.error("Crawl failed", { url: input.url, error: String(err) });
+    return {
+      pages: [],
+      total_found: 0,
+      crawled: 0,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+function buildPageExcerpt(markdown, maxChars = 600) {
+  if (!markdown) return "";
+  const paragraphs = markdown.split(/\n\n+/).map((p) => p.trim()).filter(Boolean);
+  let out = "";
+  for (const p of paragraphs) {
+    if (out.length + p.length + 2 > maxChars) {
+      const remaining = maxChars - out.length;
+      if (remaining > 80) out += (out ? "\n\n" : "") + p.slice(0, remaining) + "\u2026";
+      break;
     }
-    catch (err) {
-        log.error('Map strategy failed', { url: input.url, error: String(err) });
-        return {
-            urls: [],
-            total_found: 0,
-            sitemap_found: false,
-            crawled: 0,
-            error: err instanceof Error ? err.message : String(err),
-        };
+    out += (out ? "\n\n" : "") + p;
+  }
+  return out;
+}
+async function attachEvidence(out, input) {
+  if (out.pages.length === 0) return;
+  const includeFull = input.include_full_markdown ?? false;
+  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
+  let used = 0;
+  for (const page of out.pages) {
+    if (!page.markdown) continue;
+    const remaining = maxTokensOut - used;
+    if (remaining <= 0) break;
+    const evs = await buildEvidenceFromMarkdown(
+      page.title || page.url,
+      page.title,
+      page.url,
+      page.markdown,
+      { maxItems: 1, maxTokensOut: remaining }
+    );
+    if (evs.length > 0) {
+      page.evidence = evs;
+      for (const ev of evs) used += countTokens(ev.excerpt);
+    }
+  }
+  if (!includeFull) {
+    for (const page of out.pages) {
+      if (!page.evidence || page.evidence.length === 0) {
+        const excerpt = buildPageExcerpt(page.markdown);
+        if (excerpt) page.excerpt = excerpt;
+      }
+      page.markdown = "";
     }
+  } else {
+    applyAggregateMarkdownBudget(
+      out.pages,
+      (p) => p.markdown ?? "",
+      (p, body) => {
+        p.markdown = body;
+      },
+      { maxTokensOut }
+    );
+  }
+}
+async function handleMapStrategy(input, router) {
+  const httpFetchFn = async (url) => {
+    const raw = await router.fetch(url, { renderJs: "never" });
+    return { html: raw.html, finalUrl: raw.finalUrl, statusCode: raw.statusCode };
+  };
+  try {
+    const mapResult = await mapUrls(
+      {
+        url: input.url,
+        max_depth: input.max_depth,
+        max_pages: input.max_pages,
+        include_patterns: input.include_patterns,
+        exclude_patterns: input.exclude_patterns
+      },
+      httpFetchFn
+    );
+    log.info("Map complete", {
+      url: input.url,
+      total_found: mapResult.total_found,
+      sitemap_found: mapResult.sitemap_found
+    });
+    return {
+      urls: mapResult.urls,
+      total_found: mapResult.total_found,
+      sitemap_found: mapResult.sitemap_found,
+      crawled: 0,
+      ...mapResult.error ? { error: mapResult.error } : {}
+    };
+  } catch (err) {
+    log.error("Map strategy failed", { url: input.url, error: String(err) });
+    return {
+      urls: [],
+      total_found: 0,
+      sitemap_found: false,
+      crawled: 0,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
 }
+export {
+  handleCrawl
+};
 //# sourceMappingURL=crawl.js.map

package/dist/tools/crawl.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,yEAAyE;QACzE,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC7B,OAAO,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACpC,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1C,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC3C,MAAM,YAAY,GAAG,gBAAgB,CACnC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAC/D,MAAM,CACP,CAAC;QAEF,uCAAuC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG,IAAI;YACP,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ;SACrD,CAAC,CAAC,CAAC;QAEJ,iCAAiC;QACjC,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,uBAAuB,CAAC;QACvE,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM;YACR,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE,aAAa,CAAC,MAAM;YAC9B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAiB,EACjB,MAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAChF,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAC7B;YACE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC,EACD,WAAW,CACZ,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;SACvC,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;YACtC,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO;YACL,IAAI,EAAE,EAAE;YACR,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,KAAK;YACpB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/tools/crawl.ts"],"sourcesContent":["import type {\n  CrawlInput,\n  CrawlOutput,\n  MapOutput,\n} from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport { Crawler } from '../crawl/crawler.js';\nimport { deduplicatePages } from '../crawl/dedup.js';\nimport { mapUrls } from '../crawl/mapper.js';\nimport { handleFetch } from './fetch.js';\nimport {\n  buildEvidenceFromMarkdown,\n  applyAggregateMarkdownBudget,\n} from '../search/evidence.js';\nimport { countTokens } from '../search/tokens.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('crawl');\n\nconst DEFAULT_MAX_TOTAL_CHARS = 100000;\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n\nexport async function handleCrawl(\n  input: CrawlInput,\n  router: SmartRouter,\n): Promise<CrawlOutput | (MapOutput & { crawled: number })> {\n  try {\n    // Map strategy: lightweight URL-only discovery, skip full crawl pipeline\n    if (input.strategy === 'map') {\n      return handleMapStrategy(input, router);\n    }\n\n    // Crawler needs full markdown internally for dedup; opt in explicitly so\n    // handleFetch's default strip does not steal page bodies mid-crawl.\n    const fetchFn = async (url: string) => {\n      const r = await handleFetch({ url, use_auth: input.use_auth, include_full_markdown: true }, router);\n      if (!r.ok) {\n        return {\n          url,\n          title: '',\n          markdown: '',\n          metadata: {},\n          links: [],\n          images: [],\n          cached: false,\n          error: r.error_reason,\n        };\n      }\n      return r.data;\n    };\n\n    const rawFetchFn = async (url: string) =>\n      router.fetch(url, { renderJs: 'never' });\n\n    const crawler = new Crawler(fetchFn, rawFetchFn);\n    const result = await crawler.crawl(input);\n\n    // Deduplicate cross-page content (pass domain for SQLite boilerplate caching)\n    const domain = new URL(input.url).hostname;\n    const dedupedPages = deduplicatePages(\n      result.pages.map((p) => ({ url: p.url, markdown: p.markdown })),\n      domain,\n    );\n\n    // Apply deduped markdown back to pages\n    const pages = result.pages.map((page, i) => ({\n      ...page,\n      markdown: dedupedPages[i]?.markdown ?? page.markdown,\n    }));\n\n    // Enforce max_total_chars budget\n    const maxTotalChars = input.max_total_chars ?? DEFAULT_MAX_TOTAL_CHARS;\n    const budgetedPages = [];\n    let charCount = 0;\n\n    for (const page of pages) {\n      if (charCount + page.markdown.length > maxTotalChars && budgetedPages.length > 0) {\n        break;\n      }\n      budgetedPages.push(page);\n      charCount += page.markdown.length;\n    }\n\n    const droppedOverBudget = result.crawled - budgetedPages.length;\n    log.info('Crawl complete', {\n      url: input.url,\n      crawled: result.crawled,\n      returned: budgetedPages.length,\n      totalChars: charCount,\n      droppedOverBudget,\n    });\n\n    const out: CrawlOutput = {\n      pages: budgetedPages,\n      total_found: result.total_found,\n      crawled: budgetedPages.length,\n      ...(droppedOverBudget > 0 ? { dropped_over_budget: droppedOverBudget } : {}),\n      ...(result.links ? { links: result.links } : {}),\n    };\n\n    await attachEvidence(out, input);\n    return out;\n  } catch (err) {\n    log.error('Crawl failed', { url: input.url, error: String(err) });\n    return {\n      pages: [],\n      total_found: 0,\n      crawled: 0,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n\nfunction buildPageExcerpt(markdown: string, maxChars = 600): string {\n  if (!markdown) return '';\n  const paragraphs = markdown.split(/\\n\\n+/).map((p) => p.trim()).filter(Boolean);\n  let out = '';\n  for (const p of paragraphs) {\n    if (out.length + p.length + 2 > maxChars) {\n      const remaining = maxChars - out.length;\n      if (remaining > 80) out += (out ? '\\n\\n' : '') + p.slice(0, remaining) + '…';\n      break;\n    }\n    out += (out ? '\\n\\n' : '') + p;\n  }\n  return out;\n}\n\nasync function attachEvidence(out: CrawlOutput, input: CrawlInput): Promise<void> {\n  if (out.pages.length === 0) return;\n  const includeFull = input.include_full_markdown ?? false;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n\n  let used = 0;\n  for (const page of out.pages) {\n    if (!page.markdown) continue;\n    const remaining = maxTokensOut - used;\n    if (remaining <= 0) break;\n    const evs = await buildEvidenceFromMarkdown(\n      page.title || page.url,\n      page.title,\n      page.url,\n      page.markdown,\n      { maxItems: 1, maxTokensOut: remaining },\n    );\n    if (evs.length > 0) {\n      page.evidence = evs;\n      for (const ev of evs) used += countTokens(ev.excerpt);\n    }\n  }\n\n  if (!includeFull) {\n    // No full markdown: still surface a short excerpt per page so the\n    // result is useful when evidence couldn't be built (no query to highlight).\n    for (const page of out.pages) {\n      if (!page.evidence || page.evidence.length === 0) {\n        const excerpt = buildPageExcerpt(page.markdown);\n        if (excerpt) page.excerpt = excerpt;\n      }\n      page.markdown = '';\n    }\n  } else {\n    applyAggregateMarkdownBudget(\n      out.pages,\n      (p) => p.markdown ?? '',\n      (p, body) => { p.markdown = body; },\n      { maxTokensOut },\n    );\n  }\n}\n\nasync function handleMapStrategy(\n  input: CrawlInput,\n  router: SmartRouter,\n): Promise<MapOutput & { crawled: number }> {\n  const httpFetchFn = async (url: string) => {\n    const raw = await router.fetch(url, { renderJs: 'never' });\n    return { html: raw.html, finalUrl: raw.finalUrl, statusCode: raw.statusCode };\n  };\n\n  try {\n    const mapResult = await mapUrls(\n      {\n        url: input.url,\n        max_depth: input.max_depth,\n        max_pages: input.max_pages,\n        include_patterns: input.include_patterns,\n        exclude_patterns: input.exclude_patterns,\n      },\n      httpFetchFn,\n    );\n\n    log.info('Map complete', {\n      url: input.url,\n      total_found: mapResult.total_found,\n      sitemap_found: mapResult.sitemap_found,\n    });\n\n    return {\n      urls: mapResult.urls,\n      total_found: mapResult.total_found,\n      sitemap_found: mapResult.sitemap_found,\n      crawled: 0,\n      ...(mapResult.error ? { error: mapResult.error } : {}),\n    };\n  } catch (err) {\n    log.error('Map strategy failed', { url: input.url, error: String(err) });\n    return {\n      urls: [],\n      total_found: 0,\n      sitemap_found: false,\n      crawled: 0,\n      error: err instanceof Error ? err.message : String(err),\n    };\n  }\n}\n"],"mappings":"AAMA,SAAS,eAAe;AACxB,SAAS,wBAAwB;AACjC,SAAS,eAAe;AACxB,SAAS,mBAAmB;AAC5B;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,0BAA0B;AAChC,MAAM,yBAAyB;AAE/B,eAAsB,YACpB,OACA,QAC0D;AAC1D,MAAI;AAEF,QAAI,MAAM,aAAa,OAAO;AAC5B,aAAO,kBAAkB,OAAO,MAAM;AAAA,IACxC;AAIA,UAAM,UAAU,OAAO,QAAgB;AACrC,YAAM,IAAI,MAAM,YAAY,EAAE,KAAK,UAAU,MAAM,UAAU,uBAAuB,KAAK,GAAG,MAAM;AAClG,UAAI,CAAC,EAAE,IAAI;AACT,eAAO;AAAA,UACL;AAAA,UACA,OAAO;AAAA,UACP,UAAU;AAAA,UACV,UAAU,CAAC;AAAA,UACX,OAAO,CAAC;AAAA,UACR,QAAQ,CAAC;AAAA,UACT,QAAQ;AAAA,UACR,OAAO,EAAE;AAAA,QACX;AAAA,MACF;AACA,aAAO,EAAE;AAAA,IACX;AAEA,UAAM,aAAa,OAAO,QACxB,OAAO,MAAM,KAAK,EAAE,UAAU,QAAQ,CAAC;AAEzC,UAAM,UAAU,IAAI,QAAQ,SAAS,UAAU;AAC/C,UAAM,SAAS,MAAM,QAAQ,MAAM,KAAK;AAGxC,UAAM,SAAS,IAAI,IAAI,MAAM,GAAG,EAAE;AAClC,UAAM,eAAe;AAAA,MACnB,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,UAAU,EAAE,SAAS,EAAE;AAAA,MAC9D;AAAA,IACF;AAGA,UAAM,QAAQ,OAAO,MAAM,IAAI,CAAC,MAAM,OAAO;AAAA,MAC3C,GAAG;AAAA,MACH,UAAU,aAAa,CAAC,GAAG,YAAY,KAAK;AAAA,IAC9C,EAAE;AAGF,UAAM,gBAAgB,MAAM,mBAAmB;AAC/C,UAAM,gBAAgB,CAAC;AACvB,QAAI,YAAY;AAEhB,eAAW,QAAQ,OAAO;AACxB,UAAI,YAAY,KAAK,SAAS,SAAS,iBAAiB,cAAc,SAAS,GAAG;AAChF;AAAA,MACF;AACA,oBAAc,KAAK,IAAI;AACvB,mBAAa,KAAK,SAAS;AAAA,IAC7B;AAEA,UAAM,oBAAoB,OAAO,UAAU,cAAc;AACzD,QAAI,KAAK,kBAAkB;AAAA,MACzB,KAAK,MAAM;AAAA,MACX,SAAS,OAAO;AAAA,MAChB,UAAU,cAAc;AAAA,MACxB,YAAY;AAAA,MACZ;AAAA,IACF,CAAC;AAED,UAAM,MAAmB;AAAA,MACvB,OAAO;AAAA,MACP,aAAa,OAAO;AAAA,MACpB,SAAS,cAAc;AAAA,MACvB,GAAI,oBAAoB,IAAI,EAAE,qBAAqB,kBAAkB,IAAI,CAAC;AAAA,MAC1E,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;AAAA,IAChD;AAEA,UAAM,eAAe,KAAK,KAAK;AAC/B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,gBAAgB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,WAAO;AAAA,MACL,OAAO,CAAC;AAAA,MACR,aAAa;AAAA,MACb,SAAS;AAAA,MACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,UAAkB,WAAW,KAAa;AAClE,MAAI,CAAC,SAAU,QAAO;AACtB,QAAM,aAAa,SAAS,MAAM,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AAC9E,MAAI,MAAM;AACV,aAAW,KAAK,YAAY;AAC1B,QAAI,IAAI,SAAS,EAAE,SAAS,IAAI,UAAU;AACxC,YAAM,YAAY,WAAW,IAAI;AACjC,UAAI,YAAY,GAAI,SAAQ,MAAM,SAAS,MAAM,EAAE,MAAM,GAAG,SAAS,IAAI;AACzE;AAAA,IACF;AACA,YAAQ,MAAM,SAAS,MAAM;AAAA,EAC/B;AACA,SAAO;AACT;AAEA,eAAe,eAAe,KAAkB,OAAkC;AAChF,MAAI,IAAI,MAAM,WAAW,EAAG;AAC5B,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAE7C,MAAI,OAAO;AACX,aAAW,QAAQ,IAAI,OAAO;AAC5B,QAAI,CAAC,KAAK,SAAU;AACpB,UAAM,YAAY,eAAe;AACjC,QAAI,aAAa,EAAG;AACpB,UAAM,MAAM,MAAM;AAAA,MAChB,KAAK,SAAS,KAAK;AAAA,MACnB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK;AAAA,MACL,EAAE,UAAU,GAAG,cAAc,UAAU;AAAA,IACzC;AACA,QAAI,IAAI,SAAS,GAAG;AAClB,WAAK,WAAW;AAChB,iBAAW,MAAM,IAAK,SAAQ,YAAY,GAAG,OAAO;AAAA,IACtD;AAAA,EACF;AAEA,MAAI,CAAC,aAAa;AAGhB,eAAW,QAAQ,IAAI,OAAO;AAC5B,UAAI,CAAC,KAAK,YAAY,KAAK,SAAS,WAAW,GAAG;AAChD,cAAM,UAAU,iBAAiB,KAAK,QAAQ;AAC9C,YAAI,QAAS,MAAK,UAAU;AAAA,MAC9B;AACA,WAAK,WAAW;AAAA,IAClB;AAAA,EACF,OAAO;AACL;AAAA,MACE,IAAI;AAAA,MACJ,CAAC,MAAM,EAAE,YAAY;AAAA,MACrB,CAAC,GAAG,SAAS;AAAE,UAAE,WAAW;AAAA,MAAM;AAAA,MAClC,EAAE,aAAa;AAAA,IACjB;AAAA,EACF;AACF;AAEA,eAAe,kBACb,OACA,QAC0C;AAC1C,QAAM,cAAc,OAAO,QAAgB;AACzC,UAAM,MAAM,MAAM,OAAO,MAAM,KAAK,EAAE,UAAU,QAAQ,CAAC;AACzD,WAAO,EAAE,MAAM,IAAI,MAAM,UAAU,IAAI,UAAU,YAAY,IAAI,WAAW;AAAA,EAC9E;AAEA,MAAI;AACF,UAAM,YAAY,MAAM;AAAA,MACtB;AAAA,QACE,KAAK,MAAM;AAAA,QACX,WAAW,MAAM;AAAA,QACjB,WAAW,MAAM;AAAA,QACjB,kBAAkB,MAAM;AAAA,QACxB,kBAAkB,MAAM;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,KAAK,gBAAgB;AAAA,MACvB,KAAK,MAAM;AAAA,MACX,aAAa,UAAU;AAAA,MACvB,eAAe,UAAU;AAAA,IAC3B,CAAC;AAED,WAAO;AAAA,MACL,MAAM,UAAU;AAAA,MAChB,aAAa,UAAU;AAAA,MACvB,eAAe,UAAU;AAAA,MACzB,SAAS;AAAA,MACT,GAAI,UAAU,QAAQ,EAAE,OAAO,UAAU,MAAM,IAAI,CAAC;AAAA,IACtD;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,uBAAuB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AACvE,WAAO;AAAA,MACL,MAAM,CAAC;AAAA,MACP,aAAa;AAAA,MACb,eAAe;AAAA,MACf,SAAS;AAAA,MACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD;AAAA,EACF;AACF;","names":[]}

package/dist/tools/extract.d.ts

@@ -1,4 +1,4 @@
-import type { ExtractInput, ExtractOutput } from '../types.js';
+import type { ExtractInput, ExtractOutput, StageResult } from '../types.js';
 import type { SmartRouter } from '../fetch/router.js';
-export declare function handleExtract(input: ExtractInput, router: SmartRouter): Promise<ExtractOutput>;
+export declare function handleExtract(input: ExtractInput, router: SmartRouter): Promise<StageResult<ExtractOutput>>;
 //# sourceMappingURL=extract.d.ts.map

package/dist/tools/extract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA8BtD,wBAAsB,aAAa,CACjC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,aAAa,CAAC,CAoDxB"}
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC5E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA8CtD,wBAAsB,aAAa,CACjC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CA+JrC"}

package/dist/tools/extract.js

@@ -1,69 +1,185 @@
-import { extractMetadata, extractSelector, extractTables } from '../extraction/extract.js';
-import { extractWithSchema } from '../extraction/schema.js';
-import { extractJsonLd } from '../extraction/jsonld.js';
-import { getCachedContent, isExpired } from '../cache/store.js';
-import { createLogger } from '../logger.js';
-const log = createLogger('extract');
+import { extractMetadata, extractSelector, extractTables } from "../extraction/extract.js";
+import {
+  extractWithSchema,
+  extractWithSchemaDetailedAsync
+} from "../extraction/schema.js";
+import { extractJsonLd } from "../extraction/jsonld.js";
+import { extractStructured } from "../extraction/structured.js";
+import { getCachedContent, isExpired } from "../cache/store.js";
+import { fetchWithPlaywright } from "../fetch/playwright-tier.js";
+import { createLogger } from "../logger.js";
+import {
+  isNamedSchemaType,
+  extractNamedSchema,
+  NAMED_SCHEMAS
+} from "../extraction/v1/schemas/index.js";
+import { isLocalLlmEnabled, extractWithLocalLlm } from "../extraction/v1/local-llm.js";
+const log = createLogger("extract");
 async function resolveHtml(input, router) {
-    if (input.url) {
-        const cached = getCachedContent(input.url);
-        if (cached && !isExpired(cached)) {
-            log.info('Using cached HTML', { url: input.url });
-            return { html: cached.rawHtml, sourceUrl: cached.url };
-        }
-        const raw = await router.fetch(input.url, {
-            renderJs: 'auto',
-            useAuth: false,
-        });
-        return { html: raw.html, sourceUrl: raw.finalUrl };
+  if (input.execution_mode === "stealth" && input.url) {
+    const pw = await fetchWithPlaywright(input.url);
+    return { html: pw.html, sourceUrl: input.url };
+  }
+  if (input.url) {
+    const cached = getCachedContent(input.url);
+    if (cached && !isExpired(cached)) {
+      log.info("Using cached HTML", { url: input.url });
+      return { html: cached.rawHtml, sourceUrl: cached.url };
     }
-    return { html: input.html };
+    const raw = await router.fetch(input.url, {
+      renderJs: "auto",
+      useAuth: false
+    });
+    return { html: raw.html, sourceUrl: raw.finalUrl };
+  }
+  return { html: input.html };
 }
-export async function handleExtract(input, router) {
-    const mode = input.mode ?? 'metadata';
-    if (!input.url && !input.html) {
-        return { data: {}, mode, error: 'Either url or html must be provided' };
-    }
-    if (mode === 'selector' && !input.css_selector) {
-        return { data: '', mode, error: 'css_selector is required when mode is "selector"' };
+async function handleExtract(input, router) {
+  const mode = input.mode ?? "metadata";
+  if (!input.url && !input.html) {
+    return {
+      ok: false,
+      error: "invalid_input",
+      error_reason: "Either url or html must be provided",
+      stage: "extract"
+    };
+  }
+  if (input.named_schema && input.schema) {
+    return {
+      ok: false,
+      error: "invalid_input",
+      error_reason: "schema and named_schema are mutually exclusive",
+      stage: "extract"
+    };
+  }
+  if (input.named_schema && !isNamedSchemaType(input.named_schema)) {
+    return {
+      ok: false,
+      error: "invalid_input",
+      error_reason: `Unknown named_schema. Valid: ${NAMED_SCHEMAS.join(", ")}`,
+      stage: "extract"
+    };
+  }
+  if (mode === "selector" && !input.css_selector) {
+    return {
+      ok: false,
+      error: "invalid_input",
+      error_reason: 'css_selector is required when mode is "selector"',
+      stage: "extract"
+    };
+  }
+  if (mode === "schema" && !input.named_schema && (!input.schema || !input.schema.properties)) {
+    return {
+      ok: false,
+      error: "invalid_input",
+      error_reason: 'schema is required when mode is "schema" and must have properties',
+      stage: "extract"
+    };
+  }
+  try {
+    const { html, sourceUrl } = await resolveHtml(input, router);
+    if (input.named_schema) {
+      const namedData = await extractNamedSchema(input.named_schema, html, sourceUrl ?? input.url ?? "");
+      if (namedData === null) {
+        return {
+          ok: true,
+          data: {
+            data: {},
+            source_url: sourceUrl,
+            mode: "schema",
+            error: `No ${input.named_schema} data found on page`
+          }
+        };
+      }
+      return {
+        ok: true,
+        data: {
+          data: namedData,
+          source_url: sourceUrl,
+          mode: "schema"
+        }
+      };
     }
-    if (mode === 'schema' && (!input.schema || !input.schema.properties)) {
-        return { data: {}, mode, error: 'schema is required when mode is "schema" and must have properties' };
+    if (mode === "schema" && input.schema && isLocalLlmEnabled()) {
+      const llmData = await extractWithLocalLlm({
+        schema: input.schema,
+        html,
+        url: sourceUrl ?? input.url ?? ""
+      });
+      return {
+        ok: true,
+        data: {
+          data: llmData ?? {},
+          source_url: sourceUrl,
+          mode: "schema"
+        }
+      };
     }
-    try {
-        const { html, sourceUrl } = await resolveHtml(input, router);
-        let data;
-        switch (mode) {
-            case 'selector':
-                data = extractSelector(html, input.css_selector, input.multiple ?? false);
-                break;
-            case 'tables':
-                data = extractTables(html);
-                break;
-            case 'schema':
-                data = extractWithSchema(html, input.schema);
-                break;
-            case 'metadata':
-            default: {
-                const meta = extractMetadata(html);
-                const jsonld = extractJsonLd(html);
-                if (jsonld.length > 0) {
-                    meta.jsonld = jsonld;
-                }
-                data = meta;
-                break;
-            }
+    let data;
+    switch (mode) {
+      case "selector":
+        data = extractSelector(html, input.css_selector, input.multiple ?? false);
+        break;
+      case "tables":
+        data = extractTables(html);
+        break;
+      case "structured":
+        data = extractStructured(html);
+        break;
+      case "schema": {
+        const schema = input.schema;
+        if (Array.isArray(schema.required) && schema.required.length > 0) {
+          const detailed = await extractWithSchemaDetailedAsync(html, schema);
+          data = detailed.values;
+          if (detailed.warnings.length > 0) {
+            return {
+              ok: true,
+              data: {
+                data,
+                source_url: sourceUrl,
+                mode,
+                warnings: detailed.warnings
+              }
+            };
+          }
+        } else {
+          data = extractWithSchema(html, schema);
+        }
+        break;
+      }
+      case "metadata":
+      default: {
+        const meta = extractMetadata(html);
+        const jsonld = extractJsonLd(html);
+        if (jsonld.length > 0) {
+          meta.jsonld = jsonld;
         }
-        return { data, source_url: sourceUrl, mode };
+        data = meta;
+        break;
+      }
     }
-    catch (err) {
-        log.error('Extract failed', { url: input.url, error: String(err) });
-        return {
-            data: mode === 'selector' ? '' : mode === 'tables' ? [] : {},
-            source_url: input.url,
-            mode,
-            error: err instanceof Error ? err.message : String(err),
-        };
+    if (mode === "tables" && Array.isArray(data) && data.length === 0) {
+      const hint = input.execution_mode === "stealth" ? "no_tables_detected \u2014 page genuinely contains no tables" : 'no_tables_detected \u2014 page may require JavaScript; retry with execution_mode: "stealth"';
+      return {
+        ok: false,
+        error: "no_tables_detected",
+        error_reason: "No tables found on page",
+        stage: "extract",
+        hint
+      };
     }
+    return { ok: true, data: { data, source_url: sourceUrl, mode } };
+  } catch (err) {
+    log.error("Extract failed", { url: input.url, error: String(err) });
+    return {
+      ok: false,
+      error: "extract_failed",
+      error_reason: err instanceof Error ? err.message : String(err),
+      stage: "extract"
+    };
+  }
 }
+export {
+  handleExtract
+};
 //# sourceMappingURL=extract.js.map

package/dist/tools/extract.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;AAEpC,KAAK,UAAU,WAAW,CACxB,KAAmB,EACnB,MAAmB;IAEnB,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;QACzD,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAmB,EACnB,MAAmB;IAEnB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,UAAU,CAAC;IAEtC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;IAC1E,CAAC;IAED,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,kDAAkD,EAAE,CAAC;IACvF,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,mEAAmE,EAAE,CAAC;IACxG,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAE7D,IAAI,IAA2B,CAAC;QAEhC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBACb,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,YAAa,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC;gBAC3E,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,MAAO,CAAC,CAAC;gBAC9C,MAAM;YACR,KAAK,UAAU,CAAC;YAChB,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC;gBACZ,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,OAAO;YACL,IAAI,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YAC5D,UAAU,EAAE,KAAK,CAAC,GAAG;YACrB,IAAI;YACJ,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/tools/extract.ts"],"sourcesContent":["import type { ExtractInput, ExtractOutput, StageResult } from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport { extractMetadata, extractSelector, extractTables } from '../extraction/extract.js';\nimport {\n  extractWithSchema,\n  extractWithSchemaDetailedAsync,\n} from '../extraction/schema.js';\nimport { extractJsonLd } from '../extraction/jsonld.js';\nimport { extractStructured } from '../extraction/structured.js';\nimport { getCachedContent, isExpired } from '../cache/store.js';\nimport { fetchWithPlaywright } from '../fetch/playwright-tier.js';\nimport { createLogger } from '../logger.js';\nimport {\n  isNamedSchemaType,\n  extractNamedSchema,\n  NAMED_SCHEMAS,\n} from '../extraction/v1/schemas/index.js';\nimport { isLocalLlmEnabled, extractWithLocalLlm } from '../extraction/v1/local-llm.js';\n\nconst log = createLogger('extract');\n\nasync function resolveHtml(\n  input: ExtractInput,\n  router: SmartRouter,\n): Promise<{ html: string; sourceUrl?: string }> {\n  if (input.execution_mode === 'stealth' && input.url) {\n    const pw = await fetchWithPlaywright(input.url);\n    return { html: pw.html, sourceUrl: input.url };\n  }\n\n  if (input.url) {\n    const cached = getCachedContent(input.url);\n    if (cached && !isExpired(cached)) {\n      log.info('Using cached HTML', { url: input.url });\n      return { html: cached.rawHtml, sourceUrl: cached.url };\n    }\n\n    const raw = await router.fetch(input.url, {\n      renderJs: 'auto',\n      useAuth: false,\n    });\n    return { html: raw.html, sourceUrl: raw.finalUrl };\n  }\n\n  return { html: input.html! };\n}\n\nexport async function handleExtract(\n  input: ExtractInput,\n  router: SmartRouter,\n): Promise<StageResult<ExtractOutput>> {\n  const mode = input.mode ?? 'metadata';\n\n  if (!input.url && !input.html) {\n    return {\n      ok: false,\n      error: 'invalid_input',\n      error_reason: 'Either url or html must be provided',\n      stage: 'extract',\n    };\n  }\n\n  if (input.named_schema && input.schema) {\n    return {\n      ok: false,\n      error: 'invalid_input',\n      error_reason: 'schema and named_schema are mutually exclusive',\n      stage: 'extract',\n    };\n  }\n\n  if (input.named_schema && !isNamedSchemaType(input.named_schema)) {\n    return {\n      ok: false,\n      error: 'invalid_input',\n      error_reason: `Unknown named_schema. Valid: ${NAMED_SCHEMAS.join(', ')}`,\n      stage: 'extract',\n    };\n  }\n\n  if (mode === 'selector' && !input.css_selector) {\n    return {\n      ok: false,\n      error: 'invalid_input',\n      error_reason: 'css_selector is required when mode is \"selector\"',\n      stage: 'extract',\n    };\n  }\n\n  if (mode === 'schema' && !input.named_schema && (!input.schema || !input.schema.properties)) {\n    return {\n      ok: false,\n      error: 'invalid_input',\n      error_reason: 'schema is required when mode is \"schema\" and must have properties',\n      stage: 'extract',\n    };\n  }\n\n  try {\n    const { html, sourceUrl } = await resolveHtml(input, router);\n\n    if (input.named_schema) {\n      const namedData = await extractNamedSchema(input.named_schema, html, sourceUrl ?? input.url ?? '');\n      if (namedData === null) {\n        return {\n          ok: true,\n          data: {\n            data: {},\n            source_url: sourceUrl,\n            mode: 'schema',\n            error: `No ${input.named_schema} data found on page`,\n          },\n        };\n      }\n      return {\n        ok: true,\n        data: {\n          data: namedData as unknown as Record<string, unknown>,\n          source_url: sourceUrl,\n          mode: 'schema',\n        },\n      };\n    }\n\n    if (mode === 'schema' && input.schema && isLocalLlmEnabled()) {\n      const llmData = await extractWithLocalLlm({\n        schema: input.schema as unknown as Record<string, unknown>,\n        html,\n        url: sourceUrl ?? input.url ?? '',\n      });\n      return {\n        ok: true,\n        data: {\n          data: (llmData ?? {}) as Record<string, unknown>,\n          source_url: sourceUrl,\n          mode: 'schema',\n        },\n      };\n    }\n\n    let data: ExtractOutput['data'];\n\n    switch (mode) {\n      case 'selector':\n        data = extractSelector(html, input.css_selector!, input.multiple ?? false);\n        break;\n      case 'tables':\n        data = extractTables(html);\n        break;\n      case 'structured':\n        data = extractStructured(html);\n        break;\n      case 'schema': {\n        const schema = input.schema!;\n        if (Array.isArray(schema.required) && schema.required.length > 0) {\n          const detailed = await extractWithSchemaDetailedAsync(html, schema);\n          data = detailed.values;\n          if (detailed.warnings.length > 0) {\n            return {\n              ok: true,\n              data: {\n                data,\n                source_url: sourceUrl,\n                mode,\n                warnings: detailed.warnings,\n              },\n            };\n          }\n        } else {\n          data = extractWithSchema(html, schema);\n        }\n        break;\n      }\n      case 'metadata':\n      default: {\n        const meta = extractMetadata(html);\n        const jsonld = extractJsonLd(html);\n        if (jsonld.length > 0) {\n          meta.jsonld = jsonld;\n        }\n        data = meta;\n        break;\n      }\n    }\n\n    if (mode === 'tables' && Array.isArray(data) && data.length === 0) {\n      const hint =\n        input.execution_mode === 'stealth'\n          ? 'no_tables_detected — page genuinely contains no tables'\n          : 'no_tables_detected — page may require JavaScript; retry with execution_mode: \"stealth\"';\n      return {\n        ok: false,\n        error: 'no_tables_detected',\n        error_reason: 'No tables found on page',\n        stage: 'extract',\n        hint,\n      };\n    }\n\n    return { ok: true, data: { data, source_url: sourceUrl, mode } };\n  } catch (err) {\n    log.error('Extract failed', { url: input.url, error: String(err) });\n    return {\n      ok: false,\n      error: 'extract_failed',\n      error_reason: err instanceof Error ? err.message : String(err),\n      stage: 'extract',\n    };\n  }\n}\n"],"mappings":"AAEA,SAAS,iBAAiB,iBAAiB,qBAAqB;AAChE;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,qBAAqB;AAC9B,SAAS,yBAAyB;AAClC,SAAS,kBAAkB,iBAAiB;AAC5C,SAAS,2BAA2B;AACpC,SAAS,oBAAoB;AAC7B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,mBAAmB,2BAA2B;AAEvD,MAAM,MAAM,aAAa,SAAS;AAElC,eAAe,YACb,OACA,QAC+C;AAC/C,MAAI,MAAM,mBAAmB,aAAa,MAAM,KAAK;AACnD,UAAM,KAAK,MAAM,oBAAoB,MAAM,GAAG;AAC9C,WAAO,EAAE,MAAM,GAAG,MAAM,WAAW,MAAM,IAAI;AAAA,EAC/C;AAEA,MAAI,MAAM,KAAK;AACb,UAAM,SAAS,iBAAiB,MAAM,GAAG;AACzC,QAAI,UAAU,CAAC,UAAU,MAAM,GAAG;AAChC,UAAI,KAAK,qBAAqB,EAAE,KAAK,MAAM,IAAI,CAAC;AAChD,aAAO,EAAE,MAAM,OAAO,SAAS,WAAW,OAAO,IAAI;AAAA,IACvD;AAEA,UAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK;AAAA,MACxC,UAAU;AAAA,MACV,SAAS;AAAA,IACX,CAAC;AACD,WAAO,EAAE,MAAM,IAAI,MAAM,WAAW,IAAI,SAAS;AAAA,EACnD;AAEA,SAAO,EAAE,MAAM,MAAM,KAAM;AAC7B;AAEA,eAAsB,cACpB,OACA,QACqC;AACrC,QAAM,OAAO,MAAM,QAAQ;AAE3B,MAAI,CAAC,MAAM,OAAO,CAAC,MAAM,MAAM;AAC7B,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc;AAAA,MACd,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,MAAM,gBAAgB,MAAM,QAAQ;AACtC,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc;AAAA,MACd,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,MAAM,gBAAgB,CAAC,kBAAkB,MAAM,YAAY,GAAG;AAChE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,gCAAgC,cAAc,KAAK,IAAI,CAAC;AAAA,MACtE,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,SAAS,cAAc,CAAC,MAAM,cAAc;AAC9C,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc;AAAA,MACd,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,SAAS,YAAY,CAAC,MAAM,iBAAiB,CAAC,MAAM,UAAU,CAAC,MAAM,OAAO,aAAa;AAC3F,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc;AAAA,MACd,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI;AACF,UAAM,EAAE,MAAM,UAAU,IAAI,MAAM,YAAY,OAAO,MAAM;AAE3D,QAAI,MAAM,cAAc;AACtB,YAAM,YAAY,MAAM,mBAAmB,MAAM,cAAc,MAAM,aAAa,MAAM,OAAO,EAAE;AACjG,UAAI,cAAc,MAAM;AACtB,eAAO;AAAA,UACL,IAAI;AAAA,UACJ,MAAM;AAAA,YACJ,MAAM,CAAC;AAAA,YACP,YAAY;AAAA,YACZ,MAAM;AAAA,YACN,OAAO,MAAM,MAAM,YAAY;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,MAAM;AAAA,UACJ,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,YAAY,MAAM,UAAU,kBAAkB,GAAG;AAC5D,YAAM,UAAU,MAAM,oBAAoB;AAAA,QACxC,QAAQ,MAAM;AAAA,QACd;AAAA,QACA,KAAK,aAAa,MAAM,OAAO;AAAA,MACjC,CAAC;AACD,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,MAAM;AAAA,UACJ,MAAO,WAAW,CAAC;AAAA,UACnB,YAAY;AAAA,UACZ,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI;AAEJ,YAAQ,MAAM;AAAA,MACZ,KAAK;AACH,eAAO,gBAAgB,MAAM,MAAM,cAAe,MAAM,YAAY,KAAK;AACzE;AAAA,MACF,KAAK;AACH,eAAO,cAAc,IAAI;AACzB;AAAA,MACF,KAAK;AACH,eAAO,kBAAkB,IAAI;AAC7B;AAAA,MACF,KAAK,UAAU;AACb,cAAM,SAAS,MAAM;AACrB,YAAI,MAAM,QAAQ,OAAO,QAAQ,KAAK,OAAO,SAAS,SAAS,GAAG;AAChE,gBAAM,WAAW,MAAM,+BAA+B,MAAM,MAAM;AAClE,iBAAO,SAAS;AAChB,cAAI,SAAS,SAAS,SAAS,GAAG;AAChC,mBAAO;AAAA,cACL,IAAI;AAAA,cACJ,MAAM;AAAA,gBACJ;AAAA,gBACA,YAAY;AAAA,gBACZ;AAAA,gBACA,UAAU,SAAS;AAAA,cACrB;AAAA,YACF;AAAA,UACF;AAAA,QACF,OAAO;AACL,iBAAO,kBAAkB,MAAM,MAAM;AAAA,QACvC;AACA;AAAA,MACF;AAAA,MACA,KAAK;AAAA,MACL,SAAS;AACP,cAAM,OAAO,gBAAgB,IAAI;AACjC,cAAM,SAAS,cAAc,IAAI;AACjC,YAAI,OAAO,SAAS,GAAG;AACrB,eAAK,SAAS;AAAA,QAChB;AACA,eAAO;AACP;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,YAAY,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAAG;AACjE,YAAM,OACJ,MAAM,mBAAmB,YACrB,gEACA;AACN,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO;AAAA,QACP,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF;AAAA,IACF;AAEA,WAAO,EAAE,IAAI,MAAM,MAAM,EAAE,MAAM,YAAY,WAAW,KAAK,EAAE;AAAA,EACjE,SAAS,KAAK;AACZ,QAAI,MAAM,kBAAkB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAClE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MAC7D,OAAO;AAAA,IACT;AAAA,EACF;AACF;","names":[]}

package/dist/tools/fetch.d.ts

@@ -1,4 +1,4 @@
-import type { FetchInput, FetchOutput } from '../types.js';
+import type { FetchInput, FetchOutput, StageResult } from '../types.js';
 import type { SmartRouter } from '../fetch/router.js';
-export declare function handleFetch(input: FetchInput, router: SmartRouter): Promise<FetchOutput>;
+export declare function handleFetch(input: FetchInput, router: SmartRouter): Promise<StageResult<FetchOutput>>;
 //# sourceMappingURL=fetch.d.ts.map

package/dist/tools/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAoCtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,CAgDtB"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,WAAW,EAAE,MAAM,aAAa,CAAC;AAEvF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA6FtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAsHnC"}