@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/tools/fetch.js

@@ -1,76 +1,182 @@
-import { extractContent } from '../extraction/pipeline.js';
-import { getCachedContent, cacheContent, isExpired } from '../cache/store.js';
-import { extractSection } from '../extraction/markdown.js';
-import { createLogger } from '../logger.js';
-const log = createLogger('fetch');
+import { describeFetchError } from "../fetch/error-describe.js";
+import { getExtractProvider } from "../providers/extract-provider.js";
+import { getCachedContent, cacheContent, isCacheUsable } from "../cache/store.js";
+import { getConfig } from "../config.js";
+import { extractSection } from "../extraction/markdown.js";
+import { detectChange } from "../cache/change-detector.js";
+import { getEmbeddingService } from "../embedding/embed.js";
+import { truncateSmartly, applyOutputBudget } from "../search/truncate.js";
+import { buildEvidenceFromMarkdown } from "../search/evidence.js";
+import { resolveMode } from "../util/mode.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("fetch");
+const DEFAULT_MAX_TOKENS_OUT = 4e3;
+const DEFAULT_FETCH_BODY_TOKENS = 16e3;
+const AUX_FIELD_CAP_WHEN_CHARS_BOUNDED = 50;
+const AUX_FIELD_CAP_WHEN_TIGHT = 20;
+function capAuxFields(out, maxContentChars) {
+  if (maxContentChars === void 0) return;
+  const cap = maxContentChars <= 4e3 ? AUX_FIELD_CAP_WHEN_TIGHT : AUX_FIELD_CAP_WHEN_CHARS_BOUNDED;
+  if (out.links && out.links.length > cap) out.links = out.links.slice(0, cap);
+  if (out.images && out.images.length > cap) out.images = out.images.slice(0, cap);
+}
+async function attachEvidence(output, input, markdown) {
+  if (!markdown) return;
+  const includeFull = input.include_full_markdown ?? true;
+  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
+  const evidence = await buildEvidenceFromMarkdown(
+    output.title || output.url,
+    output.title,
+    output.url,
+    markdown,
+    { maxTokensOut, maxItems: 1 }
+  );
+  if (evidence.length > 0) output.evidence = evidence;
+  if (!includeFull) {
+    output.markdown = "";
+  } else if (output.markdown) {
+    output.markdown = applyOutputBudget(output.markdown, {
+      maxTokensOut: input.max_tokens_out ?? DEFAULT_FETCH_BODY_TOKENS,
+      maxChars: input.max_chars
+    });
+  }
+}
 function formatCachedResponse(cached, input) {
-    let markdown = cached.markdown;
-    let sectionMatched;
-    if (input.section) {
-        const result = extractSection(markdown, input.section, input.section_index);
-        markdown = result.content;
-        sectionMatched = result.matched;
+  let markdown = cached.markdown;
+  let sectionMatched;
+  if (input.section) {
+    const result = extractSection(markdown, input.section, input.section_index);
+    markdown = result.content;
+    sectionMatched = result.matched;
+  }
+  if (input.max_chars && markdown.length > input.max_chars) {
+    markdown = markdown.slice(0, input.max_chars);
+  }
+  if (input.max_content_chars !== void 0) {
+    markdown = truncateSmartly(markdown, input.max_content_chars);
+  }
+  const out = {
+    url: cached.url,
+    title: cached.title,
+    markdown,
+    metadata: {
+      ...JSON.parse(cached.metadata || "{}"),
+      ...sectionMatched !== void 0 ? { section_matched: sectionMatched } : {}
+    },
+    links: JSON.parse(cached.links || "[]"),
+    images: JSON.parse(cached.images || "[]"),
+    cached: true,
+    cached_at: cached.fetchedAt
+  };
+  capAuxFields(out, input.max_content_chars);
+  return out;
+}
+async function handleFetch(input, router) {
+  const mode = resolveMode(input.mode);
+  try {
+    if (!input.force_refresh) {
+      const cached = getCachedContent(input.url);
+      if (cached && (!input.actions || input.actions.length === 0)) {
+        const staleMaxSeconds = mode === "cache" ? getConfig().fastStaleMaxHours * 3600 : 0;
+        const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });
+        if (usable) {
+          log.info("Serving from cache", { url: input.url, stale });
+          const out2 = formatCachedResponse(cached, input);
+          if (stale) out2.stale = true;
+          const fullMarkdown = out2.markdown;
+          await attachEvidence(out2, input, fullMarkdown);
+          return { ok: true, data: out2 };
+        }
+      }
     }
-    if (input.max_chars && markdown.length > input.max_chars) {
-        markdown = markdown.slice(0, input.max_chars);
+    if (mode === "cache") {
+      return {
+        ok: false,
+        error: "cache_miss",
+        error_reason: `URL not in cache: ${input.url}`,
+        stage: "fetch",
+        hint: "Use mode:default to fetch live, or run search/crawl first to populate cache"
+      };
     }
-    return {
-        url: cached.url,
-        title: cached.title,
-        markdown,
-        metadata: {
-            ...JSON.parse(cached.metadata || '{}'),
-            ...(sectionMatched !== undefined ? { section_matched: sectionMatched } : {}),
-        },
-        links: JSON.parse(cached.links || '[]'),
-        images: JSON.parse(cached.images || '[]'),
-        cached: true,
-    };
-}
-export async function handleFetch(input, router) {
+    const raw = await router.fetch(input.url, {
+      renderJs: input.render_js ?? "auto",
+      useAuth: input.use_auth ?? false,
+      headers: input.headers,
+      screenshot: input.screenshot,
+      actions: input.actions,
+      mode
+    });
+    if ("error" in raw && typeof raw.error === "string") {
+      const stageErr = raw;
+      return {
+        ok: false,
+        error: stageErr.error,
+        error_reason: stageErr.error_reason ?? stageErr.error,
+        stage: stageErr.stage ?? "fetch",
+        ...stageErr.hint ? { hint: stageErr.hint } : {}
+      };
+    }
+    const extractor = await getExtractProvider();
+    const extraction = await extractor.extract(raw.html, raw.finalUrl, {
+      maxChars: input.max_chars,
+      section: input.section,
+      sectionIndex: input.section_index,
+      contentType: raw.contentType,
+      pdfBuffer: raw.rawBuffer
+    });
+    let changeResult;
     try {
-        const cached = getCachedContent(input.url);
-        if (cached && !isExpired(cached)) {
-            log.info('Serving from cache', { url: input.url });
-            return formatCachedResponse(cached, input);
-        }
-        const raw = await router.fetch(input.url, {
-            renderJs: input.render_js ?? 'auto',
-            useAuth: input.use_auth ?? false,
-            headers: input.headers,
-            screenshot: input.screenshot,
-        });
-        const extraction = await extractContent(raw.html, raw.finalUrl, {
-            maxChars: input.max_chars,
-            section: input.section,
-            sectionIndex: input.section_index,
-            contentType: raw.contentType,
-            pdfBuffer: raw.rawBuffer,
-        });
-        cacheContent(raw, extraction);
-        return {
-            url: raw.finalUrl,
-            title: extraction.title,
-            markdown: extraction.markdown,
-            metadata: extraction.metadata,
-            links: extraction.links,
-            images: extraction.images,
-            screenshot: raw.screenshot,
-            cached: false,
-        };
+      changeResult = detectChange(raw.finalUrl, extraction.markdown);
+    } catch (err) {
+      log.warn("change detection failed", { url: raw.finalUrl, error: String(err) });
+    }
+    try {
+      cacheContent(raw, extraction);
+    } catch (err) {
+      log.warn("failed to cache fetched content", { url: raw.finalUrl, error: String(err) });
     }
-    catch (err) {
-        log.error('Fetch failed', { url: input.url, error: String(err) });
-        return {
-            url: input.url,
-            title: '',
-            markdown: '',
-            metadata: {},
-            links: [],
-            images: [],
-            cached: false,
-            error: err instanceof Error ? err.message : String(err),
-        };
+    try {
+      const embeddingService = getEmbeddingService();
+      if (embeddingService.isAvailable()) {
+        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);
+      }
+    } catch (err) {
+      log.debug("embedding hook skipped", { error: String(err) });
     }
+    const finalMarkdown = input.max_content_chars !== void 0 ? truncateSmartly(extraction.markdown, input.max_content_chars) : extraction.markdown;
+    const out = {
+      url: raw.finalUrl,
+      title: extraction.title,
+      markdown: finalMarkdown,
+      metadata: extraction.metadata,
+      links: extraction.links,
+      images: extraction.images,
+      screenshot: raw.screenshot,
+      cached: false,
+      action_results: raw.actionResults,
+      ...raw.jsRequired ? { js_required: true } : {},
+      ...changeResult?.changed ? {
+        changed: true,
+        previous_hash: changeResult.previousHash,
+        diff_summary: changeResult.diffSummary
+      } : {}
+    };
+    capAuxFields(out, input.max_content_chars);
+    await attachEvidence(out, input, finalMarkdown);
+    return { ok: true, data: out };
+  } catch (err) {
+    log.error("Fetch failed", { url: input.url, error: String(err) });
+    const described = describeFetchError(err);
+    return {
+      ok: false,
+      error: "fetch_failed",
+      error_reason: described.reason,
+      stage: "fetch",
+      ...described.hint ? { hint: described.hint } : {}
+    };
+  }
 }
+export {
+  handleFetch
+};
 //# sourceMappingURL=fetch.js.map

package/dist/tools/fetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,SAAS,oBAAoB,CAAC,MAAqB,EAAE,KAAiB;IACpE,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,IAAI,cAAmC,CAAC;IAExC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5E,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC1B,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YACtC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7E;QACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACzC,MAAM,EAAE,IAAI;KACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACnD,OAAO,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,MAAM;YACnC,OAAO,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,KAAK,CAAC,UAAU;SAC7B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;YAC9D,QAAQ,EAAE,KAAK,CAAC,SAAS;YACzB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,YAAY,EAAE,KAAK,CAAC,aAAa;YACjC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC,CAAC;QAEH,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAE9B,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/tools/fetch.ts"],"sourcesContent":["import type { FetchInput, FetchOutput, CachedContent, StageResult } from '../types.js';\nimport { describeFetchError } from '../fetch/error-describe.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { getCachedContent, cacheContent, isCacheUsable } from '../cache/store.js';\nimport { getConfig } from '../config.js';\nimport { extractSection } from '../extraction/markdown.js';\nimport { detectChange } from '../cache/change-detector.js';\nimport { getEmbeddingService } from '../embedding/embed.js';\nimport { truncateSmartly, applyOutputBudget } from '../search/truncate.js';\nimport { buildEvidenceFromMarkdown } from '../search/evidence.js';\nimport { resolveMode } from '../util/mode.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('fetch');\n\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n// Fetch is single-URL — users explicitly want the body. Keep a generous cap\n// that fits typical MCP tool-result limits (~25k tokens) but prevents huge\n// pages (full doc sites) from blowing the cap. Override via max_tokens_out.\nconst DEFAULT_FETCH_BODY_TOKENS = 16000;\n// When the caller asks for a tight markdown budget, also clip the\n// auxiliary arrays — large doc pages emit thousands of links/images that\n// otherwise blow the user-requested response size.\nconst AUX_FIELD_CAP_WHEN_CHARS_BOUNDED = 50;\nconst AUX_FIELD_CAP_WHEN_TIGHT = 20;\n\nfunction capAuxFields(out: FetchOutput, maxContentChars?: number): void {\n  if (maxContentChars === undefined) return;\n  const cap = maxContentChars <= 4000 ? AUX_FIELD_CAP_WHEN_TIGHT : AUX_FIELD_CAP_WHEN_CHARS_BOUNDED;\n  if (out.links && out.links.length > cap) out.links = out.links.slice(0, cap);\n  if (out.images && out.images.length > cap) out.images = out.images.slice(0, cap);\n}\n\nasync function attachEvidence(\n  output: FetchOutput,\n  input: FetchInput,\n  markdown: string,\n): Promise<void> {\n  if (!markdown) return;\n  const includeFull = input.include_full_markdown ?? true;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n  const evidence = await buildEvidenceFromMarkdown(\n    output.title || output.url,\n    output.title,\n    output.url,\n    markdown,\n    { maxTokensOut, maxItems: 1 },\n  );\n  if (evidence.length > 0) output.evidence = evidence;\n  if (!includeFull) {\n    output.markdown = '';\n  } else if (output.markdown) {\n    output.markdown = applyOutputBudget(output.markdown, {\n      maxTokensOut: input.max_tokens_out ?? DEFAULT_FETCH_BODY_TOKENS,\n      maxChars: input.max_chars,\n    });\n  }\n}\n\nfunction formatCachedResponse(cached: CachedContent, input: FetchInput): FetchOutput {\n  let markdown = cached.markdown;\n  let sectionMatched: boolean | undefined;\n\n  if (input.section) {\n    const result = extractSection(markdown, input.section, input.section_index);\n    markdown = result.content;\n    sectionMatched = result.matched;\n  }\n\n  if (input.max_chars && markdown.length > input.max_chars) {\n    markdown = markdown.slice(0, input.max_chars);\n  }\n\n  if (input.max_content_chars !== undefined) {\n    markdown = truncateSmartly(markdown, input.max_content_chars);\n  }\n\n  const out: FetchOutput = {\n    url: cached.url,\n    title: cached.title,\n    markdown,\n    metadata: {\n      ...JSON.parse(cached.metadata || '{}'),\n      ...(sectionMatched !== undefined ? { section_matched: sectionMatched } : {}),\n    },\n    links: JSON.parse(cached.links || '[]'),\n    images: JSON.parse(cached.images || '[]'),\n    cached: true,\n    cached_at: cached.fetchedAt,\n  };\n  capAuxFields(out, input.max_content_chars);\n  return out;\n}\n\nexport async function handleFetch(\n  input: FetchInput,\n  router: SmartRouter,\n): Promise<StageResult<FetchOutput>> {\n  const mode = resolveMode(input.mode);\n  try {\n    if (!input.force_refresh) {\n      const cached = getCachedContent(input.url);\n      if (cached && (!input.actions || input.actions.length === 0)) {\n        const staleMaxSeconds = mode === 'cache' ? getConfig().fastStaleMaxHours * 3600 : 0;\n        const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });\n        if (usable) {\n          log.info('Serving from cache', { url: input.url, stale });\n          const out = formatCachedResponse(cached, input);\n          if (stale) out.stale = true;\n          const fullMarkdown = out.markdown;\n          await attachEvidence(out, input, fullMarkdown);\n          return { ok: true, data: out };\n        }\n      }\n    }\n\n    if (mode === 'cache') {\n      return {\n        ok: false,\n        error: 'cache_miss',\n        error_reason: `URL not in cache: ${input.url}`,\n        stage: 'fetch',\n        hint: 'Use mode:default to fetch live, or run search/crawl first to populate cache',\n      };\n    }\n\n    const raw = await router.fetch(input.url, {\n      renderJs: input.render_js ?? 'auto',\n      useAuth: input.use_auth ?? false,\n      headers: input.headers,\n      screenshot: input.screenshot,\n      actions: input.actions,\n      mode,\n    });\n\n    // T11: stealth mode can return a StageError (e.g., playwright_not_installed,\n    // playwright_fetch_failed). Surface it directly.\n    if ('error' in raw && typeof (raw as { error?: unknown }).error === 'string') {\n      const stageErr = raw as unknown as { error: string; error_reason?: string; stage?: string; hint?: string };\n      return {\n        ok: false,\n        error: stageErr.error,\n        error_reason: stageErr.error_reason ?? stageErr.error,\n        stage: stageErr.stage ?? 'fetch',\n        ...(stageErr.hint ? { hint: stageErr.hint } : {}),\n      };\n    }\n\n    const extractor = await getExtractProvider();\n    const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n      maxChars: input.max_chars,\n      section: input.section,\n      sectionIndex: input.section_index,\n      contentType: raw.contentType,\n      pdfBuffer: raw.rawBuffer,\n    });\n\n    let changeResult: { changed: boolean; previousHash?: string; diffSummary?: string } | undefined;\n    try {\n      changeResult = detectChange(raw.finalUrl, extraction.markdown);\n    } catch (err) {\n      log.warn('change detection failed', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      cacheContent(raw, extraction);\n    } catch (err) {\n      log.warn('failed to cache fetched content', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      const embeddingService = getEmbeddingService();\n      if (embeddingService.isAvailable()) {\n        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);\n      }\n    } catch (err) {\n      log.debug('embedding hook skipped', { error: String(err) });\n    }\n\n    const finalMarkdown = input.max_content_chars !== undefined\n      ? truncateSmartly(extraction.markdown, input.max_content_chars)\n      : extraction.markdown;\n\n    const out: FetchOutput = {\n      url: raw.finalUrl,\n      title: extraction.title,\n      markdown: finalMarkdown,\n      metadata: extraction.metadata,\n      links: extraction.links,\n      images: extraction.images,\n      screenshot: raw.screenshot,\n      cached: false,\n      action_results: raw.actionResults,\n      ...(raw.jsRequired ? { js_required: true } : {}),\n      ...(changeResult?.changed ? {\n        changed: true,\n        previous_hash: changeResult.previousHash,\n        diff_summary: changeResult.diffSummary,\n      } : {}),\n    };\n\n    capAuxFields(out, input.max_content_chars);\n    await attachEvidence(out, input, finalMarkdown);\n    return { ok: true, data: out };\n  } catch (err) {\n    log.error('Fetch failed', { url: input.url, error: String(err) });\n    const described = describeFetchError(err);\n    return {\n      ok: false,\n      error: 'fetch_failed',\n      error_reason: described.reason,\n      stage: 'fetch',\n      ...(described.hint ? { hint: described.hint } : {}),\n    };\n  }\n}\n"],"mappings":"AACA,SAAS,0BAA0B;AAEnC,SAAS,0BAA0B;AACnC,SAAS,kBAAkB,cAAc,qBAAqB;AAC9D,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAC/B,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,iBAAiB,yBAAyB;AACnD,SAAS,iCAAiC;AAC1C,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,yBAAyB;AAI/B,MAAM,4BAA4B;AAIlC,MAAM,mCAAmC;AACzC,MAAM,2BAA2B;AAEjC,SAAS,aAAa,KAAkB,iBAAgC;AACtE,MAAI,oBAAoB,OAAW;AACnC,QAAM,MAAM,mBAAmB,MAAO,2BAA2B;AACjE,MAAI,IAAI,SAAS,IAAI,MAAM,SAAS,IAAK,KAAI,QAAQ,IAAI,MAAM,MAAM,GAAG,GAAG;AAC3E,MAAI,IAAI,UAAU,IAAI,OAAO,SAAS,IAAK,KAAI,SAAS,IAAI,OAAO,MAAM,GAAG,GAAG;AACjF;AAEA,eAAe,eACb,QACA,OACA,UACe;AACf,MAAI,CAAC,SAAU;AACf,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAC7C,QAAM,WAAW,MAAM;AAAA,IACrB,OAAO,SAAS,OAAO;AAAA,IACvB,OAAO;AAAA,IACP,OAAO;AAAA,IACP;AAAA,IACA,EAAE,cAAc,UAAU,EAAE;AAAA,EAC9B;AACA,MAAI,SAAS,SAAS,EAAG,QAAO,WAAW;AAC3C,MAAI,CAAC,aAAa;AAChB,WAAO,WAAW;AAAA,EACpB,WAAW,OAAO,UAAU;AAC1B,WAAO,WAAW,kBAAkB,OAAO,UAAU;AAAA,MACnD,cAAc,MAAM,kBAAkB;AAAA,MACtC,UAAU,MAAM;AAAA,IAClB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,qBAAqB,QAAuB,OAAgC;AACnF,MAAI,WAAW,OAAO;AACtB,MAAI;AAEJ,MAAI,MAAM,SAAS;AACjB,UAAM,SAAS,eAAe,UAAU,MAAM,SAAS,MAAM,aAAa;AAC1E,eAAW,OAAO;AAClB,qBAAiB,OAAO;AAAA,EAC1B;AAEA,MAAI,MAAM,aAAa,SAAS,SAAS,MAAM,WAAW;AACxD,eAAW,SAAS,MAAM,GAAG,MAAM,SAAS;AAAA,EAC9C;AAEA,MAAI,MAAM,sBAAsB,QAAW;AACzC,eAAW,gBAAgB,UAAU,MAAM,iBAAiB;AAAA,EAC9D;AAEA,QAAM,MAAmB;AAAA,IACvB,KAAK,OAAO;AAAA,IACZ,OAAO,OAAO;AAAA,IACd;AAAA,IACA,UAAU;AAAA,MACR,GAAG,KAAK,MAAM,OAAO,YAAY,IAAI;AAAA,MACrC,GAAI,mBAAmB,SAAY,EAAE,iBAAiB,eAAe,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI;AAAA,IACtC,QAAQ,KAAK,MAAM,OAAO,UAAU,IAAI;AAAA,IACxC,QAAQ;AAAA,IACR,WAAW,OAAO;AAAA,EACpB;AACA,eAAa,KAAK,MAAM,iBAAiB;AACzC,SAAO;AACT;AAEA,eAAsB,YACpB,OACA,QACmC;AACnC,QAAM,OAAO,YAAY,MAAM,IAAI;AACnC,MAAI;AACF,QAAI,CAAC,MAAM,eAAe;AACxB,YAAM,SAAS,iBAAiB,MAAM,GAAG;AACzC,UAAI,WAAW,CAAC,MAAM,WAAW,MAAM,QAAQ,WAAW,IAAI;AAC5D,cAAM,kBAAkB,SAAS,UAAU,UAAU,EAAE,oBAAoB,OAAO;AAClF,cAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,QAAQ,EAAE,gBAAgB,CAAC;AACnE,YAAI,QAAQ;AACV,cAAI,KAAK,sBAAsB,EAAE,KAAK,MAAM,KAAK,MAAM,CAAC;AACxD,gBAAMA,OAAM,qBAAqB,QAAQ,KAAK;AAC9C,cAAI,MAAO,CAAAA,KAAI,QAAQ;AACvB,gBAAM,eAAeA,KAAI;AACzB,gBAAM,eAAeA,MAAK,OAAO,YAAY;AAC7C,iBAAO,EAAE,IAAI,MAAM,MAAMA,KAAI;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,SAAS;AACpB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO;AAAA,QACP,cAAc,qBAAqB,MAAM,GAAG;AAAA,QAC5C,OAAO;AAAA,QACP,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK;AAAA,MACxC,UAAU,MAAM,aAAa;AAAA,MAC7B,SAAS,MAAM,YAAY;AAAA,MAC3B,SAAS,MAAM;AAAA,MACf,YAAY,MAAM;AAAA,MAClB,SAAS,MAAM;AAAA,MACf;AAAA,IACF,CAAC;AAID,QAAI,WAAW,OAAO,OAAQ,IAA4B,UAAU,UAAU;AAC5E,YAAM,WAAW;AACjB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,SAAS;AAAA,QAChB,cAAc,SAAS,gBAAgB,SAAS;AAAA,QAChD,OAAO,SAAS,SAAS;AAAA,QACzB,GAAI,SAAS,OAAO,EAAE,MAAM,SAAS,KAAK,IAAI,CAAC;AAAA,MACjD;AAAA,IACF;AAEA,UAAM,YAAY,MAAM,mBAAmB;AAC3C,UAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,MACjE,UAAU,MAAM;AAAA,MAChB,SAAS,MAAM;AAAA,MACf,cAAc,MAAM;AAAA,MACpB,aAAa,IAAI;AAAA,MACjB,WAAW,IAAI;AAAA,IACjB,CAAC;AAED,QAAI;AACJ,QAAI;AACF,qBAAe,aAAa,IAAI,UAAU,WAAW,QAAQ;AAAA,IAC/D,SAAS,KAAK;AACZ,UAAI,KAAK,2BAA2B,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC/E;AAEA,QAAI;AACF,mBAAa,KAAK,UAAU;AAAA,IAC9B,SAAS,KAAK;AACZ,UAAI,KAAK,mCAAmC,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACvF;AAEA,QAAI;AACF,YAAM,mBAAmB,oBAAoB;AAC7C,UAAI,iBAAiB,YAAY,GAAG;AAClC,yBAAiB,WAAW,IAAI,UAAU,WAAW,QAAQ;AAAA,MAC/D;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC5D;AAEA,UAAM,gBAAgB,MAAM,sBAAsB,SAC9C,gBAAgB,WAAW,UAAU,MAAM,iBAAiB,IAC5D,WAAW;AAEf,UAAM,MAAmB;AAAA,MACvB,KAAK,IAAI;AAAA,MACT,OAAO,WAAW;AAAA,MAClB,UAAU;AAAA,MACV,UAAU,WAAW;AAAA,MACrB,OAAO,WAAW;AAAA,MAClB,QAAQ,WAAW;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,QAAQ;AAAA,MACR,gBAAgB,IAAI;AAAA,MACpB,GAAI,IAAI,aAAa,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MAC9C,GAAI,cAAc,UAAU;AAAA,QAC1B,SAAS;AAAA,QACT,eAAe,aAAa;AAAA,QAC5B,cAAc,aAAa;AAAA,MAC7B,IAAI,CAAC;AAAA,IACP;AAEA,iBAAa,KAAK,MAAM,iBAAiB;AACzC,UAAM,eAAe,KAAK,OAAO,aAAa;AAC9C,WAAO,EAAE,IAAI,MAAM,MAAM,IAAI;AAAA,EAC/B,SAAS,KAAK;AACZ,QAAI,MAAM,gBAAgB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,UAAM,YAAY,mBAAmB,GAAG;AACxC,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,UAAU;AAAA,MACxB,OAAO;AAAA,MACP,GAAI,UAAU,OAAO,EAAE,MAAM,UAAU,KAAK,IAAI,CAAC;AAAA,IACnD;AAAA,EACF;AACF;","names":["out"]}

package/dist/tools/find-similar.d.ts

@@ -0,0 +1,5 @@
+import type { FindSimilarInput, FindSimilarOutput, SearchEngine, StageResult } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+import type { BackendStatus } from '../server/backend-status.js';
+export declare function handleFindSimilar(input: FindSimilarInput, engines: SearchEngine[], router: SmartRouter, backendStatus?: BackendStatus): Promise<StageResult<FindSimilarOutput>>;
+//# sourceMappingURL=find-similar.d.ts.map

package/dist/tools/find-similar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-similar.d.ts","sourceRoot":"","sources":["../../src/tools/find-similar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EAEZ,WAAW,EACZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAgBjE,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,CAwFzC"}

package/dist/tools/find-similar.js

@@ -0,0 +1,127 @@
+import { findSimilar } from "../search/find-similar.js";
+import {
+  buildEvidenceFromMarkdown,
+  applyTokenBudget,
+  applyAggregateMarkdownBudget
+} from "../search/evidence.js";
+import * as cacheStore from "../cache/store.js";
+import * as searchTool from "./search.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("search");
+const MAX_RESULTS_CAP = 50;
+const DEFAULT_MAX_TOKENS_OUT = 4e3;
+async function handleFindSimilar(input, engines, router, backendStatus) {
+  try {
+    const url = input.url?.trim();
+    const concept = input.concept?.trim();
+    if (!url && !concept) {
+      return {
+        ok: false,
+        error: "invalid_input",
+        error_reason: "Either url or concept must be provided",
+        stage: "find_similar"
+      };
+    }
+    const sanitizedInput = {
+      ...input,
+      max_results: input.max_results ? Math.min(input.max_results, MAX_RESULTS_CAP) : void 0
+    };
+    log.info("find_similar request", {
+      hasUrl: !!url,
+      hasConcept: !!concept,
+      maxResults: sanitizedInput.max_results,
+      includeCache: sanitizedInput.include_cache,
+      includeWeb: sanitizedInput.include_web
+    });
+    let cacheSeeded = false;
+    if (url) {
+      try {
+        const u = new URL(url);
+        const host = u.hostname;
+        const cachedCount = cacheStore.countCachedUrlsForDomain(host);
+        if (cachedCount < 5) {
+          const rawSeg = u.pathname.split("/").filter(Boolean).pop() ?? "";
+          let lastSeg = rawSeg;
+          try {
+            lastSeg = decodeURIComponent(rawSeg);
+          } catch {
+          }
+          const seedQuery = (lastSeg.replace(/[-_]/g, " ") + " " + host.replace(/^www\./, "").split(".")[0]).trim();
+          if (seedQuery.length > 0) {
+            try {
+              await searchTool.handleSearch(
+                { query: seedQuery },
+                engines,
+                router,
+                backendStatus
+              );
+              cacheSeeded = true;
+            } catch (seedErr) {
+              log.warn("find_similar cold-start seed failed", {
+                error: seedErr instanceof Error ? seedErr.message : String(seedErr)
+              });
+            }
+          }
+        }
+      } catch (parseErr) {
+        log.warn("find_similar cold-start url parse failed", {
+          error: parseErr instanceof Error ? parseErr.message : String(parseErr)
+        });
+      }
+    }
+    const out = await findSimilar(sanitizedInput, engines, router, backendStatus);
+    await attachEvidence(out, input);
+    if (cacheSeeded) out.cache_seeded = true;
+    if (out.error) {
+      return {
+        ok: false,
+        error: out.error,
+        error_reason: out.error,
+        stage: "find_similar"
+      };
+    }
+    return { ok: true, data: out };
+  } catch (err) {
+    log.error("handleFindSimilar failed", { error: String(err) });
+    return {
+      ok: false,
+      error: "find_similar_failed",
+      error_reason: `find_similar handler error: ${err instanceof Error ? err.message : String(err)}`,
+      stage: "find_similar"
+    };
+  }
+}
+async function attachEvidence(out, input) {
+  if (out.results.length === 0) return;
+  const includeFull = input.include_full_markdown ?? false;
+  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
+  const query = input.concept?.trim() || input.url?.trim() || out.results[0].title;
+  const collected = [];
+  for (const r of out.results) {
+    if (!r.markdown) continue;
+    const evs = await buildEvidenceFromMarkdown(query, r.title, r.url, r.markdown, {
+      maxItems: 1
+    });
+    collected.push(...evs);
+  }
+  const budgeted = applyTokenBudget(collected, maxTokensOut);
+  if (budgeted.length > 0) out.evidence = budgeted;
+  if (!includeFull) {
+    for (const r of out.results) {
+      r.markdown = "";
+    }
+  } else {
+    applyAggregateMarkdownBudget(
+      out.results,
+      (r) => r.markdown ?? "",
+      (r, body) => {
+        r.markdown = body;
+      },
+      { maxTokensOut }
+    );
+  }
+}
+export {
+  handleFindSimilar
+};
+//# sourceMappingURL=find-similar.js.map

package/dist/tools/find-similar.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tools/find-similar.ts"],"sourcesContent":["import type {\n  FindSimilarInput,\n  FindSimilarOutput,\n  SearchEngine,\n  EvidenceItem,\n  StageResult,\n} from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport type { BackendStatus } from '../server/backend-status.js';\nimport { findSimilar } from '../search/find-similar.js';\nimport {\n  buildEvidenceFromMarkdown,\n  applyTokenBudget,\n  applyAggregateMarkdownBudget,\n} from '../search/evidence.js';\nimport * as cacheStore from '../cache/store.js';\nimport * as searchTool from './search.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('search');\n\nconst MAX_RESULTS_CAP = 50;\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n\nexport async function handleFindSimilar(\n  input: FindSimilarInput,\n  engines: SearchEngine[],\n  router: SmartRouter,\n  backendStatus?: BackendStatus,\n): Promise<StageResult<FindSimilarOutput>> {\n  try {\n    const url = input.url?.trim();\n    const concept = input.concept?.trim();\n\n    if (!url && !concept) {\n      return {\n        ok: false,\n        error: 'invalid_input',\n        error_reason: 'Either url or concept must be provided',\n        stage: 'find_similar',\n      };\n    }\n\n    const sanitizedInput: FindSimilarInput = {\n      ...input,\n      max_results: input.max_results\n        ? Math.min(input.max_results, MAX_RESULTS_CAP)\n        : undefined,\n    };\n\n    log.info('find_similar request', {\n      hasUrl: !!url,\n      hasConcept: !!concept,\n      maxResults: sanitizedInput.max_results,\n      includeCache: sanitizedInput.include_cache,\n      includeWeb: sanitizedInput.include_web,\n    });\n\n    let cacheSeeded = false;\n    if (url) {\n      try {\n        const u = new URL(url);\n        const host = u.hostname;\n        const cachedCount = cacheStore.countCachedUrlsForDomain(host);\n        if (cachedCount < 5) {\n          const rawSeg = u.pathname.split('/').filter(Boolean).pop() ?? '';\n          let lastSeg = rawSeg;\n          try { lastSeg = decodeURIComponent(rawSeg); } catch { /* leave raw */ }\n          const seedQuery = (\n            lastSeg.replace(/[-_]/g, ' ') +\n            ' ' +\n            host.replace(/^www\\./, '').split('.')[0]\n          ).trim();\n          if (seedQuery.length > 0) {\n            try {\n              await searchTool.handleSearch(\n                { query: seedQuery },\n                engines,\n                router,\n                backendStatus,\n              );\n              cacheSeeded = true;\n            } catch (seedErr) {\n              log.warn('find_similar cold-start seed failed', {\n                error: seedErr instanceof Error ? seedErr.message : String(seedErr),\n              });\n            }\n          }\n        }\n      } catch (parseErr) {\n        log.warn('find_similar cold-start url parse failed', {\n          error: parseErr instanceof Error ? parseErr.message : String(parseErr),\n        });\n      }\n    }\n\n    const out = await findSimilar(sanitizedInput, engines, router, backendStatus);\n    await attachEvidence(out, input);\n    if (cacheSeeded) out.cache_seeded = true;\n    if (out.error) {\n      return {\n        ok: false,\n        error: out.error,\n        error_reason: out.error,\n        stage: 'find_similar',\n      };\n    }\n    return { ok: true, data: out };\n  } catch (err) {\n    log.error('handleFindSimilar failed', { error: String(err) });\n    return {\n      ok: false,\n      error: 'find_similar_failed',\n      error_reason: `find_similar handler error: ${err instanceof Error ? err.message : String(err)}`,\n      stage: 'find_similar',\n    };\n  }\n}\n\nasync function attachEvidence(\n  out: FindSimilarOutput,\n  input: FindSimilarInput,\n): Promise<void> {\n  if (out.results.length === 0) return;\n  const includeFull = input.include_full_markdown ?? false;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n  const query = input.concept?.trim() || input.url?.trim() || out.results[0].title;\n\n  const collected: EvidenceItem[] = [];\n  for (const r of out.results) {\n    if (!r.markdown) continue;\n    const evs = await buildEvidenceFromMarkdown(query, r.title, r.url, r.markdown, {\n      maxItems: 1,\n    });\n    collected.push(...evs);\n  }\n\n  const budgeted = applyTokenBudget(collected, maxTokensOut);\n  if (budgeted.length > 0) out.evidence = budgeted;\n\n  if (!includeFull) {\n    for (const r of out.results) {\n      r.markdown = '';\n    }\n  } else {\n    applyAggregateMarkdownBudget(\n      out.results,\n      (r) => r.markdown ?? '',\n      (r, body) => { r.markdown = body; },\n      { maxTokensOut },\n    );\n  }\n}\n"],"mappings":"AASA,SAAS,mBAAmB;AAC5B;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,YAAY,gBAAgB;AAC5B,YAAY,gBAAgB;AAC5B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,kBAAkB;AACxB,MAAM,yBAAyB;AAE/B,eAAsB,kBACpB,OACA,SACA,QACA,eACyC;AACzC,MAAI;AACF,UAAM,MAAM,MAAM,KAAK,KAAK;AAC5B,UAAM,UAAU,MAAM,SAAS,KAAK;AAEpC,QAAI,CAAC,OAAO,CAAC,SAAS;AACpB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO;AAAA,QACP,cAAc;AAAA,QACd,OAAO;AAAA,MACT;AAAA,IACF;AAEA,UAAM,iBAAmC;AAAA,MACvC,GAAG;AAAA,MACH,aAAa,MAAM,cACf,KAAK,IAAI,MAAM,aAAa,eAAe,IAC3C;AAAA,IACN;AAEA,QAAI,KAAK,wBAAwB;AAAA,MAC/B,QAAQ,CAAC,CAAC;AAAA,MACV,YAAY,CAAC,CAAC;AAAA,MACd,YAAY,eAAe;AAAA,MAC3B,cAAc,eAAe;AAAA,MAC7B,YAAY,eAAe;AAAA,IAC7B,CAAC;AAED,QAAI,cAAc;AAClB,QAAI,KAAK;AACP,UAAI;AACF,cAAM,IAAI,IAAI,IAAI,GAAG;AACrB,cAAM,OAAO,EAAE;AACf,cAAM,cAAc,WAAW,yBAAyB,IAAI;AAC5D,YAAI,cAAc,GAAG;AACnB,gBAAM,SAAS,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO,EAAE,IAAI,KAAK;AAC9D,cAAI,UAAU;AACd,cAAI;AAAE,sBAAU,mBAAmB,MAAM;AAAA,UAAG,QAAQ;AAAA,UAAkB;AACtE,gBAAM,aACJ,QAAQ,QAAQ,SAAS,GAAG,IAC5B,MACA,KAAK,QAAQ,UAAU,EAAE,EAAE,MAAM,GAAG,EAAE,CAAC,GACvC,KAAK;AACP,cAAI,UAAU,SAAS,GAAG;AACxB,gBAAI;AACF,oBAAM,WAAW;AAAA,gBACf,EAAE,OAAO,UAAU;AAAA,gBACnB;AAAA,gBACA;AAAA,gBACA;AAAA,cACF;AACA,4BAAc;AAAA,YAChB,SAAS,SAAS;AAChB,kBAAI,KAAK,uCAAuC;AAAA,gBAC9C,OAAO,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,cACpE,CAAC;AAAA,YACH;AAAA,UACF;AAAA,QACF;AAAA,MACF,SAAS,UAAU;AACjB,YAAI,KAAK,4CAA4C;AAAA,UACnD,OAAO,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;AAAA,QACvE,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,YAAY,gBAAgB,SAAS,QAAQ,aAAa;AAC5E,UAAM,eAAe,KAAK,KAAK;AAC/B,QAAI,YAAa,KAAI,eAAe;AACpC,QAAI,IAAI,OAAO;AACb,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,IAAI;AAAA,QACX,cAAc,IAAI;AAAA,QAClB,OAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,MAAM,IAAI;AAAA,EAC/B,SAAS,KAAK;AACZ,QAAI,MAAM,4BAA4B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC5D,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC7F,OAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,eAAe,eACb,KACA,OACe;AACf,MAAI,IAAI,QAAQ,WAAW,EAAG;AAC9B,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAC7C,QAAM,QAAQ,MAAM,SAAS,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,IAAI,QAAQ,CAAC,EAAE;AAE3E,QAAM,YAA4B,CAAC;AACnC,aAAW,KAAK,IAAI,SAAS;AAC3B,QAAI,CAAC,EAAE,SAAU;AACjB,UAAM,MAAM,MAAM,0BAA0B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU;AAAA,MAC7E,UAAU;AAAA,IACZ,CAAC;AACD,cAAU,KAAK,GAAG,GAAG;AAAA,EACvB;AAEA,QAAM,WAAW,iBAAiB,WAAW,YAAY;AACzD,MAAI,SAAS,SAAS,EAAG,KAAI,WAAW;AAExC,MAAI,CAAC,aAAa;AAChB,eAAW,KAAK,IAAI,SAAS;AAC3B,QAAE,WAAW;AAAA,IACf;AAAA,EACF,OAAO;AACL;AAAA,MACE,IAAI;AAAA,MACJ,CAAC,MAAM,EAAE,YAAY;AAAA,MACrB,CAAC,GAAG,SAAS;AAAE,UAAE,WAAW;AAAA,MAAM;AAAA,MAClC,EAAE,aAAa;AAAA,IACjB;AAAA,EACF;AACF;","names":[]}

package/dist/tools/research.d.ts

@@ -0,0 +1,5 @@
+import type { ResearchInput, ResearchOutput, SearchEngine, StageResult } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+import type { SamplingCapableServer } from '../search/sampling.js';
+export declare function handleResearch(input: ResearchInput, engines: SearchEngine[], router: SmartRouter, _backendStatus?: unknown, server?: SamplingCapableServer): Promise<StageResult<ResearchOutput>>;
+//# sourceMappingURL=research.d.ts.map

package/dist/tools/research.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"research.d.ts","sourceRoot":"","sources":["../../src/tools/research.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAEV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,WAAW,EACZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQnE,wBAAsB,cAAc,CAClC,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,cAAc,CAAC,EAAE,OAAO,EACxB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAkDtC"}

package/dist/tools/research.js

@@ -0,0 +1,107 @@
+import { createLogger } from "../logger.js";
+import { runResearchPipeline } from "../research/pipeline.js";
+import {
+  buildEvidenceFromMarkdown,
+  applyTokenBudget,
+  applyAggregateMarkdownBudget
+} from "../search/evidence.js";
+import { applyOutputBudget } from "../search/truncate.js";
+const log = createLogger("research");
+const VALID_DEPTHS = /* @__PURE__ */ new Set(["quick", "standard", "comprehensive"]);
+const MAX_SOURCES_LIMIT = 50;
+const DEFAULT_MAX_TOKENS_OUT = 4e3;
+async function handleResearch(input, engines, router, _backendStatus, server) {
+  try {
+    if (!input.question || typeof input.question !== "string" || input.question.trim().length === 0) {
+      return invalidInput("question is required and must be a non-empty string");
+    }
+    if (input.depth && !VALID_DEPTHS.has(input.depth)) {
+      return invalidInput(
+        `depth must be one of: quick, standard, comprehensive. Got: "${input.depth}"`
+      );
+    }
+    if (input.max_sources !== void 0) {
+      if (typeof input.max_sources !== "number" || input.max_sources < 1) {
+        return invalidInput("max_sources must be a positive number");
+      }
+      if (input.max_sources > MAX_SOURCES_LIMIT) {
+        return invalidInput(`max_sources must be at most ${MAX_SOURCES_LIMIT}`);
+      }
+    }
+    log.info("research request received", {
+      question: input.question.slice(0, 100),
+      depth: input.depth ?? "standard",
+      max_sources: input.max_sources
+    });
+    const out = await runResearchPipeline(input, engines, router, server);
+    await attachEvidence(out, input);
+    if (out.error) {
+      return {
+        ok: false,
+        error: out.error,
+        error_reason: out.error,
+        stage: "research"
+      };
+    }
+    return { ok: true, data: out };
+  } catch (err) {
+    log.error("research handler failed", {
+      question: input.question?.slice(0, 100),
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return {
+      ok: false,
+      error: "research_failed",
+      error_reason: err instanceof Error ? err.message : String(err),
+      stage: "research"
+    };
+  }
+}
+async function attachEvidence(out, input) {
+  if (input.max_tokens_out !== void 0 && out.report) {
+    out.report = applyOutputBudget(out.report, { maxTokensOut: input.max_tokens_out });
+  }
+  if (out.sources.length === 0) return;
+  const includeFull = input.include_full_markdown ?? false;
+  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
+  const collected = [];
+  for (const s of out.sources) {
+    if (!s.markdown_content) continue;
+    const evs = await buildEvidenceFromMarkdown(
+      input.question,
+      s.title,
+      s.url,
+      s.markdown_content,
+      { maxItems: 1 }
+    );
+    collected.push(...evs);
+  }
+  const budgeted = applyTokenBudget(collected, maxTokensOut);
+  if (budgeted.length > 0) out.evidence = budgeted;
+  if (!includeFull) {
+    for (const s of out.sources) {
+      s.markdown_content = "";
+    }
+  } else {
+    applyAggregateMarkdownBudget(
+      out.sources,
+      (s) => s.markdown_content ?? "",
+      (s, body) => {
+        s.markdown_content = body;
+      },
+      { maxTokensOut }
+    );
+  }
+}
+function invalidInput(error_reason) {
+  return {
+    ok: false,
+    error: "invalid_input",
+    error_reason,
+    stage: "research"
+  };
+}
+export {
+  handleResearch
+};
+//# sourceMappingURL=research.js.map

package/dist/tools/research.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tools/research.ts"],"sourcesContent":["import { createLogger } from '../logger.js';\nimport { runResearchPipeline } from '../research/pipeline.js';\nimport {\n  buildEvidenceFromMarkdown,\n  applyTokenBudget,\n  applyAggregateMarkdownBudget,\n} from '../search/evidence.js';\nimport { applyOutputBudget } from '../search/truncate.js';\nimport type {\n  EvidenceItem,\n  ResearchInput,\n  ResearchOutput,\n  SearchEngine,\n  StageResult,\n} from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport type { SamplingCapableServer } from '../search/sampling.js';\n\nconst log = createLogger('research');\n\nconst VALID_DEPTHS = new Set(['quick', 'standard', 'comprehensive']);\nconst MAX_SOURCES_LIMIT = 50;\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n\nexport async function handleResearch(\n  input: ResearchInput,\n  engines: SearchEngine[],\n  router: SmartRouter,\n  _backendStatus?: unknown,\n  server?: SamplingCapableServer,\n): Promise<StageResult<ResearchOutput>> {\n  try {\n    if (!input.question || typeof input.question !== 'string' || input.question.trim().length === 0) {\n      return invalidInput('question is required and must be a non-empty string');\n    }\n\n    if (input.depth && !VALID_DEPTHS.has(input.depth)) {\n      return invalidInput(\n        `depth must be one of: quick, standard, comprehensive. Got: \"${input.depth}\"`,\n      );\n    }\n\n    if (input.max_sources !== undefined) {\n      if (typeof input.max_sources !== 'number' || input.max_sources < 1) {\n        return invalidInput('max_sources must be a positive number');\n      }\n      if (input.max_sources > MAX_SOURCES_LIMIT) {\n        return invalidInput(`max_sources must be at most ${MAX_SOURCES_LIMIT}`);\n      }\n    }\n\n    log.info('research request received', {\n      question: input.question.slice(0, 100),\n      depth: input.depth ?? 'standard',\n      max_sources: input.max_sources,\n    });\n\n    const out = await runResearchPipeline(input, engines, router, server);\n    await attachEvidence(out, input);\n    if (out.error) {\n      return {\n        ok: false,\n        error: out.error,\n        error_reason: out.error,\n        stage: 'research',\n      };\n    }\n    return { ok: true, data: out };\n  } catch (err) {\n    log.error('research handler failed', {\n      question: input.question?.slice(0, 100),\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return {\n      ok: false,\n      error: 'research_failed',\n      error_reason: err instanceof Error ? err.message : String(err),\n      stage: 'research',\n    };\n  }\n}\n\nasync function attachEvidence(out: ResearchOutput, input: ResearchInput): Promise<void> {\n  // Always honour max_tokens_out for the report text, regardless of sources.\n  if (input.max_tokens_out !== undefined && out.report) {\n    out.report = applyOutputBudget(out.report, { maxTokensOut: input.max_tokens_out });\n  }\n\n  if (out.sources.length === 0) return;\n  const includeFull = input.include_full_markdown ?? false;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n\n  const collected: EvidenceItem[] = [];\n  for (const s of out.sources) {\n    if (!s.markdown_content) continue;\n    const evs = await buildEvidenceFromMarkdown(\n      input.question,\n      s.title,\n      s.url,\n      s.markdown_content,\n      { maxItems: 1 },\n    );\n    collected.push(...evs);\n  }\n\n  const budgeted = applyTokenBudget(collected, maxTokensOut);\n  if (budgeted.length > 0) out.evidence = budgeted;\n\n  if (!includeFull) {\n    for (const s of out.sources) {\n      s.markdown_content = '';\n    }\n  } else {\n    applyAggregateMarkdownBudget(\n      out.sources,\n      (s) => s.markdown_content ?? '',\n      (s, body) => { s.markdown_content = body; },\n      { maxTokensOut },\n    );\n  }\n}\n\nfunction invalidInput(error_reason: string): StageResult<ResearchOutput> {\n  return {\n    ok: false,\n    error: 'invalid_input',\n    error_reason,\n    stage: 'research',\n  };\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAWlC,MAAM,MAAM,aAAa,UAAU;AAEnC,MAAM,eAAe,oBAAI,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC;AACnE,MAAM,oBAAoB;AAC1B,MAAM,yBAAyB;AAE/B,eAAsB,eACpB,OACA,SACA,QACA,gBACA,QACsC;AACtC,MAAI;AACF,QAAI,CAAC,MAAM,YAAY,OAAO,MAAM,aAAa,YAAY,MAAM,SAAS,KAAK,EAAE,WAAW,GAAG;AAC/F,aAAO,aAAa,qDAAqD;AAAA,IAC3E;AAEA,QAAI,MAAM,SAAS,CAAC,aAAa,IAAI,MAAM,KAAK,GAAG;AACjD,aAAO;AAAA,QACL,+DAA+D,MAAM,KAAK;AAAA,MAC5E;AAAA,IACF;AAEA,QAAI,MAAM,gBAAgB,QAAW;AACnC,UAAI,OAAO,MAAM,gBAAgB,YAAY,MAAM,cAAc,GAAG;AAClE,eAAO,aAAa,uCAAuC;AAAA,MAC7D;AACA,UAAI,MAAM,cAAc,mBAAmB;AACzC,eAAO,aAAa,+BAA+B,iBAAiB,EAAE;AAAA,MACxE;AAAA,IACF;AAEA,QAAI,KAAK,6BAA6B;AAAA,MACpC,UAAU,MAAM,SAAS,MAAM,GAAG,GAAG;AAAA,MACrC,OAAO,MAAM,SAAS;AAAA,MACtB,aAAa,MAAM;AAAA,IACrB,CAAC;AAED,UAAM,MAAM,MAAM,oBAAoB,OAAO,SAAS,QAAQ,MAAM;AACpE,UAAM,eAAe,KAAK,KAAK;AAC/B,QAAI,IAAI,OAAO;AACb,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,IAAI;AAAA,QACX,cAAc,IAAI;AAAA,QAClB,OAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO,EAAE,IAAI,MAAM,MAAM,IAAI;AAAA,EAC/B,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B;AAAA,MACnC,UAAU,MAAM,UAAU,MAAM,GAAG,GAAG;AAAA,MACtC,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MAC7D,OAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,eAAe,eAAe,KAAqB,OAAqC;AAEtF,MAAI,MAAM,mBAAmB,UAAa,IAAI,QAAQ;AACpD,QAAI,SAAS,kBAAkB,IAAI,QAAQ,EAAE,cAAc,MAAM,eAAe,CAAC;AAAA,EACnF;AAEA,MAAI,IAAI,QAAQ,WAAW,EAAG;AAC9B,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAE7C,QAAM,YAA4B,CAAC;AACnC,aAAW,KAAK,IAAI,SAAS;AAC3B,QAAI,CAAC,EAAE,iBAAkB;AACzB,UAAM,MAAM,MAAM;AAAA,MAChB,MAAM;AAAA,MACN,EAAE;AAAA,MACF,EAAE;AAAA,MACF,EAAE;AAAA,MACF,EAAE,UAAU,EAAE;AAAA,IAChB;AACA,cAAU,KAAK,GAAG,GAAG;AAAA,EACvB;AAEA,QAAM,WAAW,iBAAiB,WAAW,YAAY;AACzD,MAAI,SAAS,SAAS,EAAG,KAAI,WAAW;AAExC,MAAI,CAAC,aAAa;AAChB,eAAW,KAAK,IAAI,SAAS;AAC3B,QAAE,mBAAmB;AAAA,IACvB;AAAA,EACF,OAAO;AACL;AAAA,MACE,IAAI;AAAA,MACJ,CAAC,MAAM,EAAE,oBAAoB;AAAA,MAC7B,CAAC,GAAG,SAAS;AAAE,UAAE,mBAAmB;AAAA,MAAM;AAAA,MAC1C,EAAE,aAAa;AAAA,IACjB;AAAA,EACF;AACF;AAEA,SAAS,aAAa,cAAmD;AACvE,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO;AAAA,IACP;AAAA,IACA,OAAO;AAAA,EACT;AACF;","names":[]}

package/dist/tools/search.d.ts

@@ -1,4 +1,12 @@
-import type { SearchInput, SearchOutput, SearchEngine } from '../types.js';
+import type { SearchInput, SearchOutput, SearchEngine, ProgressCallback, StageResult } from '../types.js';
 import type { SmartRouter } from '../fetch/router.js';
-export declare function handleSearch(input: SearchInput, engines: SearchEngine[], router: SmartRouter): Promise<SearchOutput>;
+import type { BackendStatus } from '../server/backend-status.js';
+import type { SamplingCapableServer } from '../search/sampling.js';
+/**
+ * Thin handler — validates wiring args and delegates to the selected search
+ * provider (legacy SearXNG by default, V1 stub when `WIGOLO_SEARCH=v1`).
+ *
+ * All orchestration logic lives in `src/search/legacy/searxng-orchestrator.ts`.
+ */
+export declare function handleSearch(input: SearchInput, engines: SearchEngine[], router: SmartRouter, backendStatus?: BackendStatus, samplingServer?: SamplingCapableServer, onProgress?: ProgressCallback): Promise<StageResult<SearchOutput>>;
 //# sourceMappingURL=search.d.ts.map

package/dist/tools/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAoB,YAAY,EAAmB,MAAM,aAAa,CAAC;AAC9G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAkBtD,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,YAAY,CAAC,CA4HvB"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACZ,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAGnE;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,EAC7B,cAAc,CAAC,EAAE,qBAAqB,EACtC,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CASpC"}