@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/search/highlights.js

@@ -0,0 +1,167 @@
+import { getRerankProvider } from "../providers/rerank-provider.js";
+import { getConfig } from "../config.js";
+import { createLogger } from "../logger.js";
+import { parseHeadings, lineStartCharOffsets } from "../extraction/markdown.js";
+const log = createLogger("search");
+const MAX_PASSAGE_LENGTH = 500;
+const MIN_PASSAGE_LENGTH = 50;
+const DEFAULT_MAX_HIGHLIGHTS = 10;
+function shouldKeep(trimmed) {
+  if (trimmed.length < MIN_PASSAGE_LENGTH) return false;
+  if (trimmed.startsWith("#")) return false;
+  if (trimmed.startsWith("|")) return false;
+  if (trimmed.startsWith("```")) return false;
+  if (trimmed.startsWith("- ") && trimmed.length <= 120) return false;
+  return true;
+}
+function splitIntoPassages(markdown) {
+  if (!markdown) return [];
+  const out = [];
+  const re = /\n\n+/g;
+  let blockStart = 0;
+  let m;
+  const consider = (rawStart, rawEnd) => {
+    const raw = markdown.slice(rawStart, rawEnd);
+    if (!raw) return;
+    let leading = 0;
+    while (leading < raw.length && /\s/.test(raw[leading])) leading++;
+    let trailing = raw.length;
+    while (trailing > leading && /\s/.test(raw[trailing - 1])) trailing--;
+    if (trailing <= leading) return;
+    const trimmedStart = rawStart + leading;
+    const trimmedEnd = rawStart + trailing;
+    const trimmed = markdown.slice(trimmedStart, trimmedEnd);
+    if (!shouldKeep(trimmed)) return;
+    const text = trimmed.length > MAX_PASSAGE_LENGTH ? trimmed.slice(0, MAX_PASSAGE_LENGTH) : trimmed;
+    const charEnd = trimmedStart + text.length;
+    out.push({ text, charStart: trimmedStart, charEnd });
+  };
+  while ((m = re.exec(markdown)) !== null) {
+    consider(blockStart, m.index);
+    blockStart = m.index + m[0].length;
+  }
+  consider(blockStart, markdown.length);
+  return out;
+}
+function splitIntoPassageStrings(markdown) {
+  return splitIntoPassages(markdown).map((p) => p.text);
+}
+function mapPassageHeadings(markdown, passages) {
+  const lines = markdown.split("\n");
+  const headings = parseHeadings(lines);
+  const offsets = lineStartCharOffsets(lines);
+  const headingOffsets = headings.map((h) => ({ text: h.text, charStart: offsets[h.lineIndex] }));
+  return passages.map((p) => {
+    let nearest = null;
+    for (const h of headingOffsets) {
+      if (h.charStart <= p.charStart) nearest = h.text;
+      else break;
+    }
+    return { ...p, sectionHeading: nearest };
+  });
+}
+async function extractHighlights(query, results, maxHighlights = DEFAULT_MAX_HIGHLIGHTS) {
+  const citations = [];
+  const candidates = [];
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    citations.push({
+      index: i + 1,
+      url: r.url,
+      title: r.title,
+      snippet: r.snippet
+    });
+    const source = r.markdown_content ?? r.snippet ?? "";
+    const passages = splitIntoPassages(source);
+    const annotated = mapPassageHeadings(source, passages);
+    for (const p of annotated) {
+      candidates.push({
+        text: p.text,
+        sourceIndex: i + 1,
+        sourceUrl: r.url,
+        sourceTitle: r.title,
+        charStart: p.charStart,
+        charEnd: p.charEnd,
+        sectionHeading: p.sectionHeading
+      });
+    }
+  }
+  if (candidates.length === 0) {
+    return {
+      highlights: fallbackHighlights(results, maxHighlights),
+      citations,
+      reranker_used: false
+    };
+  }
+  const cfg = getConfig();
+  if (cfg.reranker === "onnx") {
+    try {
+      const provider = await getRerankProvider();
+      const scored = await provider.rerank(
+        query,
+        candidates.map((c, i) => ({ id: String(i), text: c.text }))
+      );
+      if (scored.length > 0) {
+        const ranked = scored.slice(0, maxHighlights);
+        const highlights = ranked.map((s) => {
+          const cand = candidates[Number(s.id)];
+          return {
+            text: cand.text,
+            source_index: cand.sourceIndex,
+            relevance_score: s.score,
+            source_url: cand.sourceUrl,
+            source_title: cand.sourceTitle,
+            section_heading: cand.sectionHeading,
+            source_span: { start: cand.charStart, end: cand.charEnd }
+          };
+        });
+        return { highlights, citations, reranker_used: true };
+      }
+    } catch (err) {
+      log.debug("rerank provider failed, using fallback passages", { error: String(err) });
+    }
+  }
+  return { highlights: fallbackHighlights(results, maxHighlights), citations, reranker_used: false };
+}
+function fallbackHighlights(results, maxHighlights) {
+  const out = [];
+  for (let i = 0; i < results.length && out.length < maxHighlights; i++) {
+    const r = results[i];
+    const source = r.markdown_content ?? "";
+    const passages = source ? splitIntoPassages(source) : [];
+    if (passages.length > 0) {
+      const annotated = mapPassageHeadings(source, [passages[0]])[0];
+      const text2 = annotated.text.slice(0, MAX_PASSAGE_LENGTH);
+      out.push({
+        text: text2,
+        source_index: i + 1,
+        relevance_score: r.relevance_score,
+        source_url: r.url,
+        source_title: r.title,
+        section_heading: annotated.sectionHeading,
+        source_span: { start: annotated.charStart, end: annotated.charStart + text2.length }
+      });
+      continue;
+    }
+    const snippet = r.snippet ?? "";
+    if (!snippet) continue;
+    const text = snippet.slice(0, MAX_PASSAGE_LENGTH);
+    out.push({
+      text,
+      source_index: i + 1,
+      relevance_score: r.relevance_score,
+      source_url: r.url,
+      source_title: r.title,
+      section_heading: null,
+      source_span: { start: 0, end: text.length }
+    });
+  }
+  return out;
+}
+export {
+  extractHighlights,
+  fallbackHighlights,
+  mapPassageHeadings,
+  splitIntoPassages
+};
+//# sourceMappingURL=highlights.js.map

package/dist/search/highlights.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/search/highlights.ts"],"sourcesContent":["import type { SearchResultItem, Citation, Highlight } from '../types.js';\nimport { getRerankProvider } from '../providers/rerank-provider.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\nimport { parseHeadings, lineStartCharOffsets } from '../extraction/markdown.js';\n\nconst log = createLogger('search');\n\nconst MAX_PASSAGE_LENGTH = 500;\nconst MIN_PASSAGE_LENGTH = 50;\nconst DEFAULT_MAX_HIGHLIGHTS = 10;\n\nexport interface HighlightSynthesisResult {\n  highlights: Highlight[];\n  citations: Citation[];\n  reranker_used: boolean;\n}\n\nexport interface Passage {\n  text: string;\n  charStart: number;\n  charEnd: number;\n}\n\ninterface PassageCandidate {\n  text: string;\n  sourceIndex: number;\n  sourceUrl: string;\n  sourceTitle: string;\n  charStart: number;\n  charEnd: number;\n  sectionHeading: string | null;\n}\n\nfunction shouldKeep(trimmed: string): boolean {\n  if (trimmed.length < MIN_PASSAGE_LENGTH) return false;\n  if (trimmed.startsWith('#')) return false;\n  if (trimmed.startsWith('|')) return false;\n  if (trimmed.startsWith('```')) return false;\n  if (trimmed.startsWith('- ') && trimmed.length <= 120) return false;\n  return true;\n}\n\n// Walk the source markdown block-by-block (separated by blank lines) tracking\n// char offsets so each surviving passage carries an accurate {charStart,\n// charEnd} range pointing back into the original markdown.\nexport function splitIntoPassages(markdown: string): Passage[] {\n  if (!markdown) return [];\n  const out: Passage[] = [];\n  const re = /\\n\\n+/g;\n  let blockStart = 0;\n  let m: RegExpExecArray | null;\n  const consider = (rawStart: number, rawEnd: number) => {\n    // raw block is markdown.slice(rawStart, rawEnd); compute trimmed range.\n    const raw = markdown.slice(rawStart, rawEnd);\n    if (!raw) return;\n    let leading = 0;\n    while (leading < raw.length && /\\s/.test(raw[leading])) leading++;\n    let trailing = raw.length;\n    while (trailing > leading && /\\s/.test(raw[trailing - 1])) trailing--;\n    if (trailing <= leading) return;\n    const trimmedStart = rawStart + leading;\n    const trimmedEnd = rawStart + trailing;\n    const trimmed = markdown.slice(trimmedStart, trimmedEnd);\n    if (!shouldKeep(trimmed)) return;\n    const text = trimmed.length > MAX_PASSAGE_LENGTH ? trimmed.slice(0, MAX_PASSAGE_LENGTH) : trimmed;\n    const charEnd = trimmedStart + text.length;\n    out.push({ text, charStart: trimmedStart, charEnd });\n  };\n  while ((m = re.exec(markdown)) !== null) {\n    consider(blockStart, m.index);\n    blockStart = m.index + m[0].length;\n  }\n  consider(blockStart, markdown.length);\n  return out;\n}\n\n// Internal helper preserved for callers that only need the text strings.\nfunction splitIntoPassageStrings(markdown: string): string[] {\n  return splitIntoPassages(markdown).map((p) => p.text);\n}\n\nexport interface AnnotatedPassage extends Passage {\n  sectionHeading: string | null;\n}\n\n// Annotate each passage with the nearest preceding markdown heading. Uses\n// `parseHeadings` and a char-offset prefix sum so the lookup is O(passages\n// * headings) without re-parsing markdown for every passage.\nexport function mapPassageHeadings(\n  markdown: string,\n  passages: Passage[],\n): AnnotatedPassage[] {\n  const lines = markdown.split('\\n');\n  const headings = parseHeadings(lines);\n  const offsets = lineStartCharOffsets(lines);\n  const headingOffsets = headings.map((h) => ({ text: h.text, charStart: offsets[h.lineIndex] }));\n  return passages.map((p) => {\n    let nearest: string | null = null;\n    for (const h of headingOffsets) {\n      if (h.charStart <= p.charStart) nearest = h.text;\n      else break;\n    }\n    return { ...p, sectionHeading: nearest };\n  });\n}\n\n// Score passages across all results and return the top N using the\n// cross-encoder rerank provider, with a graceful first-paragraph fallback\n// when reranking is disabled or fails. Each Highlight carries a\n// source_index suitable for citing.\nexport async function extractHighlights(\n  query: string,\n  results: SearchResultItem[],\n  maxHighlights: number = DEFAULT_MAX_HIGHLIGHTS,\n): Promise<HighlightSynthesisResult> {\n  const citations: Citation[] = [];\n  const candidates: PassageCandidate[] = [];\n\n  for (let i = 0; i < results.length; i++) {\n    const r = results[i];\n    citations.push({\n      index: i + 1,\n      url: r.url,\n      title: r.title,\n      snippet: r.snippet,\n    });\n\n    const source = r.markdown_content ?? r.snippet ?? '';\n    const passages = splitIntoPassages(source);\n    const annotated = mapPassageHeadings(source, passages);\n    for (const p of annotated) {\n      candidates.push({\n        text: p.text,\n        sourceIndex: i + 1,\n        sourceUrl: r.url,\n        sourceTitle: r.title,\n        charStart: p.charStart,\n        charEnd: p.charEnd,\n        sectionHeading: p.sectionHeading,\n      });\n    }\n  }\n\n  if (candidates.length === 0) {\n    return {\n      highlights: fallbackHighlights(results, maxHighlights),\n      citations,\n      reranker_used: false,\n    };\n  }\n\n  const cfg = getConfig();\n  if (cfg.reranker === 'onnx') {\n    try {\n      const provider = await getRerankProvider();\n      const scored = await provider.rerank(\n        query,\n        candidates.map((c, i) => ({ id: String(i), text: c.text })),\n      );\n      if (scored.length > 0) {\n        const ranked = scored.slice(0, maxHighlights);\n        const highlights = ranked.map<Highlight>((s) => {\n          const cand = candidates[Number(s.id)];\n          return {\n            text: cand.text,\n            source_index: cand.sourceIndex,\n            relevance_score: s.score,\n            source_url: cand.sourceUrl,\n            source_title: cand.sourceTitle,\n            section_heading: cand.sectionHeading,\n            source_span: { start: cand.charStart, end: cand.charEnd },\n          };\n        });\n        return { highlights, citations, reranker_used: true };\n      }\n    } catch (err) {\n      log.debug('rerank provider failed, using fallback passages', { error: String(err) });\n    }\n  }\n\n  return { highlights: fallbackHighlights(results, maxHighlights), citations, reranker_used: false };\n}\n\n// Fallback when the cross-encoder reranker is unavailable: take the first substantive paragraph\n// from each source (ordered by engine relevance). Preserves citation indices\n// so host LLMs can still cite [N] correctly.\nexport function fallbackHighlights(\n  results: SearchResultItem[],\n  maxHighlights: number,\n): Highlight[] {\n  const out: Highlight[] = [];\n  for (let i = 0; i < results.length && out.length < maxHighlights; i++) {\n    const r = results[i];\n    const source = r.markdown_content ?? '';\n    const passages = source ? splitIntoPassages(source) : [];\n    if (passages.length > 0) {\n      const annotated = mapPassageHeadings(source, [passages[0]])[0];\n      const text = annotated.text.slice(0, MAX_PASSAGE_LENGTH);\n      out.push({\n        text,\n        source_index: i + 1,\n        relevance_score: r.relevance_score,\n        source_url: r.url,\n        source_title: r.title,\n        section_heading: annotated.sectionHeading,\n        source_span: { start: annotated.charStart, end: annotated.charStart + text.length },\n      });\n      continue;\n    }\n    const snippet = r.snippet ?? '';\n    if (!snippet) continue;\n    const text = snippet.slice(0, MAX_PASSAGE_LENGTH);\n    out.push({\n      text,\n      source_index: i + 1,\n      relevance_score: r.relevance_score,\n      source_url: r.url,\n      source_title: r.title,\n      section_heading: null,\n      source_span: { start: 0, end: text.length },\n    });\n  }\n  return out;\n}\n"],"mappings":"AACA,SAAS,yBAAyB;AAClC,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAC7B,SAAS,eAAe,4BAA4B;AAEpD,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,qBAAqB;AAC3B,MAAM,qBAAqB;AAC3B,MAAM,yBAAyB;AAwB/B,SAAS,WAAW,SAA0B;AAC5C,MAAI,QAAQ,SAAS,mBAAoB,QAAO;AAChD,MAAI,QAAQ,WAAW,GAAG,EAAG,QAAO;AACpC,MAAI,QAAQ,WAAW,GAAG,EAAG,QAAO;AACpC,MAAI,QAAQ,WAAW,KAAK,EAAG,QAAO;AACtC,MAAI,QAAQ,WAAW,IAAI,KAAK,QAAQ,UAAU,IAAK,QAAO;AAC9D,SAAO;AACT;AAKO,SAAS,kBAAkB,UAA6B;AAC7D,MAAI,CAAC,SAAU,QAAO,CAAC;AACvB,QAAM,MAAiB,CAAC;AACxB,QAAM,KAAK;AACX,MAAI,aAAa;AACjB,MAAI;AACJ,QAAM,WAAW,CAAC,UAAkB,WAAmB;AAErD,UAAM,MAAM,SAAS,MAAM,UAAU,MAAM;AAC3C,QAAI,CAAC,IAAK;AACV,QAAI,UAAU;AACd,WAAO,UAAU,IAAI,UAAU,KAAK,KAAK,IAAI,OAAO,CAAC,EAAG;AACxD,QAAI,WAAW,IAAI;AACnB,WAAO,WAAW,WAAW,KAAK,KAAK,IAAI,WAAW,CAAC,CAAC,EAAG;AAC3D,QAAI,YAAY,QAAS;AACzB,UAAM,eAAe,WAAW;AAChC,UAAM,aAAa,WAAW;AAC9B,UAAM,UAAU,SAAS,MAAM,cAAc,UAAU;AACvD,QAAI,CAAC,WAAW,OAAO,EAAG;AAC1B,UAAM,OAAO,QAAQ,SAAS,qBAAqB,QAAQ,MAAM,GAAG,kBAAkB,IAAI;AAC1F,UAAM,UAAU,eAAe,KAAK;AACpC,QAAI,KAAK,EAAE,MAAM,WAAW,cAAc,QAAQ,CAAC;AAAA,EACrD;AACA,UAAQ,IAAI,GAAG,KAAK,QAAQ,OAAO,MAAM;AACvC,aAAS,YAAY,EAAE,KAAK;AAC5B,iBAAa,EAAE,QAAQ,EAAE,CAAC,EAAE;AAAA,EAC9B;AACA,WAAS,YAAY,SAAS,MAAM;AACpC,SAAO;AACT;AAGA,SAAS,wBAAwB,UAA4B;AAC3D,SAAO,kBAAkB,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACtD;AASO,SAAS,mBACd,UACA,UACoB;AACpB,QAAM,QAAQ,SAAS,MAAM,IAAI;AACjC,QAAM,WAAW,cAAc,KAAK;AACpC,QAAM,UAAU,qBAAqB,KAAK;AAC1C,QAAM,iBAAiB,SAAS,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,QAAQ,EAAE,SAAS,EAAE,EAAE;AAC9F,SAAO,SAAS,IAAI,CAAC,MAAM;AACzB,QAAI,UAAyB;AAC7B,eAAW,KAAK,gBAAgB;AAC9B,UAAI,EAAE,aAAa,EAAE,UAAW,WAAU,EAAE;AAAA,UACvC;AAAA,IACP;AACA,WAAO,EAAE,GAAG,GAAG,gBAAgB,QAAQ;AAAA,EACzC,CAAC;AACH;AAMA,eAAsB,kBACpB,OACA,SACA,gBAAwB,wBACW;AACnC,QAAM,YAAwB,CAAC;AAC/B,QAAM,aAAiC,CAAC;AAExC,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAM,IAAI,QAAQ,CAAC;AACnB,cAAU,KAAK;AAAA,MACb,OAAO,IAAI;AAAA,MACX,KAAK,EAAE;AAAA,MACP,OAAO,EAAE;AAAA,MACT,SAAS,EAAE;AAAA,IACb,CAAC;AAED,UAAM,SAAS,EAAE,oBAAoB,EAAE,WAAW;AAClD,UAAM,WAAW,kBAAkB,MAAM;AACzC,UAAM,YAAY,mBAAmB,QAAQ,QAAQ;AACrD,eAAW,KAAK,WAAW;AACzB,iBAAW,KAAK;AAAA,QACd,MAAM,EAAE;AAAA,QACR,aAAa,IAAI;AAAA,QACjB,WAAW,EAAE;AAAA,QACb,aAAa,EAAE;AAAA,QACf,WAAW,EAAE;AAAA,QACb,SAAS,EAAE;AAAA,QACX,gBAAgB,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,WAAW,WAAW,GAAG;AAC3B,WAAO;AAAA,MACL,YAAY,mBAAmB,SAAS,aAAa;AAAA,MACrD;AAAA,MACA,eAAe;AAAA,IACjB;AAAA,EACF;AAEA,QAAM,MAAM,UAAU;AACtB,MAAI,IAAI,aAAa,QAAQ;AAC3B,QAAI;AACF,YAAM,WAAW,MAAM,kBAAkB;AACzC,YAAM,SAAS,MAAM,SAAS;AAAA,QAC5B;AAAA,QACA,WAAW,IAAI,CAAC,GAAG,OAAO,EAAE,IAAI,OAAO,CAAC,GAAG,MAAM,EAAE,KAAK,EAAE;AAAA,MAC5D;AACA,UAAI,OAAO,SAAS,GAAG;AACrB,cAAM,SAAS,OAAO,MAAM,GAAG,aAAa;AAC5C,cAAM,aAAa,OAAO,IAAe,CAAC,MAAM;AAC9C,gBAAM,OAAO,WAAW,OAAO,EAAE,EAAE,CAAC;AACpC,iBAAO;AAAA,YACL,MAAM,KAAK;AAAA,YACX,cAAc,KAAK;AAAA,YACnB,iBAAiB,EAAE;AAAA,YACnB,YAAY,KAAK;AAAA,YACjB,cAAc,KAAK;AAAA,YACnB,iBAAiB,KAAK;AAAA,YACtB,aAAa,EAAE,OAAO,KAAK,WAAW,KAAK,KAAK,QAAQ;AAAA,UAC1D;AAAA,QACF,CAAC;AACD,eAAO,EAAE,YAAY,WAAW,eAAe,KAAK;AAAA,MACtD;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,mDAAmD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACrF;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,mBAAmB,SAAS,aAAa,GAAG,WAAW,eAAe,MAAM;AACnG;AAKO,SAAS,mBACd,SACA,eACa;AACb,QAAM,MAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,QAAQ,UAAU,IAAI,SAAS,eAAe,KAAK;AACrE,UAAM,IAAI,QAAQ,CAAC;AACnB,UAAM,SAAS,EAAE,oBAAoB;AACrC,UAAM,WAAW,SAAS,kBAAkB,MAAM,IAAI,CAAC;AACvD,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,YAAY,mBAAmB,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;AAC7D,YAAMA,QAAO,UAAU,KAAK,MAAM,GAAG,kBAAkB;AACvD,UAAI,KAAK;AAAA,QACP,MAAAA;AAAA,QACA,cAAc,IAAI;AAAA,QAClB,iBAAiB,EAAE;AAAA,QACnB,YAAY,EAAE;AAAA,QACd,cAAc,EAAE;AAAA,QAChB,iBAAiB,UAAU;AAAA,QAC3B,aAAa,EAAE,OAAO,UAAU,WAAW,KAAK,UAAU,YAAYA,MAAK,OAAO;AAAA,MACpF,CAAC;AACD;AAAA,IACF;AACA,UAAM,UAAU,EAAE,WAAW;AAC7B,QAAI,CAAC,QAAS;AACd,UAAM,OAAO,QAAQ,MAAM,GAAG,kBAAkB;AAChD,QAAI,KAAK;AAAA,MACP;AAAA,MACA,cAAc,IAAI;AAAA,MAClB,iBAAiB,EAAE;AAAA,MACnB,YAAY,EAAE;AAAA,MACd,cAAc,EAAE;AAAA,MAChB,iBAAiB;AAAA,MACjB,aAAa,EAAE,OAAO,GAAG,KAAK,KAAK,OAAO;AAAA,IAC5C,CAAC;AAAA,EACH;AACA,SAAO;AACT;","names":["text"]}

package/dist/search/language-filter.d.ts

@@ -0,0 +1,29 @@
+export interface RawSearchResult {
+    url: string;
+    title: string;
+    snippet: string;
+    engine: string;
+    [k: string]: unknown;
+}
+export interface DiscardedResult<T extends RawLike = RawSearchResult> {
+    result: T;
+    reason: 'invalid_url' | 'language_mismatch' | 'engine_batch_dropped';
+}
+export interface FilterOptions {
+    target: string;
+    dropThreshold: number;
+}
+export interface FilterResult<T extends RawLike = RawSearchResult> {
+    results: T[];
+    discarded: DiscardedResult<T>[];
+    warnings: string[];
+}
+interface RawLike {
+    url: string;
+    title: string;
+    snippet: string;
+    engine: string;
+}
+export declare function filterByLanguage<T extends RawLike>(results: T[], opts: FilterOptions): FilterResult<T>;
+export {};
+//# sourceMappingURL=language-filter.d.ts.map

package/dist/search/language-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"language-filter.d.ts","sourceRoot":"","sources":["../../src/search/language-filter.ts"],"names":[],"mappings":"AAgBA,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,eAAe,CAAC,CAAC,SAAS,OAAO,GAAG,eAAe;IAClE,MAAM,EAAE,CAAC,CAAC;IACV,MAAM,EAAE,aAAa,GAAG,mBAAmB,GAAG,sBAAsB,CAAC;CACtE;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,SAAS,OAAO,GAAG,eAAe;IAC/D,OAAO,EAAE,CAAC,EAAE,CAAC;IACb,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;IAChC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,UAAU,OAAO;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AAwBD,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,OAAO,EAChD,OAAO,EAAE,CAAC,EAAE,EACZ,IAAI,EAAE,aAAa,GAClB,YAAY,CAAC,CAAC,CAAC,CA6DjB"}

package/dist/search/language-filter.js

@@ -0,0 +1,126 @@
+import { detectAll } from "tinyld";
+import { createLogger } from "../logger.js";
+const MIN_DETECT_CHARS = 12;
+const MIN_CONFIDENCE = 0.1;
+const LATIN_LANGS = /* @__PURE__ */ new Set([
+  "en",
+  "es",
+  "fr",
+  "pt",
+  "de",
+  "it",
+  "nl",
+  "da",
+  "sv",
+  "no",
+  "fi",
+  "is",
+  "pl",
+  "cs",
+  "sk",
+  "hu",
+  "ro",
+  "hr",
+  "sl",
+  "lt",
+  "lv",
+  "et",
+  "tr",
+  "vi",
+  "id",
+  "ms",
+  "tl",
+  "sw",
+  "af",
+  "ca",
+  "gl",
+  "eu",
+  "ga",
+  "cy",
+  "mt",
+  "sq",
+  "lb",
+  "fo",
+  "ber",
+  "so",
+  "ha",
+  "yo",
+  "ig",
+  "zu",
+  "xh",
+  "st",
+  "tn"
+]);
+const log = createLogger("language-filter");
+function isValidUrl(u) {
+  try {
+    const parsed = new URL(u);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function detectLang(text) {
+  const t = text?.trim() ?? "";
+  if (t.length < MIN_DETECT_CHARS) return "und";
+  try {
+    const ranked = detectAll(t);
+    const top = ranked[0];
+    if (!top || top.accuracy < MIN_CONFIDENCE) return "und";
+    return top.lang || "und";
+  } catch {
+    return "und";
+  }
+}
+function filterByLanguage(results, opts) {
+  const discarded = [];
+  const warnings = [];
+  const urlValid = [];
+  for (const r of results) {
+    if (!isValidUrl(r.url)) {
+      discarded.push({ result: r, reason: "invalid_url" });
+      continue;
+    }
+    urlValid.push(r);
+  }
+  if (urlValid.length === 0) return { results: [], discarded, warnings };
+  const byEngine = /* @__PURE__ */ new Map();
+  for (const r of urlValid) {
+    const arr = byEngine.get(r.engine) ?? [];
+    arr.push(r);
+    byEngine.set(r.engine, arr);
+  }
+  const targetIsLatin = LATIN_LANGS.has(opts.target);
+  const isMismatch = (lang) => {
+    if (lang === opts.target || lang === "und") return false;
+    if (targetIsLatin && LATIN_LANGS.has(lang)) return false;
+    return true;
+  };
+  const kept = [];
+  for (const [engine, batch] of byEngine) {
+    let nonTarget = 0;
+    const langs = batch.map((r) => detectLang(`${r.title} ${r.snippet}`));
+    for (const l of langs) if (isMismatch(l)) nonTarget += 1;
+    const ratio = nonTarget / batch.length;
+    if (ratio > opts.dropThreshold) {
+      warnings.push(
+        `engine_language_mismatch: ${engine} returned ${Math.round(ratio * 100)}% non-${opts.target}; batch dropped`
+      );
+      for (const r of batch) discarded.push({ result: r, reason: "engine_batch_dropped" });
+      log.warn("dropped engine batch for language mismatch", { engine, ratio });
+      continue;
+    }
+    for (let i = 0; i < batch.length; i += 1) {
+      if (isMismatch(langs[i])) {
+        discarded.push({ result: batch[i], reason: "language_mismatch" });
+      } else {
+        kept.push(batch[i]);
+      }
+    }
+  }
+  return { results: kept, discarded, warnings };
+}
+export {
+  filterByLanguage
+};
+//# sourceMappingURL=language-filter.js.map

package/dist/search/language-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/search/language-filter.ts"],"sourcesContent":["import { detectAll } from 'tinyld';\nimport { createLogger } from '../logger.js';\n\nconst MIN_DETECT_CHARS = 12;\nconst MIN_CONFIDENCE = 0.1;\n\n// Languages that use Latin script — used to avoid false positives when target is Latin.\nconst LATIN_LANGS = new Set([\n  'en', 'es', 'fr', 'pt', 'de', 'it', 'nl', 'da', 'sv', 'no', 'fi', 'is',\n  'pl', 'cs', 'sk', 'hu', 'ro', 'hr', 'sl', 'lt', 'lv', 'et', 'tr', 'vi',\n  'id', 'ms', 'tl', 'sw', 'af', 'ca', 'gl', 'eu', 'ga', 'cy', 'mt', 'sq',\n  'lb', 'fo', 'ber', 'so', 'ha', 'yo', 'ig', 'zu', 'xh', 'st', 'tn',\n]);\n\nconst log = createLogger('language-filter');\n\nexport interface RawSearchResult {\n  url: string;\n  title: string;\n  snippet: string;\n  engine: string;\n  [k: string]: unknown;\n}\n\nexport interface DiscardedResult<T extends RawLike = RawSearchResult> {\n  result: T;\n  reason: 'invalid_url' | 'language_mismatch' | 'engine_batch_dropped';\n}\n\nexport interface FilterOptions {\n  target: string;            // ISO-639 code, e.g. 'en'\n  dropThreshold: number;     // fraction of batch non-target before drop, e.g. 0.4\n}\n\nexport interface FilterResult<T extends RawLike = RawSearchResult> {\n  results: T[];\n  discarded: DiscardedResult<T>[];\n  warnings: string[];\n}\n\ninterface RawLike {\n  url: string;\n  title: string;\n  snippet: string;\n  engine: string;\n}\n\nfunction isValidUrl(u: string): boolean {\n  try {\n    const parsed = new URL(u);\n    return parsed.protocol === 'http:' || parsed.protocol === 'https:';\n  } catch {\n    return false;\n  }\n}\n\nfunction detectLang(text: string): string {\n  const t = text?.trim() ?? '';\n  if (t.length < MIN_DETECT_CHARS) return 'und';\n  try {\n    const ranked = detectAll(t);\n    const top = ranked[0];\n    if (!top || top.accuracy < MIN_CONFIDENCE) return 'und';\n    return top.lang || 'und';\n  } catch {\n    return 'und';\n  }\n}\n\nexport function filterByLanguage<T extends RawLike>(\n  results: T[],\n  opts: FilterOptions,\n): FilterResult<T> {\n  const discarded: DiscardedResult<T>[] = [];\n  const warnings: string[] = [];\n\n  // Step 1: drop invalid URLs first\n  const urlValid: T[] = [];\n  for (const r of results) {\n    if (!isValidUrl(r.url)) {\n      discarded.push({ result: r, reason: 'invalid_url' });\n      continue;\n    }\n    urlValid.push(r);\n  }\n\n  if (urlValid.length === 0) return { results: [], discarded, warnings };\n\n  // Step 2: per-engine batch language check\n  const byEngine = new Map<string, T[]>();\n  for (const r of urlValid) {\n    const arr = byEngine.get(r.engine) ?? [];\n    arr.push(r);\n    byEngine.set(r.engine, arr);\n  }\n\n  const targetIsLatin = LATIN_LANGS.has(opts.target);\n  const isMismatch = (lang: string): boolean => {\n    if (lang === opts.target || lang === 'und') return false;\n    // Script-aware: Latin-target vs Latin-detected is treated as a match\n    // because tinyld misclassifies short Latin-script text into other Latin\n    // languages with low confidence.\n    if (targetIsLatin && LATIN_LANGS.has(lang)) return false;\n    return true;\n  };\n\n  const kept: T[] = [];\n  for (const [engine, batch] of byEngine) {\n    let nonTarget = 0;\n    const langs = batch.map(r => detectLang(`${r.title} ${r.snippet}`));\n    for (const l of langs) if (isMismatch(l)) nonTarget += 1;\n    const ratio = nonTarget / batch.length;\n\n    if (ratio > opts.dropThreshold) {\n      warnings.push(\n        `engine_language_mismatch: ${engine} returned ${Math.round(ratio * 100)}% non-${opts.target}; batch dropped`,\n      );\n      for (const r of batch) discarded.push({ result: r, reason: 'engine_batch_dropped' });\n      log.warn('dropped engine batch for language mismatch', { engine, ratio });\n      continue;\n    }\n\n    // Drop individual non-target results inside an otherwise-fine batch\n    for (let i = 0; i < batch.length; i += 1) {\n      if (isMismatch(langs[i])) {\n        discarded.push({ result: batch[i], reason: 'language_mismatch' });\n      } else {\n        kept.push(batch[i]);\n      }\n    }\n  }\n\n  return { results: kept, discarded, warnings };\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAE7B,MAAM,mBAAmB;AACzB,MAAM,iBAAiB;AAGvB,MAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAClE;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAClE;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAClE;AAAA,EAAM;AAAA,EAAM;AAAA,EAAO;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAC/D,CAAC;AAED,MAAM,MAAM,aAAa,iBAAiB;AAiC1C,SAAS,WAAW,GAAoB;AACtC,MAAI;AACF,UAAM,SAAS,IAAI,IAAI,CAAC;AACxB,WAAO,OAAO,aAAa,WAAW,OAAO,aAAa;AAAA,EAC5D,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,WAAW,MAAsB;AACxC,QAAM,IAAI,MAAM,KAAK,KAAK;AAC1B,MAAI,EAAE,SAAS,iBAAkB,QAAO;AACxC,MAAI;AACF,UAAM,SAAS,UAAU,CAAC;AAC1B,UAAM,MAAM,OAAO,CAAC;AACpB,QAAI,CAAC,OAAO,IAAI,WAAW,eAAgB,QAAO;AAClD,WAAO,IAAI,QAAQ;AAAA,EACrB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,iBACd,SACA,MACiB;AACjB,QAAM,YAAkC,CAAC;AACzC,QAAM,WAAqB,CAAC;AAG5B,QAAM,WAAgB,CAAC;AACvB,aAAW,KAAK,SAAS;AACvB,QAAI,CAAC,WAAW,EAAE,GAAG,GAAG;AACtB,gBAAU,KAAK,EAAE,QAAQ,GAAG,QAAQ,cAAc,CAAC;AACnD;AAAA,IACF;AACA,aAAS,KAAK,CAAC;AAAA,EACjB;AAEA,MAAI,SAAS,WAAW,EAAG,QAAO,EAAE,SAAS,CAAC,GAAG,WAAW,SAAS;AAGrE,QAAM,WAAW,oBAAI,IAAiB;AACtC,aAAW,KAAK,UAAU;AACxB,UAAM,MAAM,SAAS,IAAI,EAAE,MAAM,KAAK,CAAC;AACvC,QAAI,KAAK,CAAC;AACV,aAAS,IAAI,EAAE,QAAQ,GAAG;AAAA,EAC5B;AAEA,QAAM,gBAAgB,YAAY,IAAI,KAAK,MAAM;AACjD,QAAM,aAAa,CAAC,SAA0B;AAC5C,QAAI,SAAS,KAAK,UAAU,SAAS,MAAO,QAAO;AAInD,QAAI,iBAAiB,YAAY,IAAI,IAAI,EAAG,QAAO;AACnD,WAAO;AAAA,EACT;AAEA,QAAM,OAAY,CAAC;AACnB,aAAW,CAAC,QAAQ,KAAK,KAAK,UAAU;AACtC,QAAI,YAAY;AAChB,UAAM,QAAQ,MAAM,IAAI,OAAK,WAAW,GAAG,EAAE,KAAK,IAAI,EAAE,OAAO,EAAE,CAAC;AAClE,eAAW,KAAK,MAAO,KAAI,WAAW,CAAC,EAAG,cAAa;AACvD,UAAM,QAAQ,YAAY,MAAM;AAEhC,QAAI,QAAQ,KAAK,eAAe;AAC9B,eAAS;AAAA,QACP,6BAA6B,MAAM,aAAa,KAAK,MAAM,QAAQ,GAAG,CAAC,SAAS,KAAK,MAAM;AAAA,MAC7F;AACA,iBAAW,KAAK,MAAO,WAAU,KAAK,EAAE,QAAQ,GAAG,QAAQ,uBAAuB,CAAC;AACnF,UAAI,KAAK,8CAA8C,EAAE,QAAQ,MAAM,CAAC;AACxE;AAAA,IACF;AAGA,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AACxC,UAAI,WAAW,MAAM,CAAC,CAAC,GAAG;AACxB,kBAAU,KAAK,EAAE,QAAQ,MAAM,CAAC,GAAG,QAAQ,oBAAoB,CAAC;AAAA,MAClE,OAAO;AACL,aAAK,KAAK,MAAM,CAAC,CAAC;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,SAAS,MAAM,WAAW,SAAS;AAC9C;","names":[]}

package/dist/search/legacy/searxng-orchestrator.d.ts

@@ -0,0 +1,4 @@
+import type { SearchInput, SearchOutput, StageResult } from '../../types.js';
+import type { SearchContext } from '../../providers/search-provider.js';
+export declare function runSearxngSearch(input: SearchInput, ctx: SearchContext): Promise<StageResult<SearchOutput>>;
+//# sourceMappingURL=searxng-orchestrator.d.ts.map

package/dist/search/legacy/searxng-orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"searxng-orchestrator.d.ts","sourceRoot":"","sources":["../../../src/search/legacy/searxng-orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EAGZ,WAAW,EACZ,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AA0BxE,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,aAAa,GACjB,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAqdpC"}