kiku-agent 1.2.0 → 1.3.1

package/server/agent/websearch/extract.js

@@ -0,0 +1,605 @@
+let _cheerioLoad = null;
+
+export async function loadCheerio() {
+  if (_cheerioLoad) return _cheerioLoad;
+  try {
+    const mod = await import('cheerio');
+    _cheerioLoad = mod.load;
+    return _cheerioLoad;
+  } catch {
+    return null;
+  }
+}
+
+const CHARSET_ALIASES = new Map(Object.entries({
+  'utf8': 'utf-8',
+  'utf-8': 'utf-8',
+  'unicode-1-1-utf-8': 'utf-8',
+  'utf-16': 'utf-16le',
+  'utf-16le': 'utf-16le',
+  'unicodefeff': 'utf-16le',
+  'utf-16be': 'utf-16be',
+  'unicodefffe': 'utf-16be',
+  'latin1': 'windows-1252',
+  'iso-8859-1': 'windows-1252',
+  'iso8859-1': 'windows-1252',
+  'windows-1252': 'windows-1252',
+  'cp1252': 'windows-1252',
+  'ascii': 'windows-1252',
+  'us-ascii': 'windows-1252',
+  'tis-620': 'windows-874',
+  'iso-8859-11': 'windows-874',
+  'windows-874': 'windows-874',
+  'dos-874': 'windows-874',
+  'gb2312': 'gbk',
+  'gb_2312': 'gbk',
+  'gbk': 'gbk',
+  'gb18030': 'gb18030',
+  'big5': 'big5',
+  'big5-hkscs': 'big5',
+  'shift_jis': 'shift_jis',
+  'shift-jis': 'shift_jis',
+  'sjis': 'shift_jis',
+  'cp932': 'shift_jis',
+  'euc-kr': 'euc-kr',
+  'ksc5601': 'euc-kr',
+  'windows-1251': 'windows-1251',
+  'koi8-r': 'koi8-r',
+}));
+
+export function normalizeCharsetLabel(raw) {
+  if (!raw) return null;
+  const key = String(raw).trim().toLowerCase().replace(/["'\s]/g, '');
+  return CHARSET_ALIASES.get(key) || null;
+}
+
+export function decodeHtmlBytes(bytes, contentTypeHeader = '') {
+  const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || []);
+  if (buf.length >= 3 && buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) {
+    return { text: new TextDecoder('utf-8').decode(buf.subarray(3)), charset: 'utf-8' };
+  }
+  if (buf.length >= 2 && buf[0] === 0xFF && buf[1] === 0xFE) {
+    return { text: new TextDecoder('utf-16le').decode(buf.subarray(2)), charset: 'utf-16le' };
+  }
+  if (buf.length >= 2 && buf[0] === 0xFE && buf[1] === 0xFF) {
+    return { text: new TextDecoder('utf-16be').decode(buf.subarray(2)), charset: 'utf-16be' };
+  }
+
+  let charset = null;
+  const headerMatch = /charset\s*=\s*"?([A-Za-z0-9_\-.:]+)"?/i.exec(String(contentTypeHeader || ''));
+  if (headerMatch) charset = headerMatch[1];
+  if (!charset) {
+    const head = buf.subarray(0, Math.min(buf.length, 4096)).toString('latin1');
+    const metaMatch = /charset\s*=\s*["']?\s*([A-Za-z0-9_\-.:]+)/i.exec(head);
+    if (metaMatch) charset = metaMatch[1];
+  }
+
+  const label = normalizeCharsetLabel(charset);
+  if (!label) {
+    return { text: new TextDecoder('utf-8').decode(buf), charset: 'utf-8' };
+  }
+  try {
+    return { text: new TextDecoder(label).decode(buf), charset: label };
+  } catch {
+    return { text: new TextDecoder('utf-8').decode(buf), charset: 'utf-8' };
+  }
+}
+
+const NAMED_ENTITIES = {
+  nbsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'",
+  mdash: '—', ndash: '–', hellip: '...', rsquo: '’', lsquo: '‘',
+  rdquo: '”', ldquo: '“', copy: '\u00A9', reg: '\u00AE', trade: '\u2122',
+  laquo: '«', raquo: '»', times: '×', divide: '÷', cent: '¢',
+  pound: '£', euro: '€', yen: '¥', sect: '§', middot: '·',
+  bull: '•', deg: '°', plusmn: '±', frac12: '½', sup2: '²',
+  sup3: '³', micro: 'µ', para: '¶', eacute: 'é', egrave: 'è',
+  agrave: 'à', ccedil: 'ç', uuml: 'ü', ouml: 'ö', auml: 'ä', ntilde: 'ñ',
+};
+
+export function decodeEntities(str) {
+  if (!str || typeof str !== 'string' || !str.includes('&')) return str || '';
+  return str
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex) => {
+      try { return String.fromCodePoint(parseInt(hex, 16)); } catch { return ''; }
+    })
+    .replace(/&#(\d+);/g, (_, dec) => {
+      try { return String.fromCodePoint(Number(dec)); } catch { return ''; }
+    })
+    .replace(/&([a-z][a-z0-9]*);/gi, (match, name) => {
+      const value = NAMED_ENTITIES[name.toLowerCase()];
+      return value !== undefined ? value : match;
+    });
+}
+
+const ZERO_WIDTH_RE = /[\u200B-\u200D\uFEFF\u2060]/g;
+
+export function normalizeText(str) {
+  if (!str || typeof str !== 'string') return '';
+  return str
+    .normalize('NFC')
+    .replace(ZERO_WIDTH_RE, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+const CONTENT_HINT_RE = /(article|content|post|entry|story|body|main|blog|news|wiki|markdown|rich-?text|editor|mp-editor|prose|detail)/i;
+const NOISE_HINT_RE = /(nav|menu|header|footer|sidebar|aside|comment|promo|banner|ads?-|advert|social|share|related|recommend|breadcrumb|widget|tag-|category-list|pagination|subscribe|newsletter|cookie|popup|modal|searchbox|form|button|toolbar|copyright|disclaimer)/i;
+
+function extractAuthor(author) {
+  if (!author) return '';
+  if (typeof author === 'string') return author;
+  if (Array.isArray(author)) return author.map(extractAuthor).filter(Boolean).join(', ');
+  if (typeof author === 'object' && author.name) return String(author.name);
+  return '';
+}
+
+export function parseJsonLd($) {
+  const found = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      collectJsonLd(JSON.parse($(el).text() || '{}'), found);
+    } catch {}
+  });
+
+  let best = null;
+  for (const item of found) {
+    const body = typeof item.articleBody === 'string'
+      ? item.articleBody
+      : (Array.isArray(item.articleBody) ? item.articleBody.join('\n\n') : '');
+    const headline = typeof item.headline === 'string' ? item.headline : '';
+    if (!body && !headline) continue;
+    if (!best || body.length >= best.body.length) {
+      best = {
+        body,
+        headline,
+        datePublished: item.datePublished || null,
+        dateModified: item.dateModified || null,
+        author: extractAuthor(item.author),
+      };
+    }
+  }
+  return best;
+}
+
+function collectJsonLd(node, out) {
+  if (Array.isArray(node)) {
+    node.forEach(entry => collectJsonLd(entry, out));
+    return;
+  }
+  if (!node || typeof node !== 'object') return;
+  if (node['@graph']) {
+    collectJsonLd(node['@graph'], out);
+    return;
+  }
+  const type = String(node['@type'] || '').toLowerCase();
+  if (/article|blogposting|newsarticle|webpage|discussion|report|answer/.test(type) || node.articleBody) {
+    out.push(node);
+  }
+}
+
+const MAX_BLOCKS = 500;
+const MAX_MARKDOWN_CHARS = 120_000;
+
+export async function extractMainContent(html, { url = '' } = {}) {
+  const htmlChars = String(html || '').length;
+  const cheerioLoad = await loadCheerio();
+
+  if (!cheerioLoad) {
+    const text = normalizeText(decodeEntities(
+      String(html || '')
+        .replace(/<script\b[\s\S]*?<\/script>/gi, ' ')
+        .replace(/<style\b[\s\S]*?<\/style>/gi, ' ')
+        .replace(/<[^>]+>/g, ' ')
+    ));
+    return buildExtractionResult({ markdown: text, meta: {}, headings: [], paragraphs: [], links: [], images: [], jsonLd: null, url, htmlChars });
+  }
+
+  const $ = cheerioLoad(html);
+  $('script:not([type="application/ld+json"]), style, noscript, iframe, svg, template, form, button').remove();
+
+  const meta = {};
+  meta.title = $('title').first().text().trim() || '';
+  meta.description = $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '';
+  meta.author = $('meta[name="author"]').attr('content') || $('meta[property="article:author"]').attr('content') || '';
+  meta.publishedAt = $('meta[itemprop="datePublished"]').attr('content') || $('meta[property="article:published_time"]').attr('content') || $('meta[name="date"]').attr('content') || '';
+  meta.keywords = $('meta[name="keywords"]').attr('content') || '';
+  meta.ogTitle = $('meta[property="og:title"]').attr('content') || '';
+  meta.ogImage = $('meta[property="og:image"]').attr('content') || '';
+  meta.ogType = $('meta[property="og:type"]').attr('content') || '';
+  meta.canonical = $('link[rel="canonical"]').attr('href') || '';
+  meta.favicon = $('link[rel="icon"], link[rel="shortcut icon"]').attr('href') || '';
+  meta.lang = $('html').attr('lang') || '';
+
+  const jsonLd = parseJsonLd($);
+  const container = pickMainContainer($);
+
+  const blocks = collectStructuredBlocks($, container);
+  let markdown = joinBlocks(blocks);
+  let paragraphs = blocks.filter(b => b.type === 'p' && b.text.length > 20).map(b => b.text).slice(0, 30);
+  let headings = blocks
+    .filter(b => b.type === 'heading')
+    .slice(0, 12)
+    .map(b => ({ level: b.level, text: b.text }));
+
+  if (paragraphs.length === 0) {
+    $('p').each((i, el) => {
+      if (paragraphs.length >= 15) return;
+      const txt = normalizeText(decodeEntities($(el).text()));
+      if (txt && txt.length > 30) paragraphs.push(txt);
+    });
+    if (markdown.length < 100 && paragraphs.length > 0) {
+      markdown = paragraphs.join('\n\n');
+    }
+  }
+  if (headings.length === 0) {
+    $('h1, h2, h3').each((i, el) => {
+      if (headings.length >= 10) return;
+      const txt = normalizeText($(el).text());
+      if (txt) headings.push({ level: el.tagName, text: txt });
+    });
+  }
+
+  const scope = container ? $(container) : $.root();
+  const links = [];
+  scope.find('a').each((i, el) => {
+    if (links.length >= 15) return;
+    const href = $(el).attr('href');
+    const text = normalizeText($(el).text());
+    if (href && text && text.length > 2) {
+      let absoluteUrl = href;
+      try { absoluteUrl = new URL(href, url || undefined).href; } catch {}
+      links.push({ text: text.slice(0, 80), url: absoluteUrl });
+    }
+  });
+
+  const images = [];
+  scope.find('img').each((i, el) => {
+    if (images.length >= 8) return;
+    let rawSrc = $(el).attr('data-src') || $(el).attr('src') || $(el).attr('data-original') || '';
+    const alt = $(el).attr('alt') || $(el).attr('title') || '';
+    if (rawSrc) {
+      if (rawSrc.startsWith('//')) rawSrc = `https:${rawSrc}`;
+      let absoluteUrl = rawSrc;
+      try { absoluteUrl = new URL(rawSrc, url || undefined).href; } catch {}
+      images.push({ src: absoluteUrl, alt: alt.slice(0, 100) });
+    }
+  });
+
+  return buildExtractionResult({ markdown, meta, headings, paragraphs, links, images, jsonLd, url, htmlChars });
+}
+
+function buildExtractionResult({ markdown, meta, headings, paragraphs, links, images, jsonLd, url, htmlChars }) {
+  const textChars = markdown.length;
+  const density = htmlChars > 0 ? Number((textChars / htmlChars).toFixed(4)) : 0;
+  const isThin = (textChars < 400 && htmlChars > 15_000) || textChars < 200;
+  return {
+    markdown,
+    meta: Object.fromEntries(Object.entries(meta || {}).filter(([, v]) => v)),
+    headings: headings || [],
+    paragraphs: paragraphs || [],
+    links: links || [],
+    images: images || [],
+    jsonLd: jsonLd || null,
+    quality: { textChars, htmlChars, density, isThin },
+  };
+}
+
+function pickMainContainer($) {
+  const scored = [];
+  $('article, main, [role="main"], div, section').each((i, el) => {
+    if (scored.length >= 150) return;
+    const $el = $(el);
+    const attr = `${$el.attr('class') || ''} ${$el.attr('id') || ''} ${$el.attr('role') || ''}`;
+    if (NOISE_HINT_RE.test(attr)) return;
+    const text = $el.text().replace(/\s+/g, ' ').trim();
+    if (text.length < 120) return;
+
+    const anchorTextLen = $el.find('a').text().replace(/\s+/g, ' ').trim().length;
+    const linkDensity = text.length ? anchorTextLen / text.length : 0;
+    const paragraphCount = $el.find('p').filter((_, p) => $(p).text().trim().length > 40).length;
+    const punctuationPer100 = (text.match(/[.!?。！？,、;:]/g) || []).length / Math.max(1, text.length / 100);
+
+    let score = Math.min(text.length, 6000) / 100;
+    score += Math.min(paragraphCount, 30) * 8;
+    score += Math.min(60, punctuationPer100 * 2);
+    score *= 1 - Math.min(1, linkDensity) * 0.9;
+    if (CONTENT_HINT_RE.test(attr)) score *= 1.35;
+    if (el.tagName === 'article' || el.tagName === 'main') score *= 1.5;
+    scored.push({ el, score, textLen: text.length });
+  });
+
+  scored.sort((a, b) => b.score - a.score);
+  if (scored.length > 0 && scored[0].textLen >= 200) {
+    return scored[0].el;
+  }
+  return $('article').first().get(0) || $('main').first().get(0) || $('body').first().get(0) || null;
+}
+
+const HANDLED_TAGS = new Set(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'li', 'table', 'pre', 'blockquote']);
+
+function collectStructuredBlocks($, container) {
+  const blocks = [];
+  if (!container) return blocks;
+  const $container = $(container);
+
+  const isInsideHandledTag = $el => {
+    const parents = $el.parents().toArray();
+    for (const ancestor of parents) {
+      if (ancestor === container) return false;
+      const tag = String(ancestor.tagName || '').toLowerCase();
+      if (HANDLED_TAGS.has(tag)) return true;
+    }
+    return true;
+  };
+
+  $container.find('h1,h2,h3,h4,h5,h6,p,li,table,pre,blockquote').each((_, el) => {
+    if (blocks.length >= MAX_BLOCKS) return;
+    const tag = String(el.tagName || '').toLowerCase();
+    if (!HANDLED_TAGS.has(tag)) return;
+    const $el = $(el);
+    if (isInsideHandledTag($el)) return;
+
+    if (tag === 'table') {
+      const rows = [];
+      $el.find('tr').each((__, tr) => {
+        if (rows.length >= 40) return;
+        const cells = [];
+        $(tr).find('th,td').each((___, cell) => {
+          if (cells.length >= 12) return;
+          const text = normalizeText(decodeEntities($(cell).text())).slice(0, 80);
+          cells.push((text || ' ').replace(/\|/g, '\\|'));
+        });
+        if (cells.length > 0) rows.push(`| ${cells.join(' | ')} |`);
+      });
+      if (rows.length >= 2) blocks.push({ type: 'table', text: rows.join('\n') });
+      return;
+    }
+
+    if (tag === 'pre') {
+      const raw = decodeEntities($el.text()).replace(/\s+$/, '').slice(0, 2000);
+      if (raw) blocks.push({ type: 'pre', text: `---\n${raw}\n---` });
+      return;
+    }
+
+    const text = normalizeText(decodeEntities($el.text()));
+    if (!text) return;
+
+    if (tag !== 'table' && tag !== 'pre') {
+      const anchorTextLen = $el.find('a').text().replace(/\s+/g, ' ').trim().length;
+      if (text.length < 120 && anchorTextLen > text.length * 0.5) return;
+    }
+
+    if (tag === 'li') {
+      blocks.push({ type: 'li', text: `- ${text}` });
+    } else if (tag === 'blockquote') {
+      blocks.push({ type: 'p', text: `> ${text}` });
+    } else if (tag.startsWith('h')) {
+      const level = Number(tag[1]) || 2;
+      const hashes = '#'.repeat(level <= 2 ? 2 : Math.min(level + 0, 4));
+      blocks.push({ type: 'heading', level: tag, text: `${hashes} ${text}` });
+    } else {
+      blocks.push({ type: 'p', text });
+    }
+  });
+
+  return blocks;
+}
+
+function joinBlocks(blocks) {
+  let markdown = '';
+  let prevType = null;
+  for (const block of blocks) {
+    const separator = !markdown ? '' : (block.type === 'li' && prevType === 'li' ? '\n' : '\n\n');
+    markdown += separator + block.text;
+    prevType = block.type;
+    if (markdown.length >= MAX_MARKDOWN_CHARS) break;
+  }
+  return markdown;
+}
+
+const _segmenters = new Map();
+
+function getSegmenter(lang) {
+  if (typeof Intl === 'undefined' || !Intl.Segmenter) return null;
+  const key = lang || 'en';
+  if (!_segmenters.has(key)) {
+    try {
+      _segmenters.set(key, new Intl.Segmenter(key, { granularity: 'word' }));
+    } catch {
+      try {
+        _segmenters.set(key, new Intl.Segmenter('en', { granularity: 'word' }));
+      } catch {
+        _segmenters.set(key, null);
+      }
+    }
+  }
+  return _segmenters.get(key);
+}
+
+const THAI_RUN_RE = /[\u0E00-\u0E7F]+/g;
+const CJK_RUN_RE = /[\u3040-\u30FF\u3400-\u9FFF\uAC00-\uD7AF]+/g;
+const LATIN_WORD_RE = /[a-z0-9à-öø-ÿāáǎàēéěèīíǐìōóǒòūúǔùǖǘǚǜ]{2,}/g;
+
+export function truncateByWords(text, maxChars = 8000, langHint = 'en') {
+  const str = String(text || '');
+  if (str.length <= maxChars) return str;
+  const slice = str.slice(0, maxChars);
+  const segmenter = getSegmenter(langHint);
+  if (!segmenter) return slice;
+  try {
+    let cut = -1;
+    const boundaryTarget = Math.floor(maxChars * 0.85);
+    for (const segment of segmenter.segment(slice)) {
+      if (segment.index > boundaryTarget) break;
+      cut = segment.index + segment.segment.length;
+    }
+    return cut > 0 ? slice.slice(0, cut) : slice;
+  } catch {
+    return slice;
+  }
+}
+
+export function tokenizeForMatch(text, langHint) {
+  const normalized = normalizeText(String(text || '')).toLowerCase();
+  if (!normalized) return [];
+
+  const thaiRuns = normalized.match(THAI_RUN_RE);
+  if (thaiRuns) {
+    const segmenter = getSegmenter('th');
+    if (segmenter) {
+      const words = [];
+      try {
+        for (const segment of segmenter.segment(normalized)) {
+          if (segment.isWordLike && segment.segment.trim()) words.push(segment.segment);
+        }
+      } catch {}
+      if (words.length > 0) return words;
+    }
+    return thaiRuns;
+  }
+
+  const cjkRuns = normalized.match(CJK_RUN_RE);
+  if (cjkRuns) {
+    const grams = [];
+    for (const run of cjkRuns) {
+      if (run.length === 1) {
+        grams.push(run);
+        continue;
+      }
+      for (let i = 0; i < run.length - 1; i++) grams.push(run.slice(i, i + 2));
+    }
+    const latin = normalized.match(/[a-z0-9]{2,}/g) || [];
+    return [...grams, ...latin];
+  }
+
+  return normalized.match(LATIN_WORD_RE) || [];
+}
+
+export function scorePassages(passages, query) {
+  const queryTokens = [...new Set(tokenizeForMatch(query || ''))];
+  return (Array.isArray(passages) ? passages : []).map(passage => {
+    if (!queryTokens.length) return { passage, score: 0 };
+    const tokens = tokenizeForMatch(passage.text || '');
+    if (!tokens.length) return { passage, score: 0 };
+    const tokenSet = new Set(tokens);
+    let covered = 0;
+    for (const token of queryTokens) {
+      if (tokenSet.has(token)) covered++;
+    }
+    const coverage = covered / queryTokens.length;
+    const density = covered / Math.max(8, Math.min(400, tokens.length));
+    const headingTokens = passage.heading ? new Set(tokenizeForMatch(passage.heading)) : null;
+    const headingBonus = headingTokens && queryTokens.some(t => headingTokens.has(t)) ? 0.15 : 0;
+    return { passage, score: Number((coverage * 0.7 + Math.min(1, density) * 0.2 + headingBonus).toFixed(4)) };
+  });
+}
+
+export function splitIntoPassages(markdown, { maxChars = 1600 } = {}) {
+  const limit = Math.max(200, maxChars);
+  const lines = String(markdown || '').split('\n');
+  const passages = [];
+  let heading = '';
+  let buffer = [];
+
+  const flush = () => {
+    const body = buffer.join('\n').trim();
+    if (body) {
+      passages.push({ heading, text: heading ? `${heading}\n${body}` : body });
+    }
+    buffer = [];
+  };
+
+  for (const line of lines) {
+    if (/^#{1,6}\s+\S/.test(line)) {
+      flush();
+      heading = line.trim();
+      continue;
+    }
+    buffer.push(line);
+    if (buffer.join('\n').length >= limit) flush();
+  }
+  flush();
+
+  if (passages.length === 0 && String(markdown || '').trim()) {
+    passages.push({ heading: '', text: String(markdown).trim() });
+  }
+  return passages.map((p, index) => ({ ...p, index }));
+}
+
+function fnv1a64(str) {
+  let hash = 0xcbf29ce484222325n;
+  for (let i = 0; i < str.length; i++) {
+    hash ^= BigInt(str.charCodeAt(i));
+    hash = (hash * 0x100000001b3n) & 0xffffffffffffffffn;
+  }
+  return hash;
+}
+
+export function simhashText(text) {
+  const tokens = tokenizeForMatch(text);
+  if (tokens.length < 3) {
+    return tokens.length ? fnv1a64(tokens.join(' ')) : 0n;
+  }
+  const bitWeights = new Array(64).fill(0);
+  for (let i = 0; i + 3 <= tokens.length; i++) {
+    const hash = fnv1a64(tokens.slice(i, i + 3).join(' '));
+    for (let bit = 0; bit < 64; bit++) {
+      bitWeights[bit] += (hash >> BigInt(bit)) & 1n ? 1 : -1;
+    }
+  }
+  let fingerprint = 0n;
+  for (let bit = 0; bit < 64; bit++) {
+    if (bitWeights[bit] > 0) fingerprint |= 1n << BigInt(bit);
+  }
+  return fingerprint;
+}
+
+export function hamming64(a, b) {
+  let xor = (a ^ b) & 0xffffffffffffffffn;
+  let count = 0;
+  while (xor) {
+    if (xor & 1n) count++;
+    xor >>= 1n;
+  }
+  return count;
+}
+
+export function filterUniquePassages(passages, state = { hashes: [] }, { maxHamming = 6 } = {}) {
+  const hashes = Array.isArray(state?.hashes) ? state.hashes : [];
+  return (Array.isArray(passages) ? passages : []).filter(passage => {
+    const fingerprint = simhashText(passage.text || '');
+    for (const kept of hashes) {
+      if (hamming64(kept, fingerprint) <= maxHamming) return false;
+    }
+    hashes.push(fingerprint);
+    return true;
+  });
+}
+
+export function dedupePassages(passages, { maxHamming = 6 } = {}) {
+  return filterUniquePassages(passages, { hashes: [] }, { maxHamming });
+}
+
+export function selectPassages(passages, query, { budgetChars = 8000, maxHamming = 6 } = {}) {
+  const total = Array.isArray(passages) ? passages.length : 0;
+  if (total === 0) return { text: '', selected: 0, total: 0 };
+  const budget = Number.isFinite(Number(budgetChars)) && Number(budgetChars) > 0 ? Number(budgetChars) : 8000;
+  const scored = scorePassages(dedupePassages(passages, { maxHamming }), query || '');
+  const ranked = [...scored].sort((a, b) => b.score - a.score);
+
+  const kept = [];
+  let used = 0;
+  for (const entry of ranked) {
+    const cost = entry.passage.text.length + 2;
+    if (kept.length > 0 && used + cost > budget) continue;
+    kept.push(entry);
+    used += cost;
+    if (used >= budget) break;
+  }
+  kept.sort((a, b) => a.passage.index - b.passage.index);
+  return {
+    text: kept.map(entry => entry.passage.text).join('\n\n'),
+    selected: kept.length,
+    total,
+  };
+}

package/server/agent/websearch/index.js

@@ -0,0 +1,43 @@
+import { MetaSearchOrchestrator, defaultMetaSearch } from './orchestrator.js';
+import { fuseSearchResults } from './core/fusion.js';
+import { canonicalizeUrl } from './core/canonical.js';
+import { createSearchResult } from './core/types.js';
+import { searchDuckDuckGo } from './adapters/duckduckgo.js';
+import { searchSearXNG } from './adapters/searxng.js';
+import { searchJina } from './adapters/jina.js';
+import { TokenBucketLimiter, defaultLimiters } from './resilience/limiter.js';
+import { CircuitBreaker, defaultBreakers } from './resilience/breaker.js';
+import { SearchResponseCache, defaultSearchCache } from './resilience/cache.js';
+import { safeFetchWithPinning, assertSafeUrlHostname, isPrivateIPv4, isPrivateIPv6 } from './security/ssrf.js';
+import { DomainFilter, defaultDomainFilter } from './security/filter.js';
+import { SearchTelemetry, defaultTelemetry } from './telemetry.js';
+
+
+export async function hybridWebSearch(query, options = {}) {
+  return defaultMetaSearch.executeSearch(query, options);
+}
+
+export {
+  MetaSearchOrchestrator,
+  defaultMetaSearch,
+  fuseSearchResults,
+  canonicalizeUrl,
+  createSearchResult,
+  searchDuckDuckGo,
+  searchSearXNG,
+  searchJina,
+  TokenBucketLimiter,
+  defaultLimiters,
+  CircuitBreaker,
+  defaultBreakers,
+  SearchResponseCache,
+  defaultSearchCache,
+  safeFetchWithPinning,
+  assertSafeUrlHostname,
+  isPrivateIPv4,
+  isPrivateIPv6,
+  DomainFilter,
+  defaultDomainFilter,
+  SearchTelemetry,
+  defaultTelemetry
+};