@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/agent/planner.js

@@ -0,0 +1,271 @@
+import { createLogger } from "../logger.js";
+import {
+  requestSampling,
+  checkSamplingSupport
+} from "../search/sampling.js";
+import { preFilterCandidates } from "./relevance.js";
+const log = createLogger("agent");
+const MAX_SEARCH_QUERIES = 5;
+const MAX_QUERY_LENGTH = 250;
+function applyUrlFilter(urls) {
+  const filtered = preFilterCandidates(urls.map((url) => ({ url })));
+  const kept = filtered.kept.map((k) => k.url);
+  const excluded = filtered.excluded.map((e) => ({ url: e.item.url, reason: e.reason }));
+  if (excluded.length > 0) {
+    log.info("agent pre-filter", { kept: kept.length, excluded: excluded.length });
+  }
+  return { kept, excluded };
+}
+async function planExecution(prompt, urls, server) {
+  const trimmedPrompt = prompt.trim();
+  if (!trimmedPrompt) {
+    const { kept: kept2, excluded: excluded2 } = applyUrlFilter(urls ?? []);
+    return {
+      searches: [],
+      urls: kept2,
+      notes: "Empty prompt provided",
+      samplingUsed: false,
+      excluded_urls: excluded2
+    };
+  }
+  if (server) {
+    try {
+      const samplingResult = await planWithSampling(trimmedPrompt, server);
+      if (samplingResult) {
+        const mergedUrls2 = mergeUrls(samplingResult.urls, urls ?? []);
+        const { kept: kept2, excluded: excluded2 } = applyUrlFilter(mergedUrls2);
+        return {
+          searches: samplingResult.searches,
+          urls: kept2,
+          notes: samplingResult.notes,
+          samplingUsed: true,
+          excluded_urls: excluded2
+        };
+      }
+    } catch (err) {
+      log.warn("sampling planning failed, using fallback", {
+        error: err instanceof Error ? err.message : String(err)
+      });
+    }
+  }
+  const fallback = planWithFallback(trimmedPrompt);
+  const mergedUrls = mergeUrls(fallback.urls, urls ?? []);
+  const { kept, excluded } = applyUrlFilter(mergedUrls);
+  return {
+    searches: fallback.searches,
+    urls: kept,
+    notes: fallback.notes,
+    samplingUsed: false,
+    excluded_urls: excluded
+  };
+}
+async function planWithSampling(prompt, server) {
+  try {
+    if (!checkSamplingSupport(server)) {
+      log.debug("client does not support sampling for planning");
+      return null;
+    }
+    const samplingPrompt = `You are a data gathering assistant. Given the user's prompt, create an execution plan.
+
+Return a JSON object with:
+- "searches": array of 1-5 search engine queries to find the needed data
+- "urls": array of specific URLs to visit (if any are obvious from the prompt)
+- "notes": brief string with any relevant observations
+
+Prompt: ${prompt}
+
+Respond with ONLY valid JSON: {"searches": [...], "urls": [...], "notes": "..."}`;
+    const response = await requestSampling(
+      server,
+      [{ role: "user", content: { type: "text", text: samplingPrompt } }],
+      500
+    );
+    if (!response?.content?.text) {
+      log.debug("sampling returned empty response for planning");
+      return null;
+    }
+    const text = response.content.text.trim();
+    let parsed;
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      const jsonMatch = text.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        try {
+          parsed = JSON.parse(jsonMatch[0]);
+        } catch {
+          log.debug("could not extract JSON from sampling plan response");
+          return null;
+        }
+      } else {
+        return null;
+      }
+    }
+    if (typeof parsed !== "object" || parsed === null) return null;
+    const obj = parsed;
+    const searches = Array.isArray(obj.searches) ? obj.searches.filter((s) => typeof s === "string" && s.trim().length > 0).map((s) => s.trim()) : [];
+    const extractedUrls = Array.isArray(obj.urls) ? obj.urls.filter((u) => typeof u === "string" && isValidUrl(u)) : [];
+    const notes = typeof obj.notes === "string" ? obj.notes : "";
+    if (searches.length === 0 && extractedUrls.length === 0) {
+      log.debug("sampling plan had no searches or URLs");
+      return null;
+    }
+    return { searches: searches.slice(0, MAX_SEARCH_QUERIES), urls: extractedUrls, notes };
+  } catch (err) {
+    log.debug("sampling planning request failed", {
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return null;
+  }
+}
+function planWithFallback(prompt) {
+  const extractedUrls = extractUrlsFromText(prompt);
+  const searches = generateSearchQueries(prompt);
+  return {
+    searches,
+    urls: extractedUrls,
+    notes: "Fallback plan from keyword extraction"
+  };
+}
+function extractUrlsFromText(text) {
+  const urlRegex = /https?:\/\/[^\s,;"')\]]+/gi;
+  const matches = text.match(urlRegex) ?? [];
+  const cleaned = matches.map((url) => url.replace(/[.,;:!?)]+$/, ""));
+  return [...new Set(cleaned)].filter(isValidUrl);
+}
+function isValidUrl(url) {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function generateSearchQueries(prompt) {
+  const stopWords = /* @__PURE__ */ new Set([
+    "the",
+    "a",
+    "an",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "could",
+    "should",
+    "may",
+    "might",
+    "can",
+    "shall",
+    "to",
+    "of",
+    "in",
+    "for",
+    "on",
+    "with",
+    "at",
+    "by",
+    "from",
+    "as",
+    "into",
+    "through",
+    "and",
+    "but",
+    "or",
+    "not",
+    "so",
+    "yet",
+    "find",
+    "get",
+    "search",
+    "look",
+    "show",
+    "me",
+    "i",
+    "we",
+    "you",
+    "they",
+    "it",
+    "this",
+    "that",
+    "my",
+    "your",
+    "our",
+    "their",
+    "all",
+    "each",
+    "every",
+    "some",
+    "any",
+    "also",
+    "about",
+    "up",
+    "out",
+    "if",
+    "then",
+    "than",
+    "too",
+    "very",
+    "just",
+    "please",
+    "need",
+    "want"
+  ]);
+  const urlFree = prompt.replace(/https?:\/\/[^\s]+/g, "").trim();
+  if (!urlFree) return [];
+  const words = urlFree.replace(/[?!.,;:'"()\[\]{}]/g, " ").split(/\s+/).filter(Boolean);
+  const contentWords = words.filter((w) => !stopWords.has(w.toLowerCase()) && w.length > 1);
+  if (contentWords.length === 0) return [];
+  const queries = [];
+  const fullQuery = contentWords.join(" ");
+  if (fullQuery.length <= MAX_QUERY_LENGTH) {
+    queries.push(fullQuery);
+  } else {
+    queries.push(fullQuery.slice(0, MAX_QUERY_LENGTH));
+  }
+  if (contentWords.length >= 4) {
+    const half = Math.ceil(contentWords.length / 2);
+    queries.push(contentWords.slice(0, half).join(" "));
+  }
+  if (contentWords.length >= 4) {
+    const half = Math.ceil(contentWords.length / 2);
+    queries.push(contentWords.slice(half).join(" "));
+  }
+  if (contentWords.length >= 3) {
+    const bigrams = [];
+    for (let i = 0; i < contentWords.length - 1 && bigrams.length < 3; i++) {
+      bigrams.push(`${contentWords[i]} ${contentWords[i + 1]}`);
+    }
+    if (bigrams.length > 0) {
+      queries.push(bigrams.join(" "));
+    }
+  }
+  const keyTerms = contentWords.filter(
+    (w) => w.length > 4 || w[0] === w[0].toUpperCase()
+  );
+  if (keyTerms.length >= 2) {
+    queries.push(keyTerms.slice(0, 5).join(" "));
+  }
+  const unique = [...new Set(queries)].map(
+    (q) => q.length > MAX_QUERY_LENGTH ? q.slice(0, MAX_QUERY_LENGTH) : q
+  );
+  return unique.slice(0, MAX_SEARCH_QUERIES);
+}
+function mergeUrls(samplingUrls, explicitUrls) {
+  const all = [...samplingUrls, ...explicitUrls];
+  return [...new Set(all)];
+}
+export {
+  planExecution
+};
+//# sourceMappingURL=planner.js.map

package/dist/agent/planner.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/agent/planner.ts"],"sourcesContent":["import { createLogger } from '../logger.js';\nimport {\n  type SamplingCapableServer,\n  requestSampling,\n  checkSamplingSupport,\n} from '../search/sampling.js';\nimport { preFilterCandidates } from './relevance.js';\n\nconst log = createLogger('agent');\n\nconst MAX_SEARCH_QUERIES = 5;\nconst MAX_QUERY_LENGTH = 250;\n\nexport interface AgentPlan {\n  searches: string[];\n  urls: string[];\n  notes: string;\n  samplingUsed: boolean;\n  excluded_urls?: { url: string; reason: 'invalid_url' | 'blocklisted_domain' }[];\n}\n\nfunction applyUrlFilter(urls: string[]): {\n  kept: string[];\n  excluded: { url: string; reason: 'invalid_url' | 'blocklisted_domain' }[];\n} {\n  const filtered = preFilterCandidates(urls.map((url) => ({ url })));\n  const kept = filtered.kept.map((k) => k.url);\n  const excluded = filtered.excluded.map((e) => ({ url: e.item.url, reason: e.reason }));\n  if (excluded.length > 0) {\n    log.info('agent pre-filter', { kept: kept.length, excluded: excluded.length });\n  }\n  return { kept, excluded };\n}\n\nexport async function planExecution(\n  prompt: string,\n  urls?: string[],\n  server?: SamplingCapableServer,\n): Promise<AgentPlan> {\n  const trimmedPrompt = prompt.trim();\n\n  if (!trimmedPrompt) {\n    const { kept, excluded } = applyUrlFilter(urls ?? []);\n    return {\n      searches: [],\n      urls: kept,\n      notes: 'Empty prompt provided',\n      samplingUsed: false,\n      excluded_urls: excluded,\n    };\n  }\n\n  if (server) {\n    try {\n      const samplingResult = await planWithSampling(trimmedPrompt, server);\n      if (samplingResult) {\n        const mergedUrls = mergeUrls(samplingResult.urls, urls ?? []);\n        const { kept, excluded } = applyUrlFilter(mergedUrls);\n        return {\n          searches: samplingResult.searches,\n          urls: kept,\n          notes: samplingResult.notes,\n          samplingUsed: true,\n          excluded_urls: excluded,\n        };\n      }\n    } catch (err) {\n      log.warn('sampling planning failed, using fallback', {\n        error: err instanceof Error ? err.message : String(err),\n      });\n    }\n  }\n\n  const fallback = planWithFallback(trimmedPrompt);\n  const mergedUrls = mergeUrls(fallback.urls, urls ?? []);\n  const { kept, excluded } = applyUrlFilter(mergedUrls);\n\n  return {\n    searches: fallback.searches,\n    urls: kept,\n    notes: fallback.notes,\n    samplingUsed: false,\n    excluded_urls: excluded,\n  };\n}\n\nasync function planWithSampling(\n  prompt: string,\n  server: SamplingCapableServer,\n): Promise<{ searches: string[]; urls: string[]; notes: string } | null> {\n  try {\n    if (!checkSamplingSupport(server)) {\n      log.debug('client does not support sampling for planning');\n      return null;\n    }\n\n    const samplingPrompt = `You are a data gathering assistant. Given the user's prompt, create an execution plan.\n\nReturn a JSON object with:\n- \"searches\": array of 1-5 search engine queries to find the needed data\n- \"urls\": array of specific URLs to visit (if any are obvious from the prompt)\n- \"notes\": brief string with any relevant observations\n\nPrompt: ${prompt}\n\nRespond with ONLY valid JSON: {\"searches\": [...], \"urls\": [...], \"notes\": \"...\"}`;\n\n    const response = await requestSampling(\n      server,\n      [{ role: 'user', content: { type: 'text', text: samplingPrompt } }],\n      500,\n    );\n\n    if (!response?.content?.text) {\n      log.debug('sampling returned empty response for planning');\n      return null;\n    }\n\n    const text = response.content.text.trim();\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(text);\n    } catch {\n      const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n      if (jsonMatch) {\n        try {\n          parsed = JSON.parse(jsonMatch[0]);\n        } catch {\n          log.debug('could not extract JSON from sampling plan response');\n          return null;\n        }\n      } else {\n        return null;\n      }\n    }\n\n    if (typeof parsed !== 'object' || parsed === null) return null;\n\n    const obj = parsed as Record<string, unknown>;\n    const searches = Array.isArray(obj.searches)\n      ? obj.searches.filter((s): s is string => typeof s === 'string' && s.trim().length > 0).map((s) => s.trim())\n      : [];\n    const extractedUrls = Array.isArray(obj.urls)\n      ? obj.urls.filter((u): u is string => typeof u === 'string' && isValidUrl(u))\n      : [];\n    const notes = typeof obj.notes === 'string' ? obj.notes : '';\n\n    if (searches.length === 0 && extractedUrls.length === 0) {\n      log.debug('sampling plan had no searches or URLs');\n      return null;\n    }\n\n    return { searches: searches.slice(0, MAX_SEARCH_QUERIES), urls: extractedUrls, notes };\n  } catch (err) {\n    log.debug('sampling planning request failed', {\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return null;\n  }\n}\n\nfunction planWithFallback(prompt: string): { searches: string[]; urls: string[]; notes: string } {\n  const extractedUrls = extractUrlsFromText(prompt);\n  const searches = generateSearchQueries(prompt);\n\n  return {\n    searches,\n    urls: extractedUrls,\n    notes: 'Fallback plan from keyword extraction',\n  };\n}\n\nfunction extractUrlsFromText(text: string): string[] {\n  const urlRegex = /https?:\\/\\/[^\\s,;\"')\\]]+/gi;\n  const matches = text.match(urlRegex) ?? [];\n  const cleaned = matches.map((url) => url.replace(/[.,;:!?)]+$/, ''));\n  return [...new Set(cleaned)].filter(isValidUrl);\n}\n\nfunction isValidUrl(url: string): boolean {\n  try {\n    new URL(url);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nfunction generateSearchQueries(prompt: string): string[] {\n  const stopWords = new Set([\n    'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',\n    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',\n    'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',\n    'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'and',\n    'but', 'or', 'not', 'so', 'yet', 'find', 'get', 'search', 'look',\n    'show', 'me', 'i', 'we', 'you', 'they', 'it', 'this', 'that',\n    'my', 'your', 'our', 'their', 'all', 'each', 'every', 'some',\n    'any', 'also', 'about', 'up', 'out', 'if', 'then', 'than',\n    'too', 'very', 'just', 'please', 'need', 'want',\n  ]);\n\n  const urlFree = prompt.replace(/https?:\\/\\/[^\\s]+/g, '').trim();\n  if (!urlFree) return [];\n\n  const words = urlFree\n    .replace(/[?!.,;:'\"()\\[\\]{}]/g, ' ')\n    .split(/\\s+/)\n    .filter(Boolean);\n\n  const contentWords = words.filter((w) => !stopWords.has(w.toLowerCase()) && w.length > 1);\n\n  if (contentWords.length === 0) return [];\n\n  const queries: string[] = [];\n\n  const fullQuery = contentWords.join(' ');\n  if (fullQuery.length <= MAX_QUERY_LENGTH) {\n    queries.push(fullQuery);\n  } else {\n    queries.push(fullQuery.slice(0, MAX_QUERY_LENGTH));\n  }\n\n  if (contentWords.length >= 4) {\n    const half = Math.ceil(contentWords.length / 2);\n    queries.push(contentWords.slice(0, half).join(' '));\n  }\n\n  if (contentWords.length >= 4) {\n    const half = Math.ceil(contentWords.length / 2);\n    queries.push(contentWords.slice(half).join(' '));\n  }\n\n  if (contentWords.length >= 3) {\n    const bigrams: string[] = [];\n    for (let i = 0; i < contentWords.length - 1 && bigrams.length < 3; i++) {\n      bigrams.push(`${contentWords[i]} ${contentWords[i + 1]}`);\n    }\n    if (bigrams.length > 0) {\n      queries.push(bigrams.join(' '));\n    }\n  }\n\n  const keyTerms = contentWords.filter(\n    (w) => w.length > 4 || w[0] === w[0].toUpperCase(),\n  );\n  if (keyTerms.length >= 2) {\n    queries.push(keyTerms.slice(0, 5).join(' '));\n  }\n\n  const unique = [...new Set(queries)].map((q) =>\n    q.length > MAX_QUERY_LENGTH ? q.slice(0, MAX_QUERY_LENGTH) : q,\n  );\n  return unique.slice(0, MAX_SEARCH_QUERIES);\n}\n\nfunction mergeUrls(samplingUrls: string[], explicitUrls: string[]): string[] {\n  const all = [...samplingUrls, ...explicitUrls];\n  return [...new Set(all)];\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAC7B;AAAA,EAEE;AAAA,EACA;AAAA,OACK;AACP,SAAS,2BAA2B;AAEpC,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,qBAAqB;AAC3B,MAAM,mBAAmB;AAUzB,SAAS,eAAe,MAGtB;AACA,QAAM,WAAW,oBAAoB,KAAK,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC;AACjE,QAAM,OAAO,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,GAAG;AAC3C,QAAM,WAAW,SAAS,SAAS,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,QAAQ,EAAE,OAAO,EAAE;AACrF,MAAI,SAAS,SAAS,GAAG;AACvB,QAAI,KAAK,oBAAoB,EAAE,MAAM,KAAK,QAAQ,UAAU,SAAS,OAAO,CAAC;AAAA,EAC/E;AACA,SAAO,EAAE,MAAM,SAAS;AAC1B;AAEA,eAAsB,cACpB,QACA,MACA,QACoB;AACpB,QAAM,gBAAgB,OAAO,KAAK;AAElC,MAAI,CAAC,eAAe;AAClB,UAAM,EAAE,MAAAA,OAAM,UAAAC,UAAS,IAAI,eAAe,QAAQ,CAAC,CAAC;AACpD,WAAO;AAAA,MACL,UAAU,CAAC;AAAA,MACX,MAAMD;AAAA,MACN,OAAO;AAAA,MACP,cAAc;AAAA,MACd,eAAeC;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,QAAQ;AACV,QAAI;AACF,YAAM,iBAAiB,MAAM,iBAAiB,eAAe,MAAM;AACnE,UAAI,gBAAgB;AAClB,cAAMC,cAAa,UAAU,eAAe,MAAM,QAAQ,CAAC,CAAC;AAC5D,cAAM,EAAE,MAAAF,OAAM,UAAAC,UAAS,IAAI,eAAeC,WAAU;AACpD,eAAO;AAAA,UACL,UAAU,eAAe;AAAA,UACzB,MAAMF;AAAA,UACN,OAAO,eAAe;AAAA,UACtB,cAAc;AAAA,UACd,eAAeC;AAAA,QACjB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,KAAK,4CAA4C;AAAA,QACnD,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,WAAW,iBAAiB,aAAa;AAC/C,QAAM,aAAa,UAAU,SAAS,MAAM,QAAQ,CAAC,CAAC;AACtD,QAAM,EAAE,MAAM,SAAS,IAAI,eAAe,UAAU;AAEpD,SAAO;AAAA,IACL,UAAU,SAAS;AAAA,IACnB,MAAM;AAAA,IACN,OAAO,SAAS;AAAA,IAChB,cAAc;AAAA,IACd,eAAe;AAAA,EACjB;AACF;AAEA,eAAe,iBACb,QACA,QACuE;AACvE,MAAI;AACF,QAAI,CAAC,qBAAqB,MAAM,GAAG;AACjC,UAAI,MAAM,+CAA+C;AACzD,aAAO;AAAA,IACT;AAEA,UAAM,iBAAiB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,UAOjB,MAAM;AAAA;AAAA;AAIZ,UAAM,WAAW,MAAM;AAAA,MACrB;AAAA,MACA,CAAC,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,QAAQ,MAAM,eAAe,EAAE,CAAC;AAAA,MAClE;AAAA,IACF;AAEA,QAAI,CAAC,UAAU,SAAS,MAAM;AAC5B,UAAI,MAAM,+CAA+C;AACzD,aAAO;AAAA,IACT;AAEA,UAAM,OAAO,SAAS,QAAQ,KAAK,KAAK;AACxC,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,IAAI;AAAA,IAC1B,QAAQ;AACN,YAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,UAAI,WAAW;AACb,YAAI;AACF,mBAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AAAA,QAClC,QAAQ;AACN,cAAI,MAAM,oDAAoD;AAC9D,iBAAO;AAAA,QACT;AAAA,MACF,OAAO;AACL,eAAO;AAAA,MACT;AAAA,IACF;AAEA,QAAI,OAAO,WAAW,YAAY,WAAW,KAAM,QAAO;AAE1D,UAAM,MAAM;AACZ,UAAM,WAAW,MAAM,QAAQ,IAAI,QAAQ,IACvC,IAAI,SAAS,OAAO,CAAC,MAAmB,OAAO,MAAM,YAAY,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IACzG,CAAC;AACL,UAAM,gBAAgB,MAAM,QAAQ,IAAI,IAAI,IACxC,IAAI,KAAK,OAAO,CAAC,MAAmB,OAAO,MAAM,YAAY,WAAW,CAAC,CAAC,IAC1E,CAAC;AACL,UAAM,QAAQ,OAAO,IAAI,UAAU,WAAW,IAAI,QAAQ;AAE1D,QAAI,SAAS,WAAW,KAAK,cAAc,WAAW,GAAG;AACvD,UAAI,MAAM,uCAAuC;AACjD,aAAO;AAAA,IACT;AAEA,WAAO,EAAE,UAAU,SAAS,MAAM,GAAG,kBAAkB,GAAG,MAAM,eAAe,MAAM;AAAA,EACvF,SAAS,KAAK;AACZ,QAAI,MAAM,oCAAoC;AAAA,MAC5C,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,EACT;AACF;AAEA,SAAS,iBAAiB,QAAuE;AAC/F,QAAM,gBAAgB,oBAAoB,MAAM;AAChD,QAAM,WAAW,sBAAsB,MAAM;AAE7C,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AACF;AAEA,SAAS,oBAAoB,MAAwB;AACnD,QAAM,WAAW;AACjB,QAAM,UAAU,KAAK,MAAM,QAAQ,KAAK,CAAC;AACzC,QAAM,UAAU,QAAQ,IAAI,CAAC,QAAQ,IAAI,QAAQ,eAAe,EAAE,CAAC;AACnE,SAAO,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,EAAE,OAAO,UAAU;AAChD;AAEA,SAAS,WAAW,KAAsB;AACxC,MAAI;AACF,QAAI,IAAI,GAAG;AACX,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,sBAAsB,QAA0B;AACvD,QAAM,YAAY,oBAAI,IAAI;AAAA,IACxB;AAAA,IAAO;AAAA,IAAK;AAAA,IAAM;AAAA,IAAM;AAAA,IAAO;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAM;AAAA,IAAQ;AAAA,IAC5D;AAAA,IAAQ;AAAA,IAAO;AAAA,IAAO;AAAA,IAAM;AAAA,IAAQ;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAS;AAAA,IAC5D;AAAA,IAAU;AAAA,IAAO;AAAA,IAAS;AAAA,IAAO;AAAA,IAAS;AAAA,IAAM;AAAA,IAAM;AAAA,IAAM;AAAA,IAC5D;AAAA,IAAM;AAAA,IAAQ;AAAA,IAAM;AAAA,IAAM;AAAA,IAAQ;AAAA,IAAM;AAAA,IAAQ;AAAA,IAAW;AAAA,IAC3D;AAAA,IAAO;AAAA,IAAM;AAAA,IAAO;AAAA,IAAM;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAO;AAAA,IAAU;AAAA,IAC1D;AAAA,IAAQ;AAAA,IAAM;AAAA,IAAK;AAAA,IAAM;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAM;AAAA,IAAQ;AAAA,IACtD;AAAA,IAAM;AAAA,IAAQ;AAAA,IAAO;AAAA,IAAS;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAS;AAAA,IACtD;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAS;AAAA,IAAM;AAAA,IAAO;AAAA,IAAM;AAAA,IAAQ;AAAA,IACnD;AAAA,IAAO;AAAA,IAAQ;AAAA,IAAQ;AAAA,IAAU;AAAA,IAAQ;AAAA,EAC3C,CAAC;AAED,QAAM,UAAU,OAAO,QAAQ,sBAAsB,EAAE,EAAE,KAAK;AAC9D,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,QAAQ,QACX,QAAQ,uBAAuB,GAAG,EAClC,MAAM,KAAK,EACX,OAAO,OAAO;AAEjB,QAAM,eAAe,MAAM,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC;AAExF,MAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAEvC,QAAM,UAAoB,CAAC;AAE3B,QAAM,YAAY,aAAa,KAAK,GAAG;AACvC,MAAI,UAAU,UAAU,kBAAkB;AACxC,YAAQ,KAAK,SAAS;AAAA,EACxB,OAAO;AACL,YAAQ,KAAK,UAAU,MAAM,GAAG,gBAAgB,CAAC;AAAA,EACnD;AAEA,MAAI,aAAa,UAAU,GAAG;AAC5B,UAAM,OAAO,KAAK,KAAK,aAAa,SAAS,CAAC;AAC9C,YAAQ,KAAK,aAAa,MAAM,GAAG,IAAI,EAAE,KAAK,GAAG,CAAC;AAAA,EACpD;AAEA,MAAI,aAAa,UAAU,GAAG;AAC5B,UAAM,OAAO,KAAK,KAAK,aAAa,SAAS,CAAC;AAC9C,YAAQ,KAAK,aAAa,MAAM,IAAI,EAAE,KAAK,GAAG,CAAC;AAAA,EACjD;AAEA,MAAI,aAAa,UAAU,GAAG;AAC5B,UAAM,UAAoB,CAAC;AAC3B,aAAS,IAAI,GAAG,IAAI,aAAa,SAAS,KAAK,QAAQ,SAAS,GAAG,KAAK;AACtE,cAAQ,KAAK,GAAG,aAAa,CAAC,CAAC,IAAI,aAAa,IAAI,CAAC,CAAC,EAAE;AAAA,IAC1D;AACA,QAAI,QAAQ,SAAS,GAAG;AACtB,cAAQ,KAAK,QAAQ,KAAK,GAAG,CAAC;AAAA,IAChC;AAAA,EACF;AAEA,QAAM,WAAW,aAAa;AAAA,IAC5B,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,YAAY;AAAA,EACnD;AACA,MAAI,SAAS,UAAU,GAAG;AACxB,YAAQ,KAAK,SAAS,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC;AAAA,EAC7C;AAEA,QAAM,SAAS,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,EAAE;AAAA,IAAI,CAAC,MACxC,EAAE,SAAS,mBAAmB,EAAE,MAAM,GAAG,gBAAgB,IAAI;AAAA,EAC/D;AACA,SAAO,OAAO,MAAM,GAAG,kBAAkB;AAC3C;AAEA,SAAS,UAAU,cAAwB,cAAkC;AAC3E,QAAM,MAAM,CAAC,GAAG,cAAc,GAAG,YAAY;AAC7C,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;","names":["kept","excluded","mergedUrls"]}

package/dist/agent/relevance.d.ts

@@ -0,0 +1,15 @@
+export declare function isValidCandidateUrl(u: string): boolean;
+export declare function isBlocklistedDomain(u: string): boolean;
+export interface PreFilterResult<T extends {
+    url: string;
+}> {
+    kept: T[];
+    excluded: {
+        item: T;
+        reason: 'invalid_url' | 'blocklisted_domain';
+    }[];
+}
+export declare function preFilterCandidates<T extends {
+    url: string;
+}>(items: T[]): PreFilterResult<T>;
+//# sourceMappingURL=relevance.d.ts.map

package/dist/agent/relevance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"relevance.d.ts","sourceRoot":"","sources":["../../src/agent/relevance.ts"],"names":[],"mappings":"AAeA,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAYtD;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAWtD;AAED,MAAM,WAAW,eAAe,CAAC,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE;IACxD,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,QAAQ,EAAE;QAAE,IAAI,EAAE,CAAC,CAAC;QAAC,MAAM,EAAE,aAAa,GAAG,oBAAoB,CAAA;KAAE,EAAE,CAAC;CACvE;AAED,wBAAgB,mBAAmB,CAAC,CAAC,SAAS;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,CAe7F"}

package/dist/agent/relevance.js

@@ -0,0 +1,60 @@
+const BLOCKLIST = /* @__PURE__ */ new Set([
+  "elitepvpers.com",
+  "zhihu.com",
+  "baidu.com",
+  "zhidao.baidu.com",
+  "jingyan.baidu.com",
+  "wenku.baidu.com",
+  "tieba.baidu.com"
+]);
+const REDIRECT_HOSTS = /* @__PURE__ */ new Set([
+  "duckduckgo.com",
+  "r.jina.ai"
+]);
+function isValidCandidateUrl(u) {
+  try {
+    const parsed = new URL(u);
+    if (!["http:", "https:"].includes(parsed.protocol)) return false;
+    if (REDIRECT_HOSTS.has(parsed.hostname)) {
+      if (parsed.hostname === "duckduckgo.com" && parsed.pathname.startsWith("/l/")) return false;
+      if (parsed.hostname === "r.jina.ai") return false;
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
+function isBlocklistedDomain(u) {
+  try {
+    const host = new URL(u).hostname.toLowerCase();
+    if (BLOCKLIST.has(host)) return true;
+    for (const dom of BLOCKLIST) {
+      if (host.endsWith(`.${dom}`)) return true;
+    }
+    return false;
+  } catch {
+    return true;
+  }
+}
+function preFilterCandidates(items) {
+  const kept = [];
+  const excluded = [];
+  for (const it of items) {
+    if (!isValidCandidateUrl(it.url)) {
+      excluded.push({ item: it, reason: "invalid_url" });
+      continue;
+    }
+    if (isBlocklistedDomain(it.url)) {
+      excluded.push({ item: it, reason: "blocklisted_domain" });
+      continue;
+    }
+    kept.push(it);
+  }
+  return { kept, excluded };
+}
+export {
+  isBlocklistedDomain,
+  isValidCandidateUrl,
+  preFilterCandidates
+};
+//# sourceMappingURL=relevance.js.map

package/dist/agent/relevance.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/agent/relevance.ts"],"sourcesContent":["const BLOCKLIST: ReadonlySet<string> = new Set([\n  'elitepvpers.com',\n  'zhihu.com',\n  'baidu.com',\n  'zhidao.baidu.com',\n  'jingyan.baidu.com',\n  'wenku.baidu.com',\n  'tieba.baidu.com',\n]);\n\nconst REDIRECT_HOSTS: ReadonlySet<string> = new Set([\n  'duckduckgo.com',\n  'r.jina.ai',\n]);\n\nexport function isValidCandidateUrl(u: string): boolean {\n  try {\n    const parsed = new URL(u);\n    if (!['http:', 'https:'].includes(parsed.protocol)) return false;\n    if (REDIRECT_HOSTS.has(parsed.hostname)) {\n      if (parsed.hostname === 'duckduckgo.com' && parsed.pathname.startsWith('/l/')) return false;\n      if (parsed.hostname === 'r.jina.ai') return false;\n    }\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nexport function isBlocklistedDomain(u: string): boolean {\n  try {\n    const host = new URL(u).hostname.toLowerCase();\n    if (BLOCKLIST.has(host)) return true;\n    for (const dom of BLOCKLIST) {\n      if (host.endsWith(`.${dom}`)) return true;\n    }\n    return false;\n  } catch {\n    return true;\n  }\n}\n\nexport interface PreFilterResult<T extends { url: string }> {\n  kept: T[];\n  excluded: { item: T; reason: 'invalid_url' | 'blocklisted_domain' }[];\n}\n\nexport function preFilterCandidates<T extends { url: string }>(items: T[]): PreFilterResult<T> {\n  const kept: T[] = [];\n  const excluded: PreFilterResult<T>['excluded'] = [];\n  for (const it of items) {\n    if (!isValidCandidateUrl(it.url)) {\n      excluded.push({ item: it, reason: 'invalid_url' });\n      continue;\n    }\n    if (isBlocklistedDomain(it.url)) {\n      excluded.push({ item: it, reason: 'blocklisted_domain' });\n      continue;\n    }\n    kept.push(it);\n  }\n  return { kept, excluded };\n}\n"],"mappings":"AAAA,MAAM,YAAiC,oBAAI,IAAI;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,MAAM,iBAAsC,oBAAI,IAAI;AAAA,EAClD;AAAA,EACA;AACF,CAAC;AAEM,SAAS,oBAAoB,GAAoB;AACtD,MAAI;AACF,UAAM,SAAS,IAAI,IAAI,CAAC;AACxB,QAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,OAAO,QAAQ,EAAG,QAAO;AAC3D,QAAI,eAAe,IAAI,OAAO,QAAQ,GAAG;AACvC,UAAI,OAAO,aAAa,oBAAoB,OAAO,SAAS,WAAW,KAAK,EAAG,QAAO;AACtF,UAAI,OAAO,aAAa,YAAa,QAAO;AAAA,IAC9C;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,oBAAoB,GAAoB;AACtD,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,CAAC,EAAE,SAAS,YAAY;AAC7C,QAAI,UAAU,IAAI,IAAI,EAAG,QAAO;AAChC,eAAW,OAAO,WAAW;AAC3B,UAAI,KAAK,SAAS,IAAI,GAAG,EAAE,EAAG,QAAO;AAAA,IACvC;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOO,SAAS,oBAA+C,OAAgC;AAC7F,QAAM,OAAY,CAAC;AACnB,QAAM,WAA2C,CAAC;AAClD,aAAW,MAAM,OAAO;AACtB,QAAI,CAAC,oBAAoB,GAAG,GAAG,GAAG;AAChC,eAAS,KAAK,EAAE,MAAM,IAAI,QAAQ,cAAc,CAAC;AACjD;AAAA,IACF;AACA,QAAI,oBAAoB,GAAG,GAAG,GAAG;AAC/B,eAAS,KAAK,EAAE,MAAM,IAAI,QAAQ,qBAAqB,CAAC;AACxD;AAAA,IACF;AACA,SAAK,KAAK,EAAE;AAAA,EACd;AACA,SAAO,EAAE,MAAM,SAAS;AAC1B;","names":[]}

package/dist/cache/backfill-embeddings.d.ts

@@ -0,0 +1,23 @@
+export interface BackfillOptions {
+    dataDir: string;
+    limit?: number;
+    batchSize?: number;
+    dryRun?: boolean;
+    onProgress?: (done: number, total: number) => void;
+}
+export interface BackfillResult {
+    scanned: number;
+    embedded: number;
+    skipped: number;
+    errors: number;
+    modelId: string;
+    reason?: string;
+}
+/**
+ * Walk url_cache rows that have no corresponding vec_id_map entry, embed
+ * title + a snippet of markdown, and upsert into the vector store. Used to
+ * recover pages cached before the sqlite-vec switch — find_similar's
+ * embedding path skips them otherwise.
+ */
+export declare function backfillEmbeddings(opts: BackfillOptions): Promise<BackfillResult>;
+//# sourceMappingURL=backfill-embeddings.d.ts.map

package/dist/cache/backfill-embeddings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill-embeddings.d.ts","sourceRoot":"","sources":["../../src/cache/backfill-embeddings.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAkBD;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CA8FvF"}

package/dist/cache/backfill-embeddings.js

@@ -0,0 +1,105 @@
+import { join } from "node:path";
+import { createLogger } from "../logger.js";
+import { initDatabase, closeDatabase, isVecExtensionLoaded } from "./db.js";
+import { getEmbedProvider } from "../providers/embed-provider.js";
+import { getVectorStore } from "../providers/vector-store.js";
+const log = createLogger("cache");
+const SELECT_PENDING_SQL = `
+  SELECT url, title, markdown, content_hash
+  FROM url_cache uc
+  WHERE NOT EXISTS (
+    SELECT 1 FROM vec_id_map vm WHERE vm.external_id = uc.url
+  )
+  ORDER BY uc.id ASC
+`;
+async function backfillEmbeddings(opts) {
+  const { dataDir, limit, batchSize = 32, dryRun = false, onProgress } = opts;
+  const db = initDatabase(join(dataDir, "wigolo.db"));
+  try {
+    if (!isVecExtensionLoaded()) {
+      return {
+        scanned: 0,
+        embedded: 0,
+        skipped: 0,
+        errors: 0,
+        modelId: "",
+        reason: "sqlite-vec extension not loaded \u2014 backfill skipped"
+      };
+    }
+    let provider;
+    let store;
+    try {
+      provider = await getEmbedProvider();
+      store = await getVectorStore();
+    } catch (err) {
+      return {
+        scanned: 0,
+        embedded: 0,
+        skipped: 0,
+        errors: 0,
+        modelId: "",
+        reason: `embedding pipeline unavailable: ${err instanceof Error ? err.message : String(err)} \u2014 run \`wigolo warmup --embeddings\` first`
+      };
+    }
+    let rows = db.prepare(SELECT_PENDING_SQL).all();
+    if (typeof limit === "number" && limit > 0) {
+      rows = rows.slice(0, limit);
+    }
+    const result = {
+      scanned: rows.length,
+      embedded: 0,
+      skipped: 0,
+      errors: 0,
+      modelId: provider.modelId
+    };
+    for (let i = 0; i < rows.length; i += batchSize) {
+      const batch = rows.slice(i, i + batchSize);
+      const usable = batch.filter(
+        (r) => r.title && r.title.trim().length > 0 || r.markdown && r.markdown.trim().length > 0
+      );
+      result.skipped += batch.length - usable.length;
+      if (usable.length === 0) {
+        onProgress?.(i + batch.length, rows.length);
+        continue;
+      }
+      const texts = usable.map((r) => {
+        const title = (r.title ?? "").trim();
+        const body = (r.markdown ?? "").slice(0, 500).trim();
+        return `${title}
+${body}`.trim();
+      });
+      try {
+        const vectors = await provider.embed(texts);
+        if (!dryRun) {
+          await store.upsert(
+            usable.map((r, idx) => ({
+              id: r.url,
+              vector: vectors[idx],
+              metadata: {
+                url: r.url,
+                contentHash: r.content_hash ?? "",
+                modelId: provider.modelId
+              }
+            }))
+          );
+        }
+        result.embedded += usable.length;
+      } catch (err) {
+        log.warn("backfill batch failed", {
+          batchStart: i,
+          batchSize: usable.length,
+          error: err instanceof Error ? err.message : String(err)
+        });
+        result.errors += usable.length;
+      }
+      onProgress?.(i + batch.length, rows.length);
+    }
+    return result;
+  } finally {
+    closeDatabase();
+  }
+}
+export {
+  backfillEmbeddings
+};
+//# sourceMappingURL=backfill-embeddings.js.map

package/dist/cache/backfill-embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/cache/backfill-embeddings.ts"],"sourcesContent":["import { join } from 'node:path';\nimport { createLogger } from '../logger.js';\nimport { initDatabase, closeDatabase, isVecExtensionLoaded } from './db.js';\nimport { getEmbedProvider } from '../providers/embed-provider.js';\nimport { getVectorStore } from '../providers/vector-store.js';\n\nconst log = createLogger('cache');\n\nexport interface BackfillOptions {\n  dataDir: string;\n  limit?: number;\n  batchSize?: number;\n  dryRun?: boolean;\n  onProgress?: (done: number, total: number) => void;\n}\n\nexport interface BackfillResult {\n  scanned: number;\n  embedded: number;\n  skipped: number;\n  errors: number;\n  modelId: string;\n  reason?: string;\n}\n\ninterface CacheRow {\n  url: string;\n  title: string | null;\n  markdown: string | null;\n  content_hash: string | null;\n}\n\nconst SELECT_PENDING_SQL = `\n  SELECT url, title, markdown, content_hash\n  FROM url_cache uc\n  WHERE NOT EXISTS (\n    SELECT 1 FROM vec_id_map vm WHERE vm.external_id = uc.url\n  )\n  ORDER BY uc.id ASC\n`;\n\n/**\n * Walk url_cache rows that have no corresponding vec_id_map entry, embed\n * title + a snippet of markdown, and upsert into the vector store. Used to\n * recover pages cached before the sqlite-vec switch — find_similar's\n * embedding path skips them otherwise.\n */\nexport async function backfillEmbeddings(opts: BackfillOptions): Promise<BackfillResult> {\n  const { dataDir, limit, batchSize = 32, dryRun = false, onProgress } = opts;\n\n  const db = initDatabase(join(dataDir, 'wigolo.db'));\n  try {\n    if (!isVecExtensionLoaded()) {\n      return {\n        scanned: 0,\n        embedded: 0,\n        skipped: 0,\n        errors: 0,\n        modelId: '',\n        reason: 'sqlite-vec extension not loaded — backfill skipped',\n      };\n    }\n\n    let provider: Awaited<ReturnType<typeof getEmbedProvider>>;\n    let store: Awaited<ReturnType<typeof getVectorStore>>;\n    try {\n      provider = await getEmbedProvider();\n      store = await getVectorStore();\n    } catch (err) {\n      return {\n        scanned: 0,\n        embedded: 0,\n        skipped: 0,\n        errors: 0,\n        modelId: '',\n        reason: `embedding pipeline unavailable: ${err instanceof Error ? err.message : String(err)} — run \\`wigolo warmup --embeddings\\` first`,\n      };\n    }\n\n    let rows = db.prepare(SELECT_PENDING_SQL).all() as CacheRow[];\n    if (typeof limit === 'number' && limit > 0) {\n      rows = rows.slice(0, limit);\n    }\n\n    const result: BackfillResult = {\n      scanned: rows.length,\n      embedded: 0,\n      skipped: 0,\n      errors: 0,\n      modelId: provider.modelId,\n    };\n\n    for (let i = 0; i < rows.length; i += batchSize) {\n      const batch = rows.slice(i, i + batchSize);\n      const usable = batch.filter(\n        (r) => (r.title && r.title.trim().length > 0) || (r.markdown && r.markdown.trim().length > 0),\n      );\n      result.skipped += batch.length - usable.length;\n      if (usable.length === 0) {\n        onProgress?.(i + batch.length, rows.length);\n        continue;\n      }\n\n      const texts = usable.map((r) => {\n        const title = (r.title ?? '').trim();\n        const body = (r.markdown ?? '').slice(0, 500).trim();\n        return `${title}\\n${body}`.trim();\n      });\n\n      try {\n        const vectors = await provider.embed(texts);\n        if (!dryRun) {\n          await store.upsert(\n            usable.map((r, idx) => ({\n              id: r.url,\n              vector: vectors[idx],\n              metadata: {\n                url: r.url,\n                contentHash: r.content_hash ?? '',\n                modelId: provider.modelId,\n              },\n            })),\n          );\n        }\n        result.embedded += usable.length;\n      } catch (err) {\n        log.warn('backfill batch failed', {\n          batchStart: i,\n          batchSize: usable.length,\n          error: err instanceof Error ? err.message : String(err),\n        });\n        result.errors += usable.length;\n      }\n\n      onProgress?.(i + batch.length, rows.length);\n    }\n\n    return result;\n  } finally {\n    closeDatabase();\n  }\n}\n"],"mappings":"AAAA,SAAS,YAAY;AACrB,SAAS,oBAAoB;AAC7B,SAAS,cAAc,eAAe,4BAA4B;AAClE,SAAS,wBAAwB;AACjC,SAAS,sBAAsB;AAE/B,MAAM,MAAM,aAAa,OAAO;AA0BhC,MAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe3B,eAAsB,mBAAmB,MAAgD;AACvF,QAAM,EAAE,SAAS,OAAO,YAAY,IAAI,SAAS,OAAO,WAAW,IAAI;AAEvE,QAAM,KAAK,aAAa,KAAK,SAAS,WAAW,CAAC;AAClD,MAAI;AACF,QAAI,CAAC,qBAAqB,GAAG;AAC3B,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI;AACJ,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,iBAAiB;AAClC,cAAQ,MAAM,eAAe;AAAA,IAC/B,SAAS,KAAK;AACZ,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,SAAS;AAAA,QACT,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,QAAQ,mCAAmC,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC7F;AAAA,IACF;AAEA,QAAI,OAAO,GAAG,QAAQ,kBAAkB,EAAE,IAAI;AAC9C,QAAI,OAAO,UAAU,YAAY,QAAQ,GAAG;AAC1C,aAAO,KAAK,MAAM,GAAG,KAAK;AAAA,IAC5B;AAEA,UAAM,SAAyB;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,UAAU;AAAA,MACV,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,SAAS,SAAS;AAAA,IACpB;AAEA,aAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,WAAW;AAC/C,YAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,SAAS;AACzC,YAAM,SAAS,MAAM;AAAA,QACnB,CAAC,MAAO,EAAE,SAAS,EAAE,MAAM,KAAK,EAAE,SAAS,KAAO,EAAE,YAAY,EAAE,SAAS,KAAK,EAAE,SAAS;AAAA,MAC7F;AACA,aAAO,WAAW,MAAM,SAAS,OAAO;AACxC,UAAI,OAAO,WAAW,GAAG;AACvB,qBAAa,IAAI,MAAM,QAAQ,KAAK,MAAM;AAC1C;AAAA,MACF;AAEA,YAAM,QAAQ,OAAO,IAAI,CAAC,MAAM;AAC9B,cAAM,SAAS,EAAE,SAAS,IAAI,KAAK;AACnC,cAAM,QAAQ,EAAE,YAAY,IAAI,MAAM,GAAG,GAAG,EAAE,KAAK;AACnD,eAAO,GAAG,KAAK;AAAA,EAAK,IAAI,GAAG,KAAK;AAAA,MAClC,CAAC;AAED,UAAI;AACF,cAAM,UAAU,MAAM,SAAS,MAAM,KAAK;AAC1C,YAAI,CAAC,QAAQ;AACX,gBAAM,MAAM;AAAA,YACV,OAAO,IAAI,CAAC,GAAG,SAAS;AAAA,cACtB,IAAI,EAAE;AAAA,cACN,QAAQ,QAAQ,GAAG;AAAA,cACnB,UAAU;AAAA,gBACR,KAAK,EAAE;AAAA,gBACP,aAAa,EAAE,gBAAgB;AAAA,gBAC/B,SAAS,SAAS;AAAA,cACpB;AAAA,YACF,EAAE;AAAA,UACJ;AAAA,QACF;AACA,eAAO,YAAY,OAAO;AAAA,MAC5B,SAAS,KAAK;AACZ,YAAI,KAAK,yBAAyB;AAAA,UAChC,YAAY;AAAA,UACZ,WAAW,OAAO;AAAA,UAClB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,QACxD,CAAC;AACD,eAAO,UAAU,OAAO;AAAA,MAC1B;AAEA,mBAAa,IAAI,MAAM,QAAQ,KAAK,MAAM;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT,UAAE;AACA,kBAAc;AAAA,EAChB;AACF;","names":[]}

package/dist/cache/change-detector.d.ts

@@ -0,0 +1,7 @@
+export interface ChangeResult {
+    changed: boolean;
+    previousHash?: string;
+    diffSummary?: string;
+}
+export declare function detectChange(url: string, newMarkdown: string): ChangeResult;
+//# sourceMappingURL=change-detector.d.ts.map

package/dist/cache/change-detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-detector.d.ts","sourceRoot":"","sources":["../../src/cache/change-detector.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAyC3E"}

package/dist/cache/change-detector.js

@@ -0,0 +1,43 @@
+import { createHash } from "node:crypto";
+import { createLogger } from "../logger.js";
+import { normalizeUrl, getHashForNormalizedUrl, getMarkdownForNormalizedUrl } from "./store.js";
+import { computeDiffSummary } from "./diff-summary.js";
+const log = createLogger("cache");
+function detectChange(url, newMarkdown) {
+  try {
+    const normalizedUrl = normalizeUrl(url);
+    const previousHash = getHashForNormalizedUrl(normalizedUrl);
+    if (previousHash === null) {
+      log.debug("no cached entry for change detection", { url: normalizedUrl });
+      return { changed: false };
+    }
+    const newHash = createHash("sha256").update(newMarkdown).digest("hex");
+    if (newHash === previousHash) {
+      log.debug("content unchanged", { url: normalizedUrl, hash: newHash });
+      return { changed: false };
+    }
+    const previousMarkdown = getMarkdownForNormalizedUrl(normalizedUrl);
+    const diffSummary = previousMarkdown !== null ? computeDiffSummary(previousMarkdown, newMarkdown) : void 0;
+    log.info("content change detected", {
+      url: normalizedUrl,
+      previousHash,
+      newHash,
+      diffSummary
+    });
+    return {
+      changed: true,
+      previousHash,
+      diffSummary
+    };
+  } catch (err) {
+    log.error("change detection failed", {
+      url,
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return { changed: false };
+  }
+}
+export {
+  detectChange
+};
+//# sourceMappingURL=change-detector.js.map

package/dist/cache/change-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/cache/change-detector.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { createLogger } from '../logger.js';\nimport { normalizeUrl, getHashForNormalizedUrl, getMarkdownForNormalizedUrl } from './store.js';\nimport { computeDiffSummary } from './diff-summary.js';\n\nconst log = createLogger('cache');\n\nexport interface ChangeResult {\n  changed: boolean;\n  previousHash?: string;\n  diffSummary?: string;\n}\n\nexport function detectChange(url: string, newMarkdown: string): ChangeResult {\n  try {\n    const normalizedUrl = normalizeUrl(url);\n    const previousHash = getHashForNormalizedUrl(normalizedUrl);\n\n    if (previousHash === null) {\n      log.debug('no cached entry for change detection', { url: normalizedUrl });\n      return { changed: false };\n    }\n\n    const newHash = createHash('sha256').update(newMarkdown).digest('hex');\n\n    if (newHash === previousHash) {\n      log.debug('content unchanged', { url: normalizedUrl, hash: newHash });\n      return { changed: false };\n    }\n\n    const previousMarkdown = getMarkdownForNormalizedUrl(normalizedUrl);\n    const diffSummary = previousMarkdown !== null\n      ? computeDiffSummary(previousMarkdown, newMarkdown)\n      : undefined;\n\n    log.info('content change detected', {\n      url: normalizedUrl,\n      previousHash,\n      newHash,\n      diffSummary,\n    });\n\n    return {\n      changed: true,\n      previousHash,\n      diffSummary,\n    };\n  } catch (err) {\n    log.error('change detection failed', {\n      url,\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return { changed: false };\n  }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAC7B,SAAS,cAAc,yBAAyB,mCAAmC;AACnF,SAAS,0BAA0B;AAEnC,MAAM,MAAM,aAAa,OAAO;AAQzB,SAAS,aAAa,KAAa,aAAmC;AAC3E,MAAI;AACF,UAAM,gBAAgB,aAAa,GAAG;AACtC,UAAM,eAAe,wBAAwB,aAAa;AAE1D,QAAI,iBAAiB,MAAM;AACzB,UAAI,MAAM,wCAAwC,EAAE,KAAK,cAAc,CAAC;AACxE,aAAO,EAAE,SAAS,MAAM;AAAA,IAC1B;AAEA,UAAM,UAAU,WAAW,QAAQ,EAAE,OAAO,WAAW,EAAE,OAAO,KAAK;AAErE,QAAI,YAAY,cAAc;AAC5B,UAAI,MAAM,qBAAqB,EAAE,KAAK,eAAe,MAAM,QAAQ,CAAC;AACpE,aAAO,EAAE,SAAS,MAAM;AAAA,IAC1B;AAEA,UAAM,mBAAmB,4BAA4B,aAAa;AAClE,UAAM,cAAc,qBAAqB,OACrC,mBAAmB,kBAAkB,WAAW,IAChD;AAEJ,QAAI,KAAK,2BAA2B;AAAA,MAClC,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAED,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B;AAAA,MACnC;AAAA,MACA,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO,EAAE,SAAS,MAAM;AAAA,EAC1B;AACF;","names":[]}

package/dist/cache/db.d.ts

@@ -1,4 +1,5 @@
 import Database from 'better-sqlite3';
+export declare function isVecExtensionLoaded(): boolean;
 export declare function initDatabase(dbPath: string): Database.Database;
 export declare function getDatabase(): Database.Database;
 export declare function closeDatabase(): void;

package/dist/cache/db.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/cache/db.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAItC,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAqF9D;AAED,wBAAgB,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAK/C;AAED,wBAAgB,aAAa,IAAI,IAAI,CAKpC"}
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/cache/db.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAWtC,wBAAgB,oBAAoB,IAAI,OAAO,CAE9C;AAkBD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAkJ9D;AAED,wBAAgB,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAK/C;AAED,wBAAgB,aAAa,IAAI,IAAI,CAMpC"}