@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/crawl/dedup.js

@@ -1,93 +1,136 @@
-import { createHash } from 'node:crypto';
-import { getDatabase } from '../cache/db.js';
-export function splitIntoBlocks(markdown) {
-    if (!markdown.trim())
-        return [];
-    const lines = markdown.split('\n');
-    const headingIndices = [];
-    for (let i = 0; i < lines.length; i++) {
-        const match = lines[i].match(/^(#{1,6})\s+/);
-        if (match) {
-            headingIndices.push({ level: match[1].length, lineIdx: i });
-        }
+import { createHash } from "node:crypto";
+import { getDatabase } from "../cache/db.js";
+function splitIntoBlocks(markdown) {
+  if (!markdown.trim()) return [];
+  const lines = markdown.split("\n");
+  const headingIndices = [];
+  for (let i = 0; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+/);
+    if (match) {
+      headingIndices.push({ level: match[1].length, lineIdx: i });
     }
-    // If no headings, split by double-newline (paragraph blocks)
-    if (headingIndices.length === 0) {
-        return markdown.split(/\n\n+/).map((b) => b.trim()).filter(Boolean);
-    }
-    // Non-overlapping split: each heading starts a new block, ending at the next heading of ANY level
-    const blocks = [];
-    for (let i = 0; i < headingIndices.length; i++) {
-        const start = headingIndices[i].lineIdx;
-        const end = i + 1 < headingIndices.length ? headingIndices[i + 1].lineIdx : lines.length;
-        blocks.push(lines.slice(start, end).join('\n').trim());
-    }
-    return blocks.filter(Boolean);
+  }
+  if (headingIndices.length === 0) {
+    return markdown.split(/\n\n+/).map((b) => b.trim()).filter(Boolean);
+  }
+  const blocks = [];
+  for (let i = 0; i < headingIndices.length; i++) {
+    const start = headingIndices[i].lineIdx;
+    const end = i + 1 < headingIndices.length ? headingIndices[i + 1].lineIdx : lines.length;
+    blocks.push(lines.slice(start, end).join("\n").trim());
+  }
+  return blocks.filter(Boolean);
 }
-export function normalizeBlockText(text) {
-    return text.toLowerCase().replace(/\s+/g, ' ').trim();
+function normalizeBlockText(text) {
+  return text.toLowerCase().replace(/\s+/g, " ").trim();
 }
 function hashBlock(text) {
-    return createHash('sha256').update(normalizeBlockText(text)).digest('hex');
+  return createHash("sha256").update(normalizeBlockText(text)).digest("hex");
+}
+const NAV_DEDUPE_THRESHOLD = 0.6;
+const MAX_LEADING_LINES = 30;
+const MAX_TRAILING_LINES = 20;
+const MIN_CORPUS = 4;
+function lineHash(line) {
+  return createHash("sha1").update(line.trim().toLowerCase()).digest("hex");
 }
-export function deduplicatePages(pages, domain) {
-    if (pages.length <= 1)
-        return pages.map((p) => ({ url: p.url, markdown: p.markdown }));
-    // Pre-load stored boilerplate hashes for this domain
-    const storedHashes = domain ? getStoredBoilerplate(domain) : [];
-    const boilerplateHashes = new Set(storedHashes);
-    // Split each page into blocks and hash them
-    const pageBlocks = pages.map((page) => ({
-        url: page.url,
-        blocks: splitIntoBlocks(page.markdown),
-    }));
-    // Count how many pages each block hash appears in
-    const hashPageCount = new Map();
-    for (const page of pageBlocks) {
-        const seenHashes = new Set();
-        for (const block of page.blocks) {
-            const h = hashBlock(block);
-            if (!seenHashes.has(h)) {
-                seenHashes.add(h);
-                hashPageCount.set(h, (hashPageCount.get(h) ?? 0) + 1);
-            }
-        }
+function stripRepeatedNavigationLines(pages) {
+  if (pages.length < MIN_CORPUS) return pages;
+  const lineSets = pages.map((p) => p.markdown.split("\n"));
+  const countLeading = /* @__PURE__ */ new Map();
+  const countTrailing = /* @__PURE__ */ new Map();
+  for (const lines of lineSets) {
+    const seenL = /* @__PURE__ */ new Set();
+    for (let i = 0; i < Math.min(MAX_LEADING_LINES, lines.length); i++) {
+      const h = lineHash(lines[i]);
+      if (!seenL.has(h)) {
+        seenL.add(h);
+        countLeading.set(h, (countLeading.get(h) ?? 0) + 1);
+      }
     }
-    // Mark hashes appearing in >50% of pages as boilerplate
-    const threshold = pages.length / 2;
-    for (const [hash, count] of hashPageCount) {
-        if (count > threshold) {
-            boilerplateHashes.add(hash);
-        }
+    const seenT = /* @__PURE__ */ new Set();
+    for (let i = lines.length - 1; i >= Math.max(lines.length - MAX_TRAILING_LINES, 0); i--) {
+      const h = lineHash(lines[i]);
+      if (!seenT.has(h)) {
+        seenT.add(h);
+        countTrailing.set(h, (countTrailing.get(h) ?? 0) + 1);
+      }
     }
-    // Store updated boilerplate hashes for this domain
-    if (domain) {
-        storeBoilerplate(domain, Array.from(boilerplateHashes));
+  }
+  const threshold = pages.length * NAV_DEDUPE_THRESHOLD;
+  const navLeading = new Set([...countLeading].filter(([, c]) => c >= threshold).map(([h]) => h));
+  const navTrailing = new Set([...countTrailing].filter(([, c]) => c >= threshold).map(([h]) => h));
+  return pages.map((page, i) => {
+    const lines = lineSets[i];
+    let head = 0;
+    while (head < lines.length && (lines[head].trim() === "" || navLeading.has(lineHash(lines[head])))) head++;
+    let tail = lines.length;
+    while (tail > head && (lines[tail - 1].trim() === "" || navTrailing.has(lineHash(lines[tail - 1])))) tail--;
+    return { url: page.url, markdown: lines.slice(head, tail).join("\n") };
+  });
+}
+function deduplicatePages(pages, domain) {
+  if (pages.length <= 1) return pages.map((p) => ({ url: p.url, markdown: p.markdown }));
+  const stripped = stripRepeatedNavigationLines(pages);
+  const storedHashes = domain ? getStoredBoilerplate(domain) : [];
+  const boilerplateHashes = new Set(storedHashes);
+  const pageBlocks = stripped.map((page) => ({
+    url: page.url,
+    blocks: splitIntoBlocks(page.markdown)
+  }));
+  const hashPageCount = /* @__PURE__ */ new Map();
+  for (const page of pageBlocks) {
+    const seenHashes = /* @__PURE__ */ new Set();
+    for (const block of page.blocks) {
+      const h = hashBlock(block);
+      if (!seenHashes.has(h)) {
+        seenHashes.add(h);
+        hashPageCount.set(h, (hashPageCount.get(h) ?? 0) + 1);
+      }
+    }
+  }
+  const threshold = pages.length / 2;
+  for (const [hash, count] of hashPageCount) {
+    if (count > threshold) {
+      boilerplateHashes.add(hash);
     }
-    // Strip boilerplate blocks from each page
-    return pageBlocks.map((page) => {
-        const filtered = page.blocks.filter((block) => !boilerplateHashes.has(hashBlock(block)));
-        return {
-            url: page.url,
-            markdown: filtered.join('\n\n'),
-        };
-    });
+  }
+  if (domain) {
+    storeBoilerplate(domain, Array.from(boilerplateHashes));
+  }
+  return pageBlocks.map((page) => {
+    const filtered = page.blocks.filter((block) => !boilerplateHashes.has(hashBlock(block)));
+    return {
+      url: page.url,
+      markdown: filtered.join("\n\n")
+    };
+  });
 }
-export function getStoredBoilerplate(domain) {
-    const db = getDatabase();
-    const rows = db.prepare('SELECT block_hash FROM domain_boilerplate WHERE domain = ?').all(domain);
-    return rows.map(r => r.block_hash);
+function getStoredBoilerplate(domain) {
+  const db = getDatabase();
+  const rows = db.prepare("SELECT block_hash FROM domain_boilerplate WHERE domain = ?").all(domain);
+  return rows.map((r) => r.block_hash);
 }
-export function storeBoilerplate(domain, hashes) {
-    const db = getDatabase();
-    const del = db.prepare('DELETE FROM domain_boilerplate WHERE domain = ?');
-    const insert = db.prepare('INSERT OR IGNORE INTO domain_boilerplate (domain, block_hash, sample_text) VALUES (?, ?, ?)');
-    const tx = db.transaction((items) => {
-        del.run(domain);
-        for (const hash of items) {
-            insert.run(domain, hash, null);
-        }
-    });
-    tx(hashes);
+function storeBoilerplate(domain, hashes) {
+  const db = getDatabase();
+  const del = db.prepare("DELETE FROM domain_boilerplate WHERE domain = ?");
+  const insert = db.prepare(
+    "INSERT OR IGNORE INTO domain_boilerplate (domain, block_hash, sample_text) VALUES (?, ?, ?)"
+  );
+  const tx = db.transaction((items) => {
+    del.run(domain);
+    for (const hash of items) {
+      insert.run(domain, hash, null);
+    }
+  });
+  tx(hashes);
 }
+export {
+  deduplicatePages,
+  getStoredBoilerplate,
+  normalizeBlockText,
+  splitIntoBlocks,
+  storeBoilerplate,
+  stripRepeatedNavigationLines
+};
 //# sourceMappingURL=dedup.js.map

package/dist/crawl/dedup.js.map

@@ -1 +1 @@
-{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,cAAc,GAAyC,EAAE,CAAC;IAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,kGAAkG;IAClG,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAYD,MAAM,UAAU,gBAAgB,CAAC,KAAkB,EAAE,MAAe;IAClE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEvF,qDAAqD;IACrD,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IAExD,4CAA4C;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACtC,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;KACvC,CAAC,CAAC,CAAC;IAEJ,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClB,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,0CAA0C;IAC1C,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzF,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,MAAM,CAA6B,CAAC;IAC9H,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAgB;IAC/D,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,6FAA6F,CAC9F,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;QAC5C,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,MAAM,CAAC,CAAC;AACb,CAAC"}
+{"version":3,"sources":["../../src/crawl/dedup.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { getDatabase } from '../cache/db.js';\n\nexport function splitIntoBlocks(markdown: string): string[] {\n  if (!markdown.trim()) return [];\n\n  const lines = markdown.split('\\n');\n  const headingIndices: { level: number; lineIdx: number }[] = [];\n\n  for (let i = 0; i < lines.length; i++) {\n    const match = lines[i].match(/^(#{1,6})\\s+/);\n    if (match) {\n      headingIndices.push({ level: match[1].length, lineIdx: i });\n    }\n  }\n\n  // If no headings, split by double-newline (paragraph blocks)\n  if (headingIndices.length === 0) {\n    return markdown.split(/\\n\\n+/).map((b) => b.trim()).filter(Boolean);\n  }\n\n  // Non-overlapping split: each heading starts a new block, ending at the next heading of ANY level\n  const blocks: string[] = [];\n  for (let i = 0; i < headingIndices.length; i++) {\n    const start = headingIndices[i].lineIdx;\n    const end = i + 1 < headingIndices.length ? headingIndices[i + 1].lineIdx : lines.length;\n    blocks.push(lines.slice(start, end).join('\\n').trim());\n  }\n\n  return blocks.filter(Boolean);\n}\n\nexport function normalizeBlockText(text: string): string {\n  return text.toLowerCase().replace(/\\s+/g, ' ').trim();\n}\n\nfunction hashBlock(text: string): string {\n  return createHash('sha256').update(normalizeBlockText(text)).digest('hex');\n}\n\ninterface PageInput {\n  url: string;\n  markdown: string;\n}\n\ninterface PageOutput {\n  url: string;\n  markdown: string;\n}\n\nconst NAV_DEDUPE_THRESHOLD = 0.6;\nconst MAX_LEADING_LINES = 30;\nconst MAX_TRAILING_LINES = 20;\nconst MIN_CORPUS = 4;\n\nfunction lineHash(line: string): string {\n  return createHash('sha1').update(line.trim().toLowerCase()).digest('hex');\n}\n\nexport function stripRepeatedNavigationLines(pages: PageInput[]): PageInput[] {\n  if (pages.length < MIN_CORPUS) return pages;\n  const lineSets = pages.map((p) => p.markdown.split('\\n'));\n\n  const countLeading = new Map<string, number>();\n  const countTrailing = new Map<string, number>();\n  for (const lines of lineSets) {\n    const seenL = new Set<string>();\n    for (let i = 0; i < Math.min(MAX_LEADING_LINES, lines.length); i++) {\n      const h = lineHash(lines[i]);\n      if (!seenL.has(h)) {\n        seenL.add(h);\n        countLeading.set(h, (countLeading.get(h) ?? 0) + 1);\n      }\n    }\n    const seenT = new Set<string>();\n    for (let i = lines.length - 1; i >= Math.max(lines.length - MAX_TRAILING_LINES, 0); i--) {\n      const h = lineHash(lines[i]);\n      if (!seenT.has(h)) {\n        seenT.add(h);\n        countTrailing.set(h, (countTrailing.get(h) ?? 0) + 1);\n      }\n    }\n  }\n\n  const threshold = pages.length * NAV_DEDUPE_THRESHOLD;\n  const navLeading = new Set([...countLeading].filter(([, c]) => c >= threshold).map(([h]) => h));\n  const navTrailing = new Set([...countTrailing].filter(([, c]) => c >= threshold).map(([h]) => h));\n\n  return pages.map((page, i) => {\n    const lines = lineSets[i];\n    let head = 0;\n    while (head < lines.length && (lines[head].trim() === '' || navLeading.has(lineHash(lines[head])))) head++;\n    let tail = lines.length;\n    while (tail > head && (lines[tail - 1].trim() === '' || navTrailing.has(lineHash(lines[tail - 1])))) tail--;\n    return { url: page.url, markdown: lines.slice(head, tail).join('\\n') };\n  });\n}\n\nexport function deduplicatePages(pages: PageInput[], domain?: string): PageOutput[] {\n  if (pages.length <= 1) return pages.map((p) => ({ url: p.url, markdown: p.markdown }));\n\n  const stripped = stripRepeatedNavigationLines(pages);\n\n  // Pre-load stored boilerplate hashes for this domain\n  const storedHashes = domain ? getStoredBoilerplate(domain) : [];\n  const boilerplateHashes = new Set<string>(storedHashes);\n\n  // Split each page into blocks and hash them\n  const pageBlocks = stripped.map((page) => ({\n    url: page.url,\n    blocks: splitIntoBlocks(page.markdown),\n  }));\n\n  // Count how many pages each block hash appears in\n  const hashPageCount = new Map<string, number>();\n  for (const page of pageBlocks) {\n    const seenHashes = new Set<string>();\n    for (const block of page.blocks) {\n      const h = hashBlock(block);\n      if (!seenHashes.has(h)) {\n        seenHashes.add(h);\n        hashPageCount.set(h, (hashPageCount.get(h) ?? 0) + 1);\n      }\n    }\n  }\n\n  // Mark hashes appearing in >50% of pages as boilerplate\n  const threshold = pages.length / 2;\n  for (const [hash, count] of hashPageCount) {\n    if (count > threshold) {\n      boilerplateHashes.add(hash);\n    }\n  }\n\n  // Store updated boilerplate hashes for this domain\n  if (domain) {\n    storeBoilerplate(domain, Array.from(boilerplateHashes));\n  }\n\n  // Strip boilerplate blocks from each page\n  return pageBlocks.map((page) => {\n    const filtered = page.blocks.filter((block) => !boilerplateHashes.has(hashBlock(block)));\n    return {\n      url: page.url,\n      markdown: filtered.join('\\n\\n'),\n    };\n  });\n}\n\nexport function getStoredBoilerplate(domain: string): string[] {\n  const db = getDatabase();\n  const rows = db.prepare('SELECT block_hash FROM domain_boilerplate WHERE domain = ?').all(domain) as { block_hash: string }[];\n  return rows.map(r => r.block_hash);\n}\n\nexport function storeBoilerplate(domain: string, hashes: string[]): void {\n  const db = getDatabase();\n  const del = db.prepare('DELETE FROM domain_boilerplate WHERE domain = ?');\n  const insert = db.prepare(\n    'INSERT OR IGNORE INTO domain_boilerplate (domain, block_hash, sample_text) VALUES (?, ?, ?)',\n  );\n  const tx = db.transaction((items: string[]) => {\n    del.run(domain);\n    for (const hash of items) {\n      insert.run(domain, hash, null);\n    }\n  });\n  tx(hashes);\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,mBAAmB;AAErB,SAAS,gBAAgB,UAA4B;AAC1D,MAAI,CAAC,SAAS,KAAK,EAAG,QAAO,CAAC;AAE9B,QAAM,QAAQ,SAAS,MAAM,IAAI;AACjC,QAAM,iBAAuD,CAAC;AAE9D,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAM,QAAQ,MAAM,CAAC,EAAE,MAAM,cAAc;AAC3C,QAAI,OAAO;AACT,qBAAe,KAAK,EAAE,OAAO,MAAM,CAAC,EAAE,QAAQ,SAAS,EAAE,CAAC;AAAA,IAC5D;AAAA,EACF;AAGA,MAAI,eAAe,WAAW,GAAG;AAC/B,WAAO,SAAS,MAAM,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AAAA,EACpE;AAGA,QAAM,SAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,eAAe,QAAQ,KAAK;AAC9C,UAAM,QAAQ,eAAe,CAAC,EAAE;AAChC,UAAM,MAAM,IAAI,IAAI,eAAe,SAAS,eAAe,IAAI,CAAC,EAAE,UAAU,MAAM;AAClF,WAAO,KAAK,MAAM,MAAM,OAAO,GAAG,EAAE,KAAK,IAAI,EAAE,KAAK,CAAC;AAAA,EACvD;AAEA,SAAO,OAAO,OAAO,OAAO;AAC9B;AAEO,SAAS,mBAAmB,MAAsB;AACvD,SAAO,KAAK,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACtD;AAEA,SAAS,UAAU,MAAsB;AACvC,SAAO,WAAW,QAAQ,EAAE,OAAO,mBAAmB,IAAI,CAAC,EAAE,OAAO,KAAK;AAC3E;AAYA,MAAM,uBAAuB;AAC7B,MAAM,oBAAoB;AAC1B,MAAM,qBAAqB;AAC3B,MAAM,aAAa;AAEnB,SAAS,SAAS,MAAsB;AACtC,SAAO,WAAW,MAAM,EAAE,OAAO,KAAK,KAAK,EAAE,YAAY,CAAC,EAAE,OAAO,KAAK;AAC1E;AAEO,SAAS,6BAA6B,OAAiC;AAC5E,MAAI,MAAM,SAAS,WAAY,QAAO;AACtC,QAAM,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,MAAM,IAAI,CAAC;AAExD,QAAM,eAAe,oBAAI,IAAoB;AAC7C,QAAM,gBAAgB,oBAAI,IAAoB;AAC9C,aAAW,SAAS,UAAU;AAC5B,UAAM,QAAQ,oBAAI,IAAY;AAC9B,aAAS,IAAI,GAAG,IAAI,KAAK,IAAI,mBAAmB,MAAM,MAAM,GAAG,KAAK;AAClE,YAAM,IAAI,SAAS,MAAM,CAAC,CAAC;AAC3B,UAAI,CAAC,MAAM,IAAI,CAAC,GAAG;AACjB,cAAM,IAAI,CAAC;AACX,qBAAa,IAAI,IAAI,aAAa,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,MACpD;AAAA,IACF;AACA,UAAM,QAAQ,oBAAI,IAAY;AAC9B,aAAS,IAAI,MAAM,SAAS,GAAG,KAAK,KAAK,IAAI,MAAM,SAAS,oBAAoB,CAAC,GAAG,KAAK;AACvF,YAAM,IAAI,SAAS,MAAM,CAAC,CAAC;AAC3B,UAAI,CAAC,MAAM,IAAI,CAAC,GAAG;AACjB,cAAM,IAAI,CAAC;AACX,sBAAc,IAAI,IAAI,cAAc,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,MAAM,SAAS;AACjC,QAAM,aAAa,IAAI,IAAI,CAAC,GAAG,YAAY,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAC9F,QAAM,cAAc,IAAI,IAAI,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,MAAM,KAAK,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAEhG,SAAO,MAAM,IAAI,CAAC,MAAM,MAAM;AAC5B,UAAM,QAAQ,SAAS,CAAC;AACxB,QAAI,OAAO;AACX,WAAO,OAAO,MAAM,WAAW,MAAM,IAAI,EAAE,KAAK,MAAM,MAAM,WAAW,IAAI,SAAS,MAAM,IAAI,CAAC,CAAC,GAAI;AACpG,QAAI,OAAO,MAAM;AACjB,WAAO,OAAO,SAAS,MAAM,OAAO,CAAC,EAAE,KAAK,MAAM,MAAM,YAAY,IAAI,SAAS,MAAM,OAAO,CAAC,CAAC,CAAC,GAAI;AACrG,WAAO,EAAE,KAAK,KAAK,KAAK,UAAU,MAAM,MAAM,MAAM,IAAI,EAAE,KAAK,IAAI,EAAE;AAAA,EACvE,CAAC;AACH;AAEO,SAAS,iBAAiB,OAAoB,QAA+B;AAClF,MAAI,MAAM,UAAU,EAAG,QAAO,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,UAAU,EAAE,SAAS,EAAE;AAErF,QAAM,WAAW,6BAA6B,KAAK;AAGnD,QAAM,eAAe,SAAS,qBAAqB,MAAM,IAAI,CAAC;AAC9D,QAAM,oBAAoB,IAAI,IAAY,YAAY;AAGtD,QAAM,aAAa,SAAS,IAAI,CAAC,UAAU;AAAA,IACzC,KAAK,KAAK;AAAA,IACV,QAAQ,gBAAgB,KAAK,QAAQ;AAAA,EACvC,EAAE;AAGF,QAAM,gBAAgB,oBAAI,IAAoB;AAC9C,aAAW,QAAQ,YAAY;AAC7B,UAAM,aAAa,oBAAI,IAAY;AACnC,eAAW,SAAS,KAAK,QAAQ;AAC/B,YAAM,IAAI,UAAU,KAAK;AACzB,UAAI,CAAC,WAAW,IAAI,CAAC,GAAG;AACtB,mBAAW,IAAI,CAAC;AAChB,sBAAc,IAAI,IAAI,cAAc,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAGA,QAAM,YAAY,MAAM,SAAS;AACjC,aAAW,CAAC,MAAM,KAAK,KAAK,eAAe;AACzC,QAAI,QAAQ,WAAW;AACrB,wBAAkB,IAAI,IAAI;AAAA,IAC5B;AAAA,EACF;AAGA,MAAI,QAAQ;AACV,qBAAiB,QAAQ,MAAM,KAAK,iBAAiB,CAAC;AAAA,EACxD;AAGA,SAAO,WAAW,IAAI,CAAC,SAAS;AAC9B,UAAM,WAAW,KAAK,OAAO,OAAO,CAAC,UAAU,CAAC,kBAAkB,IAAI,UAAU,KAAK,CAAC,CAAC;AACvF,WAAO;AAAA,MACL,KAAK,KAAK;AAAA,MACV,UAAU,SAAS,KAAK,MAAM;AAAA,IAChC;AAAA,EACF,CAAC;AACH;AAEO,SAAS,qBAAqB,QAA0B;AAC7D,QAAM,KAAK,YAAY;AACvB,QAAM,OAAO,GAAG,QAAQ,4DAA4D,EAAE,IAAI,MAAM;AAChG,SAAO,KAAK,IAAI,OAAK,EAAE,UAAU;AACnC;AAEO,SAAS,iBAAiB,QAAgB,QAAwB;AACvE,QAAM,KAAK,YAAY;AACvB,QAAM,MAAM,GAAG,QAAQ,iDAAiD;AACxE,QAAM,SAAS,GAAG;AAAA,IAChB;AAAA,EACF;AACA,QAAM,KAAK,GAAG,YAAY,CAAC,UAAoB;AAC7C,QAAI,IAAI,MAAM;AACd,eAAW,QAAQ,OAAO;AACxB,aAAO,IAAI,QAAQ,MAAM,IAAI;AAAA,IAC/B;AAAA,EACF,CAAC;AACD,KAAG,MAAM;AACX;","names":[]}

package/dist/crawl/etag-incremental.d.ts

@@ -0,0 +1,43 @@
+import type { RawFetchResult } from '../types.js';
+export interface CachedFetchHeaders {
+    etag?: string;
+    lastModified?: string;
+    fetchedAt: string;
+}
+/**
+ * Look up cached ETag + Last-Modified for a URL. Returns null when no row
+ * exists (first crawl, or table missing).
+ */
+export declare function getCachedHeaders(url: string): CachedFetchHeaders | null;
+/**
+ * Persist ETag + Last-Modified from a fresh fetch. Headers are looked up
+ * case-insensitively so both `ETag` and `etag` are accepted.
+ */
+export declare function saveFetchHeaders(url: string, headers: Record<string, string>): void;
+/**
+ * Update only fetched_at on the existing row (no header change).
+ */
+export declare function markFetchedNotModified(url: string): void;
+export interface ConditionalFetchOptions {
+    conditionalHeaders?: {
+        ifNoneMatch?: string;
+        ifModifiedSince?: string;
+    };
+}
+/**
+ * Conditional-fetch wrapper. The wrapped `rawFetchFn` accepts an options
+ * object so this layer can inject `If-None-Match` / `If-Modified-Since` from
+ * the cached crawl_etags row. When the server replies 304 the network
+ * payload is empty and the caller gets `notModified: true` for free; when
+ * the server returns 200 + the same ETag/Last-Modified (legacy fallback)
+ * the wrapper still detects the no-change case.
+ */
+export declare function conditionalFetch(url: string, rawFetchFn: (url: string, opts?: ConditionalFetchOptions) => Promise<RawFetchResult>): Promise<RawFetchResult & {
+    notModified?: boolean;
+}>;
+/**
+ * Test helper: wipe the crawl_etags table. Safe to call when the DB is
+ * uninitialised — it just no-ops.
+ */
+export declare function _clearEtagCacheForTest(): void;
+//# sourceMappingURL=etag-incremental.d.ts.map

package/dist/crawl/etag-incremental.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"etag-incremental.d.ts","sourceRoot":"","sources":["../../src/crawl/etag-incremental.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAIlD,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAevE;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAsBnF;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAQxD;AAED,MAAM,WAAW,uBAAuB;IACtC,kBAAkB,CAAC,EAAE;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,uBAAuB,KAAK,OAAO,CAAC,cAAc,CAAC,GACnF,OAAO,CAAC,cAAc,GAAG;IAAE,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,CA+CrD;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,IAAI,CAO7C"}

package/dist/crawl/etag-incremental.js

@@ -0,0 +1,94 @@
+import { getDatabase } from "../cache/db.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("crawl");
+function getCachedHeaders(url) {
+  try {
+    const db = getDatabase();
+    const row = db.prepare(
+      "SELECT etag, last_modified, fetched_at FROM crawl_etags WHERE url = ?"
+    ).get(url);
+    if (!row) return null;
+    const out = { fetchedAt: row.fetched_at };
+    if (row.etag) out.etag = row.etag;
+    if (row.last_modified) out.lastModified = row.last_modified;
+    return out;
+  } catch (err) {
+    log.debug("getCachedHeaders failed", { url, error: String(err) });
+    return null;
+  }
+}
+function saveFetchHeaders(url, headers) {
+  try {
+    const lower = {};
+    for (const [k, v] of Object.entries(headers)) lower[k.toLowerCase()] = v;
+    const etag = lower["etag"] ?? null;
+    const lastModified = lower["last-modified"] ?? null;
+    const origin = new URL(url).origin;
+    const fetchedAt = (/* @__PURE__ */ new Date()).toISOString();
+    const db = getDatabase();
+    db.prepare(`
+      INSERT INTO crawl_etags (url, origin, etag, last_modified, fetched_at)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(url) DO UPDATE SET
+        origin = excluded.origin,
+        etag = excluded.etag,
+        last_modified = excluded.last_modified,
+        fetched_at = excluded.fetched_at
+    `).run(url, origin, etag, lastModified, fetchedAt);
+  } catch (err) {
+    log.debug("saveFetchHeaders failed", { url, error: String(err) });
+  }
+}
+function markFetchedNotModified(url) {
+  try {
+    const db = getDatabase();
+    db.prepare("UPDATE crawl_etags SET fetched_at = ? WHERE url = ?").run((/* @__PURE__ */ new Date()).toISOString(), url);
+  } catch (err) {
+    log.debug("markFetchedNotModified failed", { url, error: String(err) });
+  }
+}
+async function conditionalFetch(url, rawFetchFn) {
+  const cached = getCachedHeaders(url);
+  const conditionalHeaders = {};
+  if (cached?.etag) conditionalHeaders.ifNoneMatch = cached.etag;
+  if (cached?.lastModified) conditionalHeaders.ifModifiedSince = cached.lastModified;
+  const opts = conditionalHeaders.ifNoneMatch || conditionalHeaders.ifModifiedSince ? { conditionalHeaders } : void 0;
+  const result = await rawFetchFn(url, opts);
+  if (result.statusCode === 304) {
+    markFetchedNotModified(url);
+    return { ...result, notModified: true };
+  }
+  const respHeaders = {};
+  for (const [k, v] of Object.entries(result.headers ?? {})) respHeaders[k.toLowerCase()] = v;
+  const respEtag = respHeaders["etag"];
+  const respLastModified = respHeaders["last-modified"];
+  let notModified = false;
+  if (cached) {
+    if (cached.etag && respEtag && cached.etag === respEtag) notModified = true;
+    else if (!respEtag && cached.lastModified && respLastModified && cached.lastModified === respLastModified) notModified = true;
+  }
+  if (respEtag || respLastModified) {
+    saveFetchHeaders(url, result.headers ?? {});
+  } else if (cached) {
+    markFetchedNotModified(url);
+  } else {
+    saveFetchHeaders(url, result.headers ?? {});
+  }
+  if (notModified) return { ...result, notModified };
+  return result;
+}
+function _clearEtagCacheForTest() {
+  try {
+    const db = getDatabase();
+    db.exec("DELETE FROM crawl_etags");
+  } catch {
+  }
+}
+export {
+  _clearEtagCacheForTest,
+  conditionalFetch,
+  getCachedHeaders,
+  markFetchedNotModified,
+  saveFetchHeaders
+};
+//# sourceMappingURL=etag-incremental.js.map

package/dist/crawl/etag-incremental.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/crawl/etag-incremental.ts"],"sourcesContent":["import { getDatabase } from '../cache/db.js';\nimport { createLogger } from '../logger.js';\nimport type { RawFetchResult } from '../types.js';\n\nconst log = createLogger('crawl');\n\nexport interface CachedFetchHeaders {\n  etag?: string;\n  lastModified?: string;\n  fetchedAt: string;\n}\n\n/**\n * Look up cached ETag + Last-Modified for a URL. Returns null when no row\n * exists (first crawl, or table missing).\n */\nexport function getCachedHeaders(url: string): CachedFetchHeaders | null {\n  try {\n    const db = getDatabase();\n    const row = db.prepare(\n      'SELECT etag, last_modified, fetched_at FROM crawl_etags WHERE url = ?',\n    ).get(url) as { etag: string | null; last_modified: string | null; fetched_at: string } | undefined;\n    if (!row) return null;\n    const out: CachedFetchHeaders = { fetchedAt: row.fetched_at };\n    if (row.etag) out.etag = row.etag;\n    if (row.last_modified) out.lastModified = row.last_modified;\n    return out;\n  } catch (err) {\n    log.debug('getCachedHeaders failed', { url, error: String(err) });\n    return null;\n  }\n}\n\n/**\n * Persist ETag + Last-Modified from a fresh fetch. Headers are looked up\n * case-insensitively so both `ETag` and `etag` are accepted.\n */\nexport function saveFetchHeaders(url: string, headers: Record<string, string>): void {\n  try {\n    const lower: Record<string, string> = {};\n    for (const [k, v] of Object.entries(headers)) lower[k.toLowerCase()] = v;\n    const etag = lower['etag'] ?? null;\n    const lastModified = lower['last-modified'] ?? null;\n    const origin = new URL(url).origin;\n    const fetchedAt = new Date().toISOString();\n\n    const db = getDatabase();\n    db.prepare(`\n      INSERT INTO crawl_etags (url, origin, etag, last_modified, fetched_at)\n      VALUES (?, ?, ?, ?, ?)\n      ON CONFLICT(url) DO UPDATE SET\n        origin = excluded.origin,\n        etag = excluded.etag,\n        last_modified = excluded.last_modified,\n        fetched_at = excluded.fetched_at\n    `).run(url, origin, etag, lastModified, fetchedAt);\n  } catch (err) {\n    log.debug('saveFetchHeaders failed', { url, error: String(err) });\n  }\n}\n\n/**\n * Update only fetched_at on the existing row (no header change).\n */\nexport function markFetchedNotModified(url: string): void {\n  try {\n    const db = getDatabase();\n    db.prepare('UPDATE crawl_etags SET fetched_at = ? WHERE url = ?')\n      .run(new Date().toISOString(), url);\n  } catch (err) {\n    log.debug('markFetchedNotModified failed', { url, error: String(err) });\n  }\n}\n\nexport interface ConditionalFetchOptions {\n  conditionalHeaders?: {\n    ifNoneMatch?: string;\n    ifModifiedSince?: string;\n  };\n}\n\n/**\n * Conditional-fetch wrapper. The wrapped `rawFetchFn` accepts an options\n * object so this layer can inject `If-None-Match` / `If-Modified-Since` from\n * the cached crawl_etags row. When the server replies 304 the network\n * payload is empty and the caller gets `notModified: true` for free; when\n * the server returns 200 + the same ETag/Last-Modified (legacy fallback)\n * the wrapper still detects the no-change case.\n */\nexport async function conditionalFetch(\n  url: string,\n  rawFetchFn: (url: string, opts?: ConditionalFetchOptions) => Promise<RawFetchResult>,\n): Promise<RawFetchResult & { notModified?: boolean }> {\n  const cached = getCachedHeaders(url);\n\n  const conditionalHeaders: ConditionalFetchOptions['conditionalHeaders'] = {};\n  if (cached?.etag) conditionalHeaders.ifNoneMatch = cached.etag;\n  if (cached?.lastModified) conditionalHeaders.ifModifiedSince = cached.lastModified;\n\n  const opts: ConditionalFetchOptions | undefined =\n    conditionalHeaders.ifNoneMatch || conditionalHeaders.ifModifiedSince\n      ? { conditionalHeaders }\n      : undefined;\n\n  const result = await rawFetchFn(url, opts);\n\n  // Server honoured the conditional GET — true short-circuit.\n  if (result.statusCode === 304) {\n    markFetchedNotModified(url);\n    return { ...result, notModified: true };\n  }\n\n  const respHeaders: Record<string, string> = {};\n  for (const [k, v] of Object.entries(result.headers ?? {})) respHeaders[k.toLowerCase()] = v;\n  const respEtag = respHeaders['etag'];\n  const respLastModified = respHeaders['last-modified'];\n\n  // 200 + unchanged validators (server didn't honour If-None-Match).\n  let notModified = false;\n  if (cached) {\n    if (cached.etag && respEtag && cached.etag === respEtag) notModified = true;\n    else if (\n      !respEtag &&\n      cached.lastModified &&\n      respLastModified &&\n      cached.lastModified === respLastModified\n    ) notModified = true;\n  }\n\n  if (respEtag || respLastModified) {\n    saveFetchHeaders(url, result.headers ?? {});\n  } else if (cached) {\n    markFetchedNotModified(url);\n  } else {\n    saveFetchHeaders(url, result.headers ?? {});\n  }\n\n  if (notModified) return { ...result, notModified };\n  return result;\n}\n\n/**\n * Test helper: wipe the crawl_etags table. Safe to call when the DB is\n * uninitialised — it just no-ops.\n */\nexport function _clearEtagCacheForTest(): void {\n  try {\n    const db = getDatabase();\n    db.exec('DELETE FROM crawl_etags');\n  } catch {\n    // DB not initialised — nothing to clear\n  }\n}\n"],"mappings":"AAAA,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAYzB,SAAS,iBAAiB,KAAwC;AACvE,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,UAAM,MAAM,GAAG;AAAA,MACb;AAAA,IACF,EAAE,IAAI,GAAG;AACT,QAAI,CAAC,IAAK,QAAO;AACjB,UAAM,MAA0B,EAAE,WAAW,IAAI,WAAW;AAC5D,QAAI,IAAI,KAAM,KAAI,OAAO,IAAI;AAC7B,QAAI,IAAI,cAAe,KAAI,eAAe,IAAI;AAC9C,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,WAAO;AAAA,EACT;AACF;AAMO,SAAS,iBAAiB,KAAa,SAAuC;AACnF,MAAI;AACF,UAAM,QAAgC,CAAC;AACvC,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,OAAO,EAAG,OAAM,EAAE,YAAY,CAAC,IAAI;AACvE,UAAM,OAAO,MAAM,MAAM,KAAK;AAC9B,UAAM,eAAe,MAAM,eAAe,KAAK;AAC/C,UAAM,SAAS,IAAI,IAAI,GAAG,EAAE;AAC5B,UAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AAEzC,UAAM,KAAK,YAAY;AACvB,OAAG,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAQV,EAAE,IAAI,KAAK,QAAQ,MAAM,cAAc,SAAS;AAAA,EACnD,SAAS,KAAK;AACZ,QAAI,MAAM,2BAA2B,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EAClE;AACF;AAKO,SAAS,uBAAuB,KAAmB;AACxD,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,OAAG,QAAQ,qDAAqD,EAC7D,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,GAAG;AAAA,EACtC,SAAS,KAAK;AACZ,QAAI,MAAM,iCAAiC,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EACxE;AACF;AAiBA,eAAsB,iBACpB,KACA,YACqD;AACrD,QAAM,SAAS,iBAAiB,GAAG;AAEnC,QAAM,qBAAoE,CAAC;AAC3E,MAAI,QAAQ,KAAM,oBAAmB,cAAc,OAAO;AAC1D,MAAI,QAAQ,aAAc,oBAAmB,kBAAkB,OAAO;AAEtE,QAAM,OACJ,mBAAmB,eAAe,mBAAmB,kBACjD,EAAE,mBAAmB,IACrB;AAEN,QAAM,SAAS,MAAM,WAAW,KAAK,IAAI;AAGzC,MAAI,OAAO,eAAe,KAAK;AAC7B,2BAAuB,GAAG;AAC1B,WAAO,EAAE,GAAG,QAAQ,aAAa,KAAK;AAAA,EACxC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,OAAO,WAAW,CAAC,CAAC,EAAG,aAAY,EAAE,YAAY,CAAC,IAAI;AAC1F,QAAM,WAAW,YAAY,MAAM;AACnC,QAAM,mBAAmB,YAAY,eAAe;AAGpD,MAAI,cAAc;AAClB,MAAI,QAAQ;AACV,QAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,SAAU,eAAc;AAAA,aAErE,CAAC,YACD,OAAO,gBACP,oBACA,OAAO,iBAAiB,iBACxB,eAAc;AAAA,EAClB;AAEA,MAAI,YAAY,kBAAkB;AAChC,qBAAiB,KAAK,OAAO,WAAW,CAAC,CAAC;AAAA,EAC5C,WAAW,QAAQ;AACjB,2BAAuB,GAAG;AAAA,EAC5B,OAAO;AACL,qBAAiB,KAAK,OAAO,WAAW,CAAC,CAAC;AAAA,EAC5C;AAEA,MAAI,YAAa,QAAO,EAAE,GAAG,QAAQ,YAAY;AACjD,SAAO;AACT;AAMO,SAAS,yBAA+B;AAC7C,MAAI;AACF,UAAM,KAAK,YAAY;AACvB,OAAG,KAAK,yBAAyB;AAAA,EACnC,QAAQ;AAAA,EAER;AACF;","names":[]}

package/dist/crawl/index-to-vec.d.ts

@@ -0,0 +1,10 @@
+import type { CrawlResultItem } from '../types.js';
+export declare function isIndexingEnabled(): boolean;
+/**
+ * Opt-in: embed (title + first 500 chars of markdown) and upsert into the
+ * vector store. Errors are logged at debug and swallowed so a misbehaving
+ * embed provider can never break a crawl. Disabled by default — gate via
+ * WIGOLO_CRAWL_INDEX=1.
+ */
+export declare function indexCrawlResult(item: CrawlResultItem): Promise<void>;
+//# sourceMappingURL=index-to-vec.d.ts.map

package/dist/crawl/index-to-vec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-to-vec.d.ts","sourceRoot":"","sources":["../../src/crawl/index-to-vec.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAOnD,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAgC3E"}

package/dist/crawl/index-to-vec.js

@@ -0,0 +1,44 @@
+import { createHash } from "crypto";
+import { createLogger } from "../logger.js";
+import { getVectorStore } from "../providers/vector-store.js";
+import { getEmbedProvider } from "../providers/embed-provider.js";
+const log = createLogger("crawl");
+const SUMMARY_CHARS = 500;
+const MIN_TEXT_LEN = 20;
+function isIndexingEnabled() {
+  return process.env.WIGOLO_CRAWL_INDEX === "1";
+}
+async function indexCrawlResult(item) {
+  try {
+    const summary = (item.markdown ?? "").slice(0, SUMMARY_CHARS);
+    const text = `${item.title ?? ""}
+${summary}`.trim();
+    if (text.length < MIN_TEXT_LEN) return;
+    const provider = await getEmbedProvider();
+    const vectors = await provider.embed([text]);
+    if (vectors.length === 0) return;
+    const store = await getVectorStore();
+    const contentHash = createHash("sha256").update(item.markdown ?? "").digest("hex");
+    await store.upsert([
+      {
+        id: item.url,
+        vector: vectors[0],
+        metadata: {
+          url: item.url,
+          contentHash,
+          modelId: provider.modelId
+        }
+      }
+    ]);
+  } catch (err) {
+    log.warn("crawl index-to-vec failed", {
+      url: item.url,
+      error: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
+export {
+  indexCrawlResult,
+  isIndexingEnabled
+};
+//# sourceMappingURL=index-to-vec.js.map

package/dist/crawl/index-to-vec.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/crawl/index-to-vec.ts"],"sourcesContent":["import { createHash } from 'crypto';\nimport { createLogger } from '../logger.js';\nimport { getVectorStore } from '../providers/vector-store.js';\nimport { getEmbedProvider } from '../providers/embed-provider.js';\nimport type { CrawlResultItem } from '../types.js';\n\nconst log = createLogger('crawl');\n\nconst SUMMARY_CHARS = 500;\nconst MIN_TEXT_LEN = 20;\n\nexport function isIndexingEnabled(): boolean {\n  return process.env.WIGOLO_CRAWL_INDEX === '1';\n}\n\n/**\n * Opt-in: embed (title + first 500 chars of markdown) and upsert into the\n * vector store. Errors are logged at debug and swallowed so a misbehaving\n * embed provider can never break a crawl. Disabled by default — gate via\n * WIGOLO_CRAWL_INDEX=1.\n */\nexport async function indexCrawlResult(item: CrawlResultItem): Promise<void> {\n  try {\n    const summary = (item.markdown ?? '').slice(0, SUMMARY_CHARS);\n    const text = `${item.title ?? ''}\\n${summary}`.trim();\n    if (text.length < MIN_TEXT_LEN) return;\n\n    const provider = await getEmbedProvider();\n    const vectors = await provider.embed([text]);\n    if (vectors.length === 0) return;\n\n    const store = await getVectorStore();\n    const contentHash = createHash('sha256')\n      .update(item.markdown ?? '')\n      .digest('hex');\n\n    await store.upsert([\n      {\n        id: item.url,\n        vector: vectors[0],\n        metadata: {\n          url: item.url,\n          contentHash,\n          modelId: provider.modelId,\n        },\n      },\n    ]);\n  } catch (err) {\n    log.warn('crawl index-to-vec failed', {\n      url: item.url,\n      error: err instanceof Error ? err.message : String(err),\n    });\n  }\n}\n"],"mappings":"AAAA,SAAS,kBAAkB;AAC3B,SAAS,oBAAoB;AAC7B,SAAS,sBAAsB;AAC/B,SAAS,wBAAwB;AAGjC,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,gBAAgB;AACtB,MAAM,eAAe;AAEd,SAAS,oBAA6B;AAC3C,SAAO,QAAQ,IAAI,uBAAuB;AAC5C;AAQA,eAAsB,iBAAiB,MAAsC;AAC3E,MAAI;AACF,UAAM,WAAW,KAAK,YAAY,IAAI,MAAM,GAAG,aAAa;AAC5D,UAAM,OAAO,GAAG,KAAK,SAAS,EAAE;AAAA,EAAK,OAAO,GAAG,KAAK;AACpD,QAAI,KAAK,SAAS,aAAc;AAEhC,UAAM,WAAW,MAAM,iBAAiB;AACxC,UAAM,UAAU,MAAM,SAAS,MAAM,CAAC,IAAI,CAAC;AAC3C,QAAI,QAAQ,WAAW,EAAG;AAE1B,UAAM,QAAQ,MAAM,eAAe;AACnC,UAAM,cAAc,WAAW,QAAQ,EACpC,OAAO,KAAK,YAAY,EAAE,EAC1B,OAAO,KAAK;AAEf,UAAM,MAAM,OAAO;AAAA,MACjB;AAAA,QACE,IAAI,KAAK;AAAA,QACT,QAAQ,QAAQ,CAAC;AAAA,QACjB,UAAU;AAAA,UACR,KAAK,KAAK;AAAA,UACV;AAAA,UACA,SAAS,SAAS;AAAA,QACpB;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH,SAAS,KAAK;AACZ,QAAI,KAAK,6BAA6B;AAAA,MACpC,KAAK,KAAK;AAAA,MACV,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AAAA,EACH;AACF;","names":[]}