@astrofoundry/grimoire 3.30.0 → 3.31.0

package/dist/{admin-AKV4CA2O.js → admin-WUNBDKBC.js}

@@ -1,12 +1,372 @@
 import {
+  __commonJS,
+  __toESM,
   bold,
   cyan,
   yellow
-} from "./chunk-BRS6X3AE.js";
+} from "./chunk-R46N6C3C.js";
+
+// node_modules/.pnpm/@joplin+turndown-plugin-gfm@1.0.67/node_modules/@joplin/turndown-plugin-gfm/lib/turndown-plugin-gfm.cjs.js
+var require_turndown_plugin_gfm_cjs = __commonJS({
+  "node_modules/.pnpm/@joplin+turndown-plugin-gfm@1.0.67/node_modules/@joplin/turndown-plugin-gfm/lib/turndown-plugin-gfm.cjs.js"(exports) {
+    "use strict";
+    Object.defineProperty(exports, "__esModule", { value: true });
+    var highlightRegExp = /highlight-(?:text|source)-([a-z0-9]+)/;
+    function highlightedCodeBlock(turndownService) {
+      turndownService.addRule("highlightedCodeBlock", {
+        filter: function(node) {
+          var firstChild = node.firstChild;
+          return node.nodeName === "DIV" && highlightRegExp.test(node.className) && firstChild && firstChild.nodeName === "PRE";
+        },
+        replacement: function(content, node, options) {
+          var className = node.className || "";
+          var language = (className.match(highlightRegExp) || [null, ""])[1];
+          return "\n\n" + options.fence + language + "\n" + node.firstChild.textContent + "\n" + options.fence + "\n\n";
+        }
+      });
+    }
+    function strikethrough(turndownService) {
+      turndownService.addRule("strikethrough", {
+        filter: ["del", "s", "strike"],
+        replacement: function(content) {
+          return "~~" + content + "~~";
+        }
+      });
+    }
+    var indexOf = Array.prototype.indexOf;
+    var every = Array.prototype.every;
+    var rules = {};
+    var alignMap = { left: ":---", right: "---:", center: ":---:" };
+    var isCodeBlock_ = null;
+    var options_ = null;
+    var tableShouldBeSkippedCache_ = /* @__PURE__ */ new WeakMap();
+    function getAlignment(node) {
+      return node ? (node.getAttribute("align") || node.style.textAlign || "").toLowerCase() : "";
+    }
+    function getBorder(alignment) {
+      return alignment ? alignMap[alignment] : "---";
+    }
+    function getColumnAlignment(table, columnIndex) {
+      var votes = {
+        left: 0,
+        right: 0,
+        center: 0,
+        "": 0
+      };
+      var align = "";
+      for (var i = 0; i < table.rows.length; ++i) {
+        var row = table.rows[i];
+        if (columnIndex < row.childNodes.length) {
+          var cellAlignment = getAlignment(row.childNodes[columnIndex]);
+          ++votes[cellAlignment];
+          if (votes[cellAlignment] > votes[align]) {
+            align = cellAlignment;
+          }
+        }
+      }
+      return align;
+    }
+    rules.tableCell = {
+      filter: ["th", "td"],
+      replacement: function(content, node) {
+        if (tableShouldBeSkipped(nodeParentTable(node))) return content;
+        return cell(content, node);
+      }
+    };
+    rules.tableRow = {
+      filter: "tr",
+      replacement: function(content, node) {
+        const parentTable = nodeParentTable(node);
+        if (tableShouldBeSkipped(parentTable)) return content;
+        var borderCells = "";
+        if (isHeadingRow(node)) {
+          const colCount = tableColCount(parentTable);
+          for (var i = 0; i < colCount; i++) {
+            const childNode = i < node.childNodes.length ? node.childNodes[i] : null;
+            var border = getBorder(getColumnAlignment(parentTable, i));
+            borderCells += cell(border, childNode, i);
+          }
+        }
+        return "\n" + content + (borderCells ? "\n" + borderCells : "");
+      }
+    };
+    rules.table = {
+      filter: function(node, options) {
+        return node.nodeName === "TABLE";
+      },
+      replacement: function(content, node) {
+        if (tableShouldBeHtml(node, options_)) {
+          let html = node.outerHTML;
+          let divParent = nodeParentDiv(node);
+          if (divParent === null || !divParent.classList.contains("joplin-table-wrapper")) {
+            return `
+
+<div class="joplin-table-wrapper">${html}</div>
+
+`;
+          } else {
+            return html;
+          }
+        } else {
+          if (tableShouldBeSkipped(node)) return content;
+          content = content.replace(/\n+/g, "\n");
+          var secondLine = content.trim().split("\n");
+          if (secondLine.length >= 2) secondLine = secondLine[1];
+          var secondLineIsDivider = /\| :?---/.test(secondLine);
+          var columnCount = tableColCount(node);
+          var emptyHeader = "";
+          if (columnCount && !secondLineIsDivider) {
+            emptyHeader = "|" + "     |".repeat(columnCount) + "\n|";
+            for (var columnIndex = 0; columnIndex < columnCount; ++columnIndex) {
+              emptyHeader += " " + getBorder(getColumnAlignment(node, columnIndex)) + " |";
+            }
+          }
+          const captionNode = node.querySelector ? node.querySelector("caption") : node.caption;
+          const captionContent = captionNode ? captionNode.textContent || "" : "";
+          const caption = captionContent ? `${captionContent}
+
+` : "";
+          const tableContent = `${emptyHeader}${content}`.trimStart();
+          return `
+
+${caption}${tableContent}
+
+`;
+        }
+      }
+    };
+    rules.tableCaption = {
+      filter: ["caption"],
+      replacement: () => ""
+    };
+    rules.tableColgroup = {
+      filter: ["colgroup", "col"],
+      replacement: () => ""
+    };
+    rules.tableSection = {
+      filter: ["thead", "tbody", "tfoot"],
+      replacement: function(content) {
+        return content;
+      }
+    };
+    function isHeadingRow(tr) {
+      var parentNode = tr.parentNode;
+      return parentNode.nodeName === "THEAD" || parentNode.firstChild === tr && (parentNode.nodeName === "TABLE" || isFirstTbody(parentNode)) && every.call(tr.childNodes, function(n) {
+        return n.nodeName === "TH";
+      });
+    }
+    function isFirstTbody(element) {
+      var previousSibling = element.previousSibling;
+      return element.nodeName === "TBODY" && (!previousSibling || previousSibling.nodeName === "THEAD" && /^\s*$/i.test(previousSibling.textContent));
+    }
+    function cell(content, node = null, index = null) {
+      if (index === null) index = indexOf.call(node.parentNode.childNodes, node);
+      var prefix = " ";
+      if (index === 0) prefix = "| ";
+      let filteredContent = content.trim().replace(/\n\r/g, "<br>").replace(/\n/g, "<br>");
+      filteredContent = filteredContent.replace(/\|+/g, "\\|");
+      while (filteredContent.length < 3) filteredContent += " ";
+      if (node) filteredContent = handleColSpan(filteredContent, node, " ");
+      return prefix + filteredContent + " |";
+    }
+    function nodeContainsTable(node) {
+      if (!node.childNodes) return false;
+      for (let i = 0; i < node.childNodes.length; i++) {
+        const child = node.childNodes[i];
+        if (child.nodeName === "TABLE") return true;
+        if (nodeContainsTable(child)) return true;
+      }
+      return false;
+    }
+    var nodeContains = (node, types) => {
+      if (!node.childNodes) return false;
+      for (let i = 0; i < node.childNodes.length; i++) {
+        const child = node.childNodes[i];
+        if (types === "code" && isCodeBlock_ && isCodeBlock_(child)) return true;
+        if (types.includes(child.nodeName)) return true;
+        if (nodeContains(child, types)) return true;
+      }
+      return false;
+    };
+    var customStyleProperties = [
+      "background-color",
+      "background",
+      "border-color",
+      "border",
+      "border-top",
+      "border-right",
+      "border-bottom",
+      "border-left",
+      "border-style",
+      "border-width",
+      "padding",
+      "padding-top",
+      "padding-right",
+      "padding-bottom",
+      "padding-left",
+      "float",
+      "margin-left",
+      "margin-right"
+    ];
+    var customAttributeNames = [
+      "bgcolor",
+      "bordercolor",
+      "background"
+    ];
+    var nodeHasCustomStyle = (node) => {
+      if (!node || !node.getAttribute) return false;
+      const styleAttr = node.getAttribute("style");
+      if (!styleAttr) return false;
+      const properties = styleAttr.split(";").map((s) => s.split(":")[0].trim().toLowerCase()).filter((s) => s.length > 0);
+      for (let i = 0; i < properties.length; i++) {
+        if (customStyleProperties.includes(properties[i])) return true;
+      }
+      return false;
+    };
+    var hasNonDefaultSpacingAttribute = (node, name) => {
+      if (!node || !node.getAttribute) return false;
+      const value = node.getAttribute(name);
+      if (value === null) return false;
+      const normalisedValue = `${value}`.trim().toLowerCase();
+      if (!normalisedValue) return false;
+      if (normalisedValue === "0" || normalisedValue === "0px") return false;
+      return true;
+    };
+    var nodeHasCustomAttributes = (node) => {
+      if (!node || !node.getAttribute) return false;
+      for (let i = 0; i < customAttributeNames.length; i++) {
+        const value = node.getAttribute(customAttributeNames[i]);
+        if (value !== null && `${value}`.trim() !== "") return true;
+      }
+      if (node.nodeName === "TABLE") {
+        if (hasNonDefaultSpacingAttribute(node, "cellpadding")) return true;
+        if (hasNonDefaultSpacingAttribute(node, "cellspacing")) return true;
+      }
+      return false;
+    };
+    var nodeHasCustomFormatting = (node) => {
+      return nodeHasCustomStyle(node) || nodeHasCustomAttributes(node);
+    };
+    var tableHasCustomStyles = (tableNode) => {
+      if (nodeHasCustomFormatting(tableNode)) return true;
+      const rows = tableNode.rows;
+      if (!rows) return false;
+      for (let i = 0; i < rows.length; i++) {
+        const row = rows[i];
+        if (nodeHasCustomFormatting(row)) return true;
+        for (let j = 0; j < row.childNodes.length; j++) {
+          const cell2 = row.childNodes[j];
+          if ((cell2.nodeName === "TD" || cell2.nodeName === "TH") && nodeHasCustomFormatting(cell2)) {
+            return true;
+          }
+        }
+      }
+      return false;
+    };
+    var tableShouldBeHtml = (tableNode, options) => {
+      const possibleTags = [
+        "UL",
+        "OL",
+        "H1",
+        "H2",
+        "H3",
+        "H4",
+        "H5",
+        "H6",
+        "HR",
+        "BLOCKQUOTE"
+      ];
+      if (options.preserveNestedTables) possibleTags.push("TABLE");
+      return nodeContains(tableNode, "code") || nodeContains(tableNode, possibleTags) || options.preserveTableStyles && tableHasCustomStyles(tableNode);
+    };
+    function tableShouldBeSkipped(tableNode) {
+      const cached = tableShouldBeSkippedCache_.get(tableNode);
+      if (cached !== void 0) return cached;
+      const result = tableShouldBeSkipped_(tableNode);
+      tableShouldBeSkippedCache_.set(tableNode, result);
+      return result;
+    }
+    function tableShouldBeSkipped_(tableNode) {
+      if (!tableNode) return true;
+      if (!tableNode.rows) return true;
+      if (tableNode.rows.length === 1 && tableNode.rows[0].childNodes.length <= 1) return true;
+      if (nodeContainsTable(tableNode)) return true;
+      return false;
+    }
+    function nodeParentDiv(node) {
+      let parent = node.parentNode;
+      while (parent.nodeName !== "DIV") {
+        parent = parent.parentNode;
+        if (!parent) return null;
+      }
+      return parent;
+    }
+    function nodeParentTable(node) {
+      let parent = node.parentNode;
+      while (parent.nodeName !== "TABLE") {
+        parent = parent.parentNode;
+        if (!parent) return null;
+      }
+      return parent;
+    }
+    function handleColSpan(content, node, emptyChar) {
+      const colspan = node.getAttribute("colspan") || 1;
+      for (let i = 1; i < colspan; i++) {
+        content += " | " + emptyChar.repeat(3);
+      }
+      return content;
+    }
+    function tableColCount(node) {
+      let maxColCount = 0;
+      for (let i = 0; i < node.rows.length; i++) {
+        const row = node.rows[i];
+        const colCount = row.childNodes.length;
+        if (colCount > maxColCount) maxColCount = colCount;
+      }
+      return maxColCount;
+    }
+    function tables2(turndownService) {
+      isCodeBlock_ = turndownService.isCodeBlock;
+      options_ = turndownService.options;
+      turndownService.keep(function(node) {
+        if (node.nodeName === "TABLE" && tableShouldBeHtml(node, turndownService.options)) return true;
+        return false;
+      });
+      for (var key in rules) turndownService.addRule(key, rules[key]);
+    }
+    function taskListItems(turndownService) {
+      turndownService.addRule("taskListItems", {
+        filter: function(node) {
+          const parent = node.parentNode;
+          const grandparent = parent.parentNode;
+          const grandparentIsListItem = !!grandparent && grandparent.nodeName === "LI";
+          return (node.type === "checkbox" || node.getAttribute("role") === "checkbox") && (parent.nodeName === "LI" || parent.nodeName === "LABEL" && grandparentIsListItem || parent.nodeName === "SPAN" && grandparentIsListItem);
+        },
+        replacement: function(content, node) {
+          const checked = node.nodeName === "INPUT" ? node.checked : node.getAttribute("aria-checked") === "true";
+          return (checked ? "[x]" : "[ ]") + " ";
+        }
+      });
+    }
+    function gfm(turndownService) {
+      turndownService.use([
+        highlightedCodeBlock,
+        strikethrough,
+        tables2,
+        taskListItems
+      ]);
+    }
+    exports.gfm = gfm;
+    exports.highlightedCodeBlock = highlightedCodeBlock;
+    exports.strikethrough = strikethrough;
+    exports.tables = tables2;
+    exports.taskListItems = taskListItems;
+  }
+});
 
 // src/admin.ts
 import { parseArgs } from "node:util";
-import { readFile as readFile3, writeFile as writeFile4, readdir, rm, mkdir as mkdir4, unlink } from "node:fs/promises";
+import { readFile as readFile3, writeFile as writeFile4, readdir, rm } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join as join4, resolve } from "node:path";
 import { createInterface } from "node:readline";
@@ -107,10 +467,14 @@ async function loadConfig(path) {
 import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { chromium } from "playwright";
+
+// src/slug.ts
 function slugifyUrl(url) {
   const parsed = new URL(url);
   return parsed.pathname.replace(/^\//, "").replace(/\/$/, "").replace(/\//g, "-").replace(/[^a-zA-Z0-9-]/g, "");
 }
+
+// src/scraper.ts
 function filterUrls(urls, includePatterns, excludePatterns) {
   let filtered = urls.filter(
     (url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#")
@@ -267,6 +631,7 @@ async function createBrowser() {
 }
 
 // src/converter.ts
+var import_turndown_plugin_gfm = __toESM(require_turndown_plugin_gfm_cjs(), 1);
 import { readFile as readFile2, writeFile as writeFile2, mkdir as mkdir2 } from "node:fs/promises";
 import { join as join2 } from "node:path";
 import { JSDOM } from "jsdom";
@@ -276,6 +641,7 @@ var turndown = new TurndownService({
   codeBlockStyle: "fenced",
   bulletListMarker: "-"
 });
+turndown.use(import_turndown_plugin_gfm.tables);
 var GENERIC_REMOVE = [
   "style",
   "script",
@@ -293,26 +659,57 @@ function cleanMarkdown(md, textPatterns) {
   }
   return cleaned.trim();
 }
+function flattenTableCellMarkup(contentEl) {
+  for (const table of contentEl.querySelectorAll("table")) {
+    if (!table.isConnected) continue;
+    for (const inner of table.querySelectorAll("table")) {
+      if (!inner.isConnected) continue;
+      const rows = [...inner.rows].map(
+        (row) => [...row.cells].map((c) => c.textContent?.replace(/\s+/g, " ").trim() ?? "").filter(Boolean).join(" ")
+      ).filter(Boolean);
+      inner.replaceWith(rows.join("; "));
+    }
+    for (const code of table.querySelectorAll("code")) {
+      code.replaceWith(code.textContent ?? "");
+    }
+    for (const list of table.querySelectorAll("ul, ol")) {
+      if (!list.isConnected) continue;
+      const items = [...list.querySelectorAll("li")].map((li) => li.textContent?.replace(/\s+/g, " ").trim() ?? "").filter(Boolean);
+      list.replaceWith(items.join("; "));
+    }
+    for (const block of table.querySelectorAll("aside, blockquote, h1, h2, h3, h4, h5, h6")) {
+      if (!block.isConnected) continue;
+      block.replaceWith(block.textContent?.replace(/\s+/g, " ").trim() ?? "");
+    }
+  }
+}
+function cleanTableRowBreaks(md) {
+  return md.split("\n").map(
+    (line) => line.startsWith("|") ? line.replace(/\s*<br\s*\/?>\s*/g, "; ") : line
+  ).join("\n");
+}
 function extractContent(html, contentSelector, removeSelectors, removeTextPatterns) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
-  const contentEl = doc.querySelector(contentSelector);
-  if (!contentEl) {
-    return cleanMarkdown(turndown.turndown(doc.body.innerHTML), removeTextPatterns);
-  }
+  const contentEl = doc.querySelector(contentSelector) ?? doc.body;
   const allSelectors = [...GENERIC_REMOVE, ...removeSelectors ?? []];
   for (const selector of allSelectors) {
     for (const el of contentEl.querySelectorAll(selector)) {
       el.remove();
     }
   }
-  return cleanMarkdown(turndown.turndown(contentEl.innerHTML), removeTextPatterns);
+  flattenTableCellMarkup(contentEl);
+  return cleanMarkdown(
+    cleanTableRowBreaks(turndown.turndown(contentEl.innerHTML)),
+    removeTextPatterns
+  );
 }
 function extractTitle(html) {
   const dom = new JSDOM(html);
-  const titleEl = dom.window.document.querySelector("title");
-  if (!titleEl) return "Untitled";
-  return titleEl.textContent?.replace(/\s*[|–—-]\s*.+$/, "").trim() ?? "Untitled";
+  const raw = dom.window.document.querySelector("title")?.textContent?.replace(/\s+/g, " ").trim();
+  if (!raw) return "Untitled";
+  const stripped = raw.replace(/\s*\|[^|]*$/, "").replace(/\s*[\u2013\u2014][^\u2013\u2014]*$/, "").replace(/\s+-\s+(?!.*\s-\s)[^|]*$/, "").trim();
+  return stripped || raw;
 }
 function buildFrontmatter(source, url, title) {
   return [
@@ -379,6 +776,13 @@ function buildChunkId(source, url, headingSlug, index) {
   const base = `${prefix}${truncatedSlug}`;
   return index !== void 0 ? `${base}-${index}` : base;
 }
+function buildEmbedText(chunk) {
+  const path = chunk.heading_path[0] === chunk.title ? chunk.heading_path.slice(1) : chunk.heading_path;
+  const context = [chunk.title, ...path].filter(Boolean).join(" > ");
+  return context ? `${context}
+
+${chunk.content}` : chunk.content;
+}
 function parseHeadingSections(markdown) {
   const lines = markdown.split("\n");
   const sections = [];
@@ -431,26 +835,135 @@ function parseHeadingSections(markdown) {
   }
   return sections;
 }
-function splitAtParagraphBoundaries(text, maxTokens) {
-  const paragraphs = text.split(/\n\n+/);
-  const parts = [];
+function isTableLine(line) {
+  return line.trimStart().startsWith("|");
+}
+function isTableSeparator(line) {
+  return /^\s*\|?[\s:|-]+\|?\s*$/.test(line) && line.includes("-");
+}
+function parseBlocks(lines) {
+  const blocks = [];
   let current = [];
-  let currentTokens = 0;
-  for (const para of paragraphs) {
-    const paraTokens = estimateTokens(para);
-    if (currentTokens + paraTokens > maxTokens && current.length > 0) {
-      parts.push(current.join("\n\n"));
-      current = [para];
-      currentTokens = paraTokens;
+  let kind = "paragraph";
+  let inFence = false;
+  let fenceMarker = "";
+  function flush() {
+    while (current.length > 0 && current[current.length - 1].trim() === "") {
+      current.pop();
+    }
+    if (current.length > 0) {
+      blocks.push({ kind, lines: current });
+    }
+    current = [];
+    kind = "paragraph";
+  }
+  for (const line of lines) {
+    if (inFence) {
+      current.push(line);
+      const fenceMatch2 = line.match(/^\s*(```+|~~~+)/);
+      if (fenceMatch2 && fenceMatch2[1][0] === fenceMarker) {
+        inFence = false;
+        flush();
+      }
+      continue;
+    }
+    const fenceMatch = line.match(/^\s*(```+|~~~+)/);
+    if (fenceMatch) {
+      flush();
+      kind = "fence";
+      inFence = true;
+      fenceMarker = fenceMatch[1][0];
+      current.push(line);
+      continue;
+    }
+    if (isTableLine(line)) {
+      if (kind !== "table") {
+        flush();
+        kind = "table";
+      }
+      current.push(line);
+      continue;
+    }
+    if (kind === "table") {
+      flush();
+    }
+    if (line.trim() === "") {
+      flush();
     } else {
-      current.push(para);
-      currentTokens += paraTokens;
+      current.push(line);
     }
   }
-  if (current.length > 0) {
-    parts.push(current.join("\n\n"));
+  flush();
+  return blocks;
+}
+function groupLines(lines, budget) {
+  const groups = [];
+  let current = [];
+  let tokens = 0;
+  for (const line of lines) {
+    const lineTokens = estimateTokens(line) + 1;
+    if (tokens + lineTokens > budget && current.length > 0) {
+      groups.push(current);
+      current = [];
+      tokens = 0;
+    }
+    current.push(line);
+    tokens += lineTokens;
+  }
+  if (current.length > 0) groups.push(current);
+  return groups;
+}
+function splitBlock(block, budget) {
+  if (block.kind === "table") {
+    const hasHeader = block.lines.length >= 2 && isTableSeparator(block.lines[1]);
+    const header = hasHeader ? block.lines.slice(0, 2) : [];
+    const rows = hasHeader ? block.lines.slice(2) : block.lines;
+    const headerTokens = estimateTokens(header.join("\n"));
+    return groupLines(rows, Math.max(budget - headerTokens, 50)).map(
+      (group) => [...header, ...group].join("\n")
+    );
+  }
+  if (block.kind === "fence") {
+    const opening = block.lines[0];
+    const closing = block.lines[block.lines.length - 1].match(/^\s*(```+|~~~+)\s*$/) ? block.lines[block.lines.length - 1] : opening.match(/^\s*(```+|~~~+)/)[1];
+    const body = block.lines.slice(1, block.lines[block.lines.length - 1] === closing ? -1 : void 0);
+    const frameTokens = estimateTokens(opening) + estimateTokens(closing) + 2;
+    return groupLines(body, Math.max(budget - frameTokens, 50)).map(
+      (group) => [opening, ...group, closing].join("\n")
+    );
+  }
+  return groupLines(block.lines, budget).map((group) => group.join("\n"));
+}
+function splitSectionIntoParts(blocks, headingLine, maxTokens) {
+  const budget = Math.max(maxTokens - estimateTokens(headingLine), 100);
+  const parts = [];
+  let current = [];
+  let tokens = 0;
+  function flush() {
+    if (current.length > 0) {
+      parts.push(current);
+      current = [];
+      tokens = 0;
+    }
+  }
+  for (const block of blocks) {
+    const text = block.lines.join("\n");
+    const blockTokens = estimateTokens(text) + 2;
+    if (blockTokens > budget) {
+      flush();
+      for (const piece of splitBlock(block, budget)) {
+        parts.push([piece]);
+      }
+      continue;
+    }
+    if (tokens + blockTokens > budget) {
+      flush();
+    }
+    current.push(text);
+    tokens += blockTokens;
   }
-  return parts;
+  flush();
+  return parts.map((blockTexts) => headingLine + blockTexts.join("\n\n"));
 }
 function stripFrontmatter(markdown) {
   if (markdown.startsWith("---")) {
@@ -484,11 +997,11 @@ function chunkMarkdown(markdown, source, url, title) {
     const headingLine = section.heading ? `${"#".repeat(section.level)} ${section.heading}
 
 ` : "";
-    const content = headingLine + section.lines.join("\n").trim();
+    const body = section.lines.join("\n").trim();
+    const content = headingLine + body;
     if (!content.trim()) continue;
     const headingSlug = section.heading ? slugifyHeading(section.heading) : "intro";
-    const tokens = estimateTokens(content);
-    if (tokens <= MAX_TOKENS) {
+    if (estimateTokens(content) <= MAX_TOKENS) {
       chunks.push({
         id: uniqueId(headingSlug),
         source,
@@ -496,29 +1009,56 @@ function chunkMarkdown(markdown, source, url, title) {
         title,
         heading_path: section.headingPath,
         content,
-        token_count: tokens
+        token_count: estimateTokens(content)
+      });
+      continue;
+    }
+    const blocks = parseBlocks(body.split("\n"));
+    const parts = splitSectionIntoParts(blocks, headingLine, MAX_TOKENS);
+    for (let i = 0; i < parts.length; i++) {
+      const partContent = parts[i].trim();
+      if (!partContent) continue;
+      const partSlug = parts.length > 1 ? `${headingSlug}-${i}` : headingSlug;
+      chunks.push({
+        id: uniqueId(partSlug),
+        source,
+        url,
+        title,
+        heading_path: section.headingPath,
+        content: partContent,
+        token_count: estimateTokens(partContent)
       });
-    } else {
-      const parts = splitAtParagraphBoundaries(content, MAX_TOKENS);
-      for (let i = 0; i < parts.length; i++) {
-        const partContent = parts[i].trim();
-        if (!partContent) continue;
-        const partSlug = parts.length > 1 ? `${headingSlug}-${i}` : headingSlug;
-        chunks.push({
-          id: uniqueId(partSlug),
-          source,
-          url,
-          title,
-          heading_path: section.headingPath,
-          content: partContent,
-          token_count: estimateTokens(partContent)
-        });
-      }
     }
   }
   return chunks;
 }
 
+// src/tokens.ts
+import { createHash } from "node:crypto";
+var IDENTIFIER_PATTERN = /(?<![A-Za-z0-9._])(?:[A-Za-z][A-Za-z0-9]*(?:[_.-][A-Za-z0-9]+)+|[a-z][a-z0-9]*(?:[A-Z][a-z0-9]*)+|(?:[A-Z][a-z0-9]+){2,})(?![A-Za-z0-9])/g;
+var MIN_TOKEN_LENGTH = 4;
+var MAX_TOKEN_LENGTH = 80;
+var MAX_TOKENS_PER_CHUNK = 100;
+function normalizeForTokens(text) {
+  return text.replace(/\]\([^)]*\)/g, "]").replace(/https?:\/\/\S+/g, " ").replace(/\\([_*[\]()#`~-])/g, "$1");
+}
+function extractIdentifierTokens(text, limit = MAX_TOKENS_PER_CHUNK) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const match of normalizeForTokens(text).matchAll(IDENTIFIER_PATTERN)) {
+    const token = match[0].toLowerCase();
+    if (token.length < MIN_TOKEN_LENGTH || token.length > MAX_TOKEN_LENGTH) continue;
+    seen.add(token);
+    if (seen.size >= limit) break;
+  }
+  return [...seen];
+}
+function extractQueryTokens(query) {
+  return extractIdentifierTokens(query, 5);
+}
+function contentHash(text) {
+  return createHash("sha256").update(text).digest("hex");
+}
+
 // src/embedder.ts
 import { GoogleGenerativeAI } from "@google/generative-ai";
 var BATCH_SIZE = 50;
@@ -528,7 +1068,6 @@ var MAX_RETRIES = 5;
 var RATE_LIMIT_BASE_DELAY_MS = 6e4;
 var NETWORK_BASE_DELAY_MS = 1e4;
 var BATCH_DELAY_MS = 2500;
-var DEFAULT_CHECKPOINT_EVERY_BATCHES = 20;
 var NETWORK_ERROR_PATTERNS = [
   "fetch failed",
   "ECONNRESET",
@@ -562,22 +1101,10 @@ function classifyError(message) {
 async function embedTexts(texts, options = {}) {
   const client = getClient();
   const model = client.getGenerativeModel({ model: MODEL });
-  const { onProgress, onCheckpoint, resumeFrom } = options;
-  const checkpointEveryBatches = options.checkpointEveryBatches ?? DEFAULT_CHECKPOINT_EVERY_BATCHES;
+  const { onProgress } = options;
   const maxRetries = options.maxRetries ?? MAX_RETRIES;
-  const embeddings = resumeFrom ? [...resumeFrom] : [];
-  const startIndex = Math.floor(embeddings.length / BATCH_SIZE) * BATCH_SIZE;
-  if (embeddings.length > startIndex) {
-    embeddings.length = startIndex;
-  }
-  if (startIndex > 0) {
-    console.log(`  Resuming from chunk ${startIndex} of ${texts.length} (${embeddings.length} cached).`);
-  }
-  if (startIndex >= texts.length) {
-    return embeddings.slice(0, texts.length);
-  }
-  let batchesSinceCheckpoint = 0;
-  for (let i = startIndex; i < texts.length; i += BATCH_SIZE) {
+  const embeddings = [];
+  for (let i = 0; i < texts.length; i += BATCH_SIZE) {
     const batch = texts.slice(i, i + BATCH_SIZE);
     const batchNumber = i / BATCH_SIZE + 1;
     let result;
@@ -609,18 +1136,10 @@ async function embedTexts(texts, options = {}) {
       embeddings.push(embedding.values);
     }
     onProgress?.(Math.min(i + BATCH_SIZE, texts.length), texts.length);
-    batchesSinceCheckpoint++;
-    if (onCheckpoint && batchesSinceCheckpoint >= checkpointEveryBatches && i + BATCH_SIZE < texts.length) {
-      await onCheckpoint(embeddings);
-      batchesSinceCheckpoint = 0;
-    }
     if (i + BATCH_SIZE < texts.length) {
       await new Promise((resolve2) => setTimeout(resolve2, BATCH_DELAY_MS));
     }
   }
-  if (onCheckpoint) {
-    await onCheckpoint(embeddings);
-  }
   return embeddings;
 }
 async function embedText(text) {
@@ -701,6 +1220,8 @@ async function storeChunks(chunks, embeddings, onProgress) {
         heading_path: chunk.heading_path,
         content: chunk.content,
         token_count: chunk.token_count,
+        tokens: extractIdentifierTokens(chunk.content),
+        content_hash: contentHash(buildEmbedText(chunk)),
         embedded_at: (/* @__PURE__ */ new Date()).toISOString(),
         embedding: FieldValue.vector(embSlice[j])
       });
@@ -767,10 +1288,38 @@ async function deleteChunksByIds(ids, onProgress) {
     onProgress?.(Math.min(i + BATCH_SIZE2, ids.length), ids.length);
   }
 }
-async function getSourceChunkIds(sourceName) {
+async function getSourceChunkHashes(sourceName) {
+  const col = chunksCol();
+  const snapshot = await col.where("source", "==", sourceName).select("content_hash").get();
+  return new Map(
+    snapshot.docs.map((doc) => [doc.id, doc.data().content_hash ?? ""])
+  );
+}
+var TOKEN_QUERY_LIMIT = 100;
+async function tokenSearch(tokens, source) {
+  if (tokens.length === 0) return [];
   const col = chunksCol();
-  const snapshot = await col.where("source", "==", sourceName).select().get();
-  return new Set(snapshot.docs.map((doc) => doc.id));
+  const hits = /* @__PURE__ */ new Map();
+  const snapshots = await Promise.all(
+    tokens.map((token) => {
+      let query = col.where("tokens", "array-contains", token);
+      if (source) {
+        query = query.where("source", "==", source);
+      }
+      return query.limit(TOKEN_QUERY_LIMIT).get();
+    })
+  );
+  for (const snapshot of snapshots) {
+    for (const doc of snapshot.docs) {
+      const existing = hits.get(doc.id);
+      if (existing) {
+        existing.matchedTokens++;
+      } else {
+        hits.set(doc.id, { id: doc.id, data: doc.data(), matchedTokens: 1 });
+      }
+    }
+  }
+  return [...hits.values()].sort((a, b) => b.matchedTokens - a.matchedTokens);
 }
 async function vectorSearch(queryEmbedding, limit, source) {
   const col = chunksCol();
@@ -816,47 +1365,92 @@ async function rerank(query, documents, topN = 5) {
 }
 
 // src/search.ts
+var DEFAULT_CANDIDATES = 50;
+var RRF_K = 60;
 function hasReranker() {
   return !!process.env.RERANKER_URL;
 }
+function isVectorLike(value) {
+  return typeof value === "object" && value !== null && typeof value.toArray === "function";
+}
+function cosineSimilarity(a, b) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+function contextualText(data) {
+  return buildEmbedText({
+    title: data.title,
+    heading_path: data.heading_path,
+    content: data.content
+  });
+}
+function toSearchResult(candidate, relevance) {
+  const data = candidate.data;
+  return {
+    id: candidate.id,
+    source: data.source,
+    url: data.url,
+    title: data.title,
+    heading_path: data.heading_path,
+    content: data.content,
+    relevance_score: relevance
+  };
+}
 async function search(query, options = {}) {
-  const { source, candidates = 20, topN = 5 } = options;
+  const { source, candidates = DEFAULT_CANDIDATES, topN = 5 } = options;
   const queryEmbedding = await embedText(query);
-  const rawResults = await vectorSearch(queryEmbedding, candidates, source);
-  if (rawResults.length === 0) return [];
+  const [vectorResults, lexicalHits] = await Promise.all([
+    vectorSearch(queryEmbedding, candidates, source),
+    tokenSearch(extractQueryTokens(query), source)
+  ]);
+  const pool = /* @__PURE__ */ new Map();
+  vectorResults.forEach((result, rank) => {
+    pool.set(result.id, {
+      id: result.id,
+      data: result.data,
+      fusedScore: 1 / (RRF_K + rank + 1),
+      similarity: 1 - result.distance
+    });
+  });
+  lexicalHits.forEach((hit, rank) => {
+    const lexicalScore = 1 / (RRF_K + rank + 1);
+    const existing = pool.get(hit.id);
+    if (existing) {
+      existing.fusedScore += lexicalScore;
+      return;
+    }
+    const embedding = hit.data.embedding;
+    delete hit.data.embedding;
+    pool.set(hit.id, {
+      id: hit.id,
+      data: hit.data,
+      fusedScore: lexicalScore,
+      similarity: isVectorLike(embedding) ? cosineSimilarity(queryEmbedding, embedding.toArray()) : null
+    });
+  });
+  const fused = [...pool.values()].sort((a, b) => b.fusedScore - a.fusedScore);
+  if (fused.length === 0) return [];
   if (hasReranker()) {
-    const documents = rawResults.map((r) => r.data.content);
+    const rerankPool = fused.slice(0, candidates);
+    const documents = rerankPool.map((c) => contextualText(c.data));
     const reranked = await rerank(query, documents, topN);
-    return reranked.map((r) => {
-      const original = rawResults[r.index];
-      const data = original.data;
-      return {
-        id: original.id,
-        source: data.source,
-        url: data.url,
-        title: data.title,
-        heading_path: data.heading_path,
-        content: data.content,
-        relevance_score: r.relevance_score
-      };
-    });
+    return reranked.map((r) => toSearchResult(rerankPool[r.index], r.relevance_score));
   }
-  return rawResults.slice(0, topN).map((r) => {
-    const data = r.data;
-    return {
-      id: r.id,
-      source: data.source,
-      url: data.url,
-      title: data.title,
-      heading_path: data.heading_path,
-      content: data.content,
-      relevance_score: Math.max(0, 1 - r.distance / 2)
-    };
+  return fused.slice(0, topN).map((candidate) => {
+    const similarity = candidate.similarity ?? 0;
+    return toSearchResult(candidate, Math.max(0, (1 + similarity) / 2));
   });
 }
 
 // src/apikey.ts
-import { randomBytes, createHash } from "node:crypto";
+import { randomBytes, createHash as createHash2 } from "node:crypto";
 import {
   getFirestore as getFirestore2
 } from "firebase-admin/firestore";
@@ -872,7 +1466,7 @@ function getDb2() {
   return db2;
 }
 function hashKey(key) {
-  return createHash("sha256").update(key).digest("hex");
+  return createHash2("sha256").update(key).digest("hex");
 }
 function apiKeysCol() {
   return getDb2().collection("grimoire_api_keys");
@@ -1166,89 +1760,85 @@ Source "${name}" added to config/sources.yaml`);
     await browser.close();
   }
 }
-var EMBEDDINGS_PART_SIZE = 1e4;
-function partPath(cachePath, index) {
-  return `${cachePath}.part-${String(index).padStart(4, "0")}`;
-}
-async function readPartFiles(cachePath) {
-  const dir = cachePath.slice(0, cachePath.lastIndexOf("/"));
-  const base = cachePath.slice(cachePath.lastIndexOf("/") + 1);
-  const entries = await readdir(dir).catch(() => []);
-  return entries.filter((e) => e.startsWith(`${base}.part-`)).sort().map((e) => join4(dir, e));
-}
-async function loadEmbeddingsCache(cachePath) {
-  const parts = await readPartFiles(cachePath);
-  if (parts.length > 0) {
-    const all = [];
-    for (const p of parts) {
-      const data = await readFile3(p, "utf-8");
-      const arr = JSON.parse(data);
-      for (const row of arr) all.push(row);
-    }
-    return all;
+var EMBED_WINDOW = 1e3;
+async function syncChunks(sourceName, allChunks, urlCount, version) {
+  console.log("  Comparing with Firestore...");
+  const existing = await getSourceChunkHashes(sourceName);
+  const currentIds = new Set(allChunks.map((c) => c.id));
+  const toDelete = [...existing.keys()].filter((id) => !currentIds.has(id));
+  const toEmbed = allChunks.filter(
+    (chunk) => existing.get(chunk.id) !== contentHash(buildEmbedText(chunk))
+  );
+  console.log(
+    `  Sync: ${toEmbed.length} to embed, ${allChunks.length - toEmbed.length} unchanged, ${toDelete.length} to delete.`
+  );
+  if (toDelete.length > 0) {
+    await deleteChunksByIds(toDelete, (cur, total) => {
+      console.log(`  [${cur}/${total}] deleted`);
+    });
   }
-  try {
-    const data = await readFile3(cachePath, "utf-8");
-    return JSON.parse(data);
-  } catch {
-    return null;
+  for (let i = 0; i < toEmbed.length; i += EMBED_WINDOW) {
+    const window = toEmbed.slice(i, i + EMBED_WINDOW);
+    const embeddings = await embedTexts(window.map((c) => buildEmbedText(c)), {
+      onProgress: (done) => {
+        console.log(`  [${i + done}/${toEmbed.length}] embedded`);
+      }
+    });
+    await storeChunks(window, embeddings, (cur) => {
+      console.log(`  [${i + cur}/${toEmbed.length}] stored`);
+    });
   }
+  await updateSourceMeta(sourceName, allChunks.length, urlCount, version);
+  console.log(`  Done. ${allChunks.length} chunks live for "${sourceName}".`);
 }
-async function saveEmbeddingsCache(cachePath, embeddings) {
-  await unlink(cachePath).catch(() => {
-  });
-  const existingParts = await readPartFiles(cachePath);
-  for (const p of existingParts) {
-    await unlink(p).catch(() => {
+async function readCachedMarkdownPages(mdDir) {
+  const mdFiles = await readdir(mdDir).catch(() => []);
+  const pages = [];
+  for (const f of mdFiles.filter((f2) => f2.endsWith(".md"))) {
+    const content = await readFile3(join4(mdDir, f), "utf-8");
+    const urlMatch = content.match(/^url: "(.+)"$/m);
+    const titleMatch = content.match(/^title: "(.+)"$/m);
+    if (!urlMatch) {
+      console.warn(`  WARNING: ${f} has no url in frontmatter, skipping page.`);
+      continue;
+    }
+    pages.push({
+      markdown: content,
+      url: urlMatch[1],
+      title: titleMatch?.[1] ?? "Untitled"
     });
   }
-  for (let i = 0; i < embeddings.length; i += EMBEDDINGS_PART_SIZE) {
-    const part = embeddings.slice(i, i + EMBEDDINGS_PART_SIZE);
-    const index = Math.floor(i / EMBEDDINGS_PART_SIZE);
-    await writeFile4(partPath(cachePath, index), JSON.stringify(part), "utf-8");
-  }
-}
-async function embedWithCheckpoint(texts, rawDir, embeddingsCachePath) {
-  await mkdir4(rawDir, { recursive: true });
-  const partialCache = await loadEmbeddingsCache(embeddingsCachePath);
-  const resumeFrom = partialCache && partialCache.length > 0 && partialCache.length < texts.length ? partialCache : void 0;
-  return embedTexts(texts, {
-    onProgress: (done, total) => {
-      console.log(`  [${done}/${total}] embedded`);
-    },
-    onCheckpoint: async (current) => {
-      await saveEmbeddingsCache(embeddingsCachePath, current);
-    },
-    resumeFrom
-  });
+  return pages;
 }
-async function storeWithStrategy(sourceName, allChunks, embeddings, urlCount, version, diff) {
-  if (diff) {
-    console.log("  Computing diff...");
-    const existingIds = await getSourceChunkIds(sourceName);
-    const newIds = new Set(allChunks.map((c) => c.id));
-    const toDelete = [...existingIds].filter((id) => !newIds.has(id));
-    console.log(`  Diff: ${toDelete.length} to delete, ${allChunks.length} to upsert (${existingIds.size} existing)`);
-    if (toDelete.length > 0) {
-      console.log("  Deleting removed chunks...");
-      await deleteChunksByIds(toDelete, (cur, total) => {
-        console.log(`  [${cur}/${total}] deleted`);
-      });
+async function recoverUrlsFromHtml(rawDir) {
+  const urlsJsonPath = join4(rawDir, "urls.json");
+  try {
+    return JSON.parse(await readFile3(urlsJsonPath, "utf-8"));
+  } catch {
+    const rawFiles = await readdir(rawDir);
+    const urls = [];
+    let skipped = 0;
+    for (const f of rawFiles.filter((f2) => f2.endsWith(".html"))) {
+      const fileSlug = f.replace(/\.html$/, "");
+      const html = await readFile3(join4(rawDir, f), "utf-8");
+      const match = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/);
+      if (match && slugifyUrl(match[1]) === fileSlug) {
+        urls.push(match[1]);
+        continue;
+      }
+      const ogMatch = html.match(/<meta[^>]+property="og:url"[^>]+content="([^"]+)"/);
+      if (ogMatch && slugifyUrl(ogMatch[1]) === fileSlug) {
+        urls.push(ogMatch[1]);
+        continue;
+      }
+      console.warn(`  WARNING: cannot recover URL for ${f}, skipping page.`);
+      skipped++;
     }
-    console.log("  Upserting chunks...");
-    await storeChunks(allChunks, embeddings, (cur, total) => {
-      console.log(`  [${cur}/${total}] stored`);
-    });
-  } else {
-    console.log("  Purging old chunks...");
-    await purgeSource(sourceName);
-    console.log("  Storing in Firestore...");
-    await storeChunks(allChunks, embeddings, (cur, total) => {
-      console.log(`  [${cur}/${total}] stored`);
-    });
+    if (skipped > 0) {
+      console.warn(`  Skipped ${skipped} pages with unrecoverable URLs. Provide urls.json to include them.`);
+    }
+    return urls;
   }
-  await updateSourceMeta(sourceName, allChunks.length, urlCount, version);
-  console.log(`  Done. ${allChunks.length} chunks stored for "${sourceName}".`);
 }
 async function cmdRefresh() {
   const args = parseArgs({
@@ -1256,12 +1846,10 @@ async function cmdRefresh() {
     options: {
       full: { type: "boolean", default: false },
       all: { type: "boolean", default: false },
-      diff: { type: "boolean", default: false },
       concurrency: { type: "string" },
       limit: { type: "string" },
       "from-html": { type: "boolean", default: false },
       "from-markdown": { type: "boolean", default: false },
-      "from-embeddings": { type: "boolean", default: false },
       "skip-store": { type: "boolean", default: false }
     },
     allowPositionals: true
@@ -1269,7 +1857,7 @@ async function cmdRefresh() {
   const config = await loadConfig(CONFIG_PATH);
   const sourcesToRefresh = args.values.all ? Object.keys(config.sources) : [args.positionals[0]];
   if (!args.values.all && !sourcesToRefresh[0]) {
-    console.error("Usage: grimoire refresh <source> [--full] [--from-html] [--from-markdown] [--from-embeddings] [--skip-store] [--limit <n>] [--concurrency <n>]");
+    console.error("Usage: grimoire refresh <source> [--full] [--from-html] [--from-markdown] [--skip-store] [--limit <n>] [--concurrency <n>]");
     process.exit(1);
   }
   const concurrencyOverride = args.values.concurrency ? parseInt(args.values.concurrency, 10) : void 0;
@@ -1285,7 +1873,6 @@ async function cmdRefresh() {
     }
     const rawDir = join4(DATA_DIR, "raw", sourceName);
     const mdDir = join4(DATA_DIR, "markdown", sourceName);
-    const embeddingsCachePath = join4(rawDir, "embeddings.json");
     console.log(`
 Refreshing "${sourceName}"...`);
     if (args.values.full) {
@@ -1295,75 +1882,16 @@ Refreshing "${sourceName}"...`);
       await rm(rawDir, { recursive: true, force: true });
       await rm(mdDir, { recursive: true, force: true });
     }
-    let urls;
-    if (args.values["from-embeddings"]) {
-      console.log("  Loading cached embeddings...");
-      const cached = await loadEmbeddingsCache(embeddingsCachePath);
-      if (!cached) {
-        console.error("  No cached embeddings found. Run without --from-embeddings first.");
-        process.exit(1);
-      }
-      const mdFiles = await readdir(mdDir);
-      const allPages = [];
-      for (const f of mdFiles.filter((f2) => f2.endsWith(".md"))) {
-        const content = await readFile3(join4(mdDir, f), "utf-8");
-        const urlMatch = content.match(/^url: "(.+)"$/m);
-        const titleMatch = content.match(/^title: "(.+)"$/m);
-        allPages.push({
-          markdown: content,
-          url: urlMatch?.[1] ?? "",
-          title: titleMatch?.[1] ?? "Untitled"
-        });
-      }
-      console.log("  Chunking...");
-      const allChunks2 = allPages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
-      console.log(`  Created ${allChunks2.length} chunks.`);
-      if (cached.length !== allChunks2.length) {
-        console.error(`  Embeddings cache (${cached.length}) doesn't match chunk count (${allChunks2.length}). Re-embed with --from-html.`);
-        process.exit(1);
-      }
-      if (args.values["skip-store"]) {
-        console.log(`  Done. ${allChunks2.length} chunks ready (skipped Firestore).`);
-        continue;
-      }
-      await storeWithStrategy(sourceName, allChunks2, cached, allPages.length, source.version, args.values.diff);
-      continue;
-    }
+    let pages;
     if (args.values["from-markdown"]) {
       console.log("  Reading cached markdown...");
-      const mdFiles = await readdir(mdDir).catch(() => []);
-      const markdownFiles = mdFiles.filter((f) => f.endsWith(".md"));
-      if (markdownFiles.length === 0) {
+      pages = await readCachedMarkdownPages(mdDir);
+      if (pages.length === 0) {
         console.error("  No cached markdown found. Run with --from-html first.");
         process.exit(1);
       }
-      const pages2 = [];
-      for (const f of markdownFiles) {
-        const content = await readFile3(join4(mdDir, f), "utf-8");
-        const urlMatch = content.match(/^url: "(.+)"$/m);
-        const titleMatch = content.match(/^title: "(.+)"$/m);
-        pages2.push({
-          markdown: content,
-          url: urlMatch?.[1] ?? "",
-          title: titleMatch?.[1] ?? "Untitled"
-        });
-      }
-      console.log(`  Found ${pages2.length} cached pages.`);
-      console.log("  Chunking...");
-      const allChunks2 = pages2.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
-      console.log(`  Created ${allChunks2.length} chunks.`);
-      console.log("  Embedding chunks...");
-      const texts2 = allChunks2.map((c) => c.content);
-      const embeddings2 = await embedWithCheckpoint(texts2, rawDir, embeddingsCachePath);
-      if (args.values["skip-store"]) {
-        console.log(`  Done. ${allChunks2.length} chunks ready (skipped Firestore).`);
-        continue;
-      }
-      await storeWithStrategy(sourceName, allChunks2, embeddings2, pages2.length, source.version, args.values.diff);
-      continue;
-    }
-    let pages;
-    if (source.llms_full_url && !args.values["from-html"]) {
+      console.log(`  Found ${pages.length} cached pages.`);
+    } else if (source.llms_full_url && !args.values["from-html"]) {
       console.log(`  Fetching llms-full.txt from ${source.llms_full_url}...`);
       pages = await ingestLlmsFull(
         source.llms_full_url,
@@ -1376,32 +1904,10 @@ Refreshing "${sourceName}"...`);
       );
       console.log(`  Extracted ${pages.length} pages.`);
     } else {
+      let urls;
       if (args.values["from-html"]) {
         console.log("  Reading URLs from cached HTML...");
-        const urlsJsonPath = join4(rawDir, "urls.json");
-        try {
-          urls = JSON.parse(await readFile3(urlsJsonPath, "utf-8"));
-        } catch {
-          const rawFiles = await readdir(rawDir);
-          const htmlFiles = rawFiles.filter((f) => f.endsWith(".html"));
-          urls = [];
-          for (const f of htmlFiles) {
-            const fileSlug = f.replace(/\.html$/, "");
-            const htmlPath = join4(rawDir, f);
-            const html = await readFile3(htmlPath, "utf-8");
-            const match = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/);
-            if (match && slugifyUrl(match[1]) === fileSlug) {
-              urls.push(match[1]);
-              continue;
-            }
-            const ogMatch = html.match(/<meta[^>]+property="og:url"[^>]+content="([^"]+)"/);
-            if (ogMatch && slugifyUrl(ogMatch[1]) === fileSlug) {
-              urls.push(ogMatch[1]);
-              continue;
-            }
-            urls.push(`https://recovered/${fileSlug}`);
-          }
-        }
+        urls = await recoverUrlsFromHtml(rawDir);
         console.log(`  Found ${urls.length} cached pages.`);
       } else {
         console.log("  Scraping URLs...");
@@ -1431,14 +1937,11 @@ Refreshing "${sourceName}"...`);
     console.log("  Chunking...");
     const allChunks = pages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
     console.log(`  Created ${allChunks.length} chunks.`);
-    console.log("  Embedding chunks...");
-    const texts = allChunks.map((c) => c.content);
-    const embeddings = await embedWithCheckpoint(texts, rawDir, embeddingsCachePath);
     if (args.values["skip-store"]) {
-      console.log(`  Done. ${allChunks.length} chunks ready (skipped Firestore).`);
+      console.log(`  Done. ${allChunks.length} chunks ready (dry run, no embed/store).`);
       continue;
     }
-    await storeWithStrategy(sourceName, allChunks, embeddings, pages.length, source.version, args.values.diff);
+    await syncChunks(sourceName, allChunks, pages.length, source.version);
   }
 }
 async function cmdSearch() {
@@ -1447,17 +1950,19 @@ async function cmdSearch() {
     options: {
       source: { type: "string" },
       top: { type: "string" },
+      candidates: { type: "string" },
       compact: { type: "boolean", default: false }
     },
     allowPositionals: true
   });
   const query = args.positionals.join(" ");
   if (!query) {
-    console.error('Usage: grimoire search "<query>" [--source <name>] [--top <n>] [--compact]');
+    console.error('Usage: grimoire search "<query>" [--source <name>] [--top <n>] [--candidates <n>] [--compact]');
     process.exit(1);
   }
   const topN = args.values.top ? parseInt(args.values.top, 10) : void 0;
-  const results = await search(query, { source: args.values.source, topN });
+  const candidates = args.values.candidates ? parseInt(args.values.candidates, 10) : void 0;
+  const results = await search(query, { source: args.values.source, topN, candidates });
   if (results.length === 0) {
     console.log("No results found.");
     return;
@@ -1644,4 +2149,4 @@ var ADMIN_COMMANDS = {
 export {
   ADMIN_COMMANDS
 };
-//# sourceMappingURL=admin-AKV4CA2O.js.map
+//# sourceMappingURL=admin-WUNBDKBC.js.map