@astrofoundry/grimoire 3.13.0 → 3.15.0

package/dist/admin-HA6FNUV4.js

@@ -0,0 +1,1516 @@
+import {
+  bold,
+  cyan,
+  yellow
+} from "./chunk-BRS6X3AE.js";
+
+// src/admin.ts
+import { parseArgs } from "node:util";
+import { readFile as readFile3, writeFile as writeFile4, readdir, rm, mkdir as mkdir4 } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join as join4, resolve } from "node:path";
+import { createInterface } from "node:readline";
+import { stringify } from "yaml";
+
+// src/config.ts
+import { readFile } from "node:fs/promises";
+import { parse } from "yaml";
+var REQUIRED_SOURCE_FIELDS = [
+  "name",
+  "start_url"
+];
+var SCRAPE_REQUIRED_FIELDS = [
+  "nav_selector",
+  "content_selector"
+];
+function validateConfig(data) {
+  if (typeof data !== "object" || data === null || !("sources" in data)) {
+    throw new Error("Config must have a 'sources' key");
+  }
+  const { sources } = data;
+  if (typeof sources !== "object" || sources === null) {
+    throw new Error("'sources' must be an object");
+  }
+  const entries = Object.entries(sources);
+  if (entries.length === 0) {
+    throw new Error("'sources' must contain at least one source");
+  }
+  const validated = {};
+  for (const [key, value] of entries) {
+    if (typeof value !== "object" || value === null) {
+      throw new Error(`Source '${key}' must be an object`);
+    }
+    const source = value;
+    for (const field of REQUIRED_SOURCE_FIELDS) {
+      if (typeof source[field] !== "string" || source[field] === "") {
+        throw new Error(`Source '${key}' is missing required field '${field}'`);
+      }
+    }
+    if (!source.llms_full_url) {
+      for (const field of SCRAPE_REQUIRED_FIELDS) {
+        if (typeof source[field] !== "string" || source[field] === "") {
+          throw new Error(`Source '${key}' is missing required field '${field}'`);
+        }
+      }
+    }
+    try {
+      new URL(source.start_url);
+    } catch {
+      throw new Error(`Source '${key}' has invalid start_url: ${source.start_url}`);
+    }
+    if (source.remove_selectors !== void 0 && !Array.isArray(source.remove_selectors)) {
+      throw new Error(`Source '${key}': remove_selectors must be an array`);
+    }
+    if (source.remove_text_patterns !== void 0 && !Array.isArray(source.remove_text_patterns)) {
+      throw new Error(`Source '${key}': remove_text_patterns must be an array`);
+    }
+    if (source.include_patterns !== void 0 && !Array.isArray(source.include_patterns)) {
+      throw new Error(`Source '${key}': include_patterns must be an array`);
+    }
+    if (source.exclude_patterns !== void 0 && !Array.isArray(source.exclude_patterns)) {
+      throw new Error(`Source '${key}': exclude_patterns must be an array`);
+    }
+    if (source.rate_limit_ms !== void 0 && typeof source.rate_limit_ms !== "number") {
+      throw new Error(`Source '${key}': rate_limit_ms must be a number`);
+    }
+    if (source.concurrency !== void 0 && typeof source.concurrency !== "number") {
+      throw new Error(`Source '${key}': concurrency must be a number`);
+    }
+    validated[key] = {
+      name: source.name,
+      version: source.version,
+      start_url: source.start_url,
+      nav_selector: source.nav_selector,
+      content_selector: source.content_selector,
+      remove_selectors: source.remove_selectors,
+      remove_text_patterns: source.remove_text_patterns,
+      include_patterns: source.include_patterns,
+      exclude_patterns: source.exclude_patterns,
+      rate_limit_ms: source.rate_limit_ms,
+      concurrency: source.concurrency,
+      headed: source.headed,
+      sitemap_url: source.sitemap_url,
+      llms_full_url: source.llms_full_url
+    };
+  }
+  return { sources: validated };
+}
+async function loadConfig(path) {
+  const content = await readFile(path, "utf-8");
+  const data = parse(content, { merge: true });
+  return validateConfig(data);
+}
+
+// src/scraper.ts
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { chromium } from "playwright";
+function slugifyUrl(url) {
+  const parsed = new URL(url);
+  return parsed.pathname.replace(/^\//, "").replace(/\/$/, "").replace(/\//g, "-").replace(/[^a-zA-Z0-9-]/g, "");
+}
+function filterUrls(urls, includePatterns, excludePatterns) {
+  let filtered = urls.filter(
+    (url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#")
+  );
+  if (includePatterns && includePatterns.length > 0) {
+    filtered = filtered.filter(
+      (url) => includePatterns.some((pattern) => url.includes(pattern))
+    );
+  }
+  if (excludePatterns && excludePatterns.length > 0) {
+    filtered = filtered.filter(
+      (url) => !excludePatterns.some((pattern) => url.includes(pattern))
+    );
+  }
+  return [...new Set(filtered)].sort();
+}
+async function fetchSitemapUrls(sitemapUrl) {
+  const response = await fetch(sitemapUrl);
+  const xml = await response.text();
+  const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+  if (xml.includes("<sitemapindex")) {
+    const nested = await Promise.all(locs.map((loc) => fetchSitemapUrls(loc)));
+    return nested.flat();
+  }
+  return locs;
+}
+async function discoverFromSitemap(sitemapUrl, source) {
+  const urls = await fetchSitemapUrls(sitemapUrl);
+  return filterUrls(urls, source.include_patterns, source.exclude_patterns);
+}
+async function discoverUrls(page, source) {
+  if (source.sitemap_url) {
+    return discoverFromSitemap(source.sitemap_url, source);
+  }
+  await page.goto(source.start_url, { waitUntil: source.headed ? "networkidle" : "domcontentloaded" });
+  const rawUrls = await page.$$eval(
+    `${source.nav_selector} a[href]`,
+    (links) => links.map((a) => a.href)
+  );
+  const discovered = filterUrls(rawUrls, source.include_patterns, source.exclude_patterns);
+  if (!discovered.includes(source.start_url)) {
+    discovered.unshift(source.start_url);
+  }
+  return discovered;
+}
+async function fetchPage(page, url, headed) {
+  await page.goto(url, { waitUntil: headed ? "networkidle" : "domcontentloaded" });
+  return page.content();
+}
+var DEFAULT_CONCURRENCY = 50;
+async function runPool(items, concurrency, fn) {
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < items.length) {
+      const index = nextIndex++;
+      await fn(items[index], index);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
+  await Promise.all(workers);
+}
+async function scrapeSource(source, sourceName, dataDir, onProgress) {
+  const rawDir = join(dataDir, "raw", sourceName);
+  await mkdir(rawDir, { recursive: true });
+  const concurrency = source.concurrency ?? DEFAULT_CONCURRENCY;
+  const browser = await chromium.launch({ channel: "chrome", headless: !source.headed });
+  const context = await browser.newContext(source.user_agent ? { userAgent: source.user_agent } : {});
+  const discoveryPage = await context.newPage();
+  const urls = await discoverUrls(discoveryPage, source);
+  await discoveryPage.close();
+  let completed = 0;
+  try {
+    await runPool(urls, concurrency, async (url) => {
+      const page = await context.newPage();
+      try {
+        const html = await fetchPage(page, url, source.headed);
+        const slug = slugifyUrl(url);
+        await writeFile(join(rawDir, `${slug}.html`), html, "utf-8");
+        completed++;
+        onProgress?.(completed, urls.length, url);
+      } finally {
+        await page.close();
+      }
+    });
+    await writeFile(join(rawDir, "urls.json"), JSON.stringify(urls), "utf-8");
+    return urls;
+  } finally {
+    await browser.close();
+  }
+}
+async function createBrowser() {
+  return chromium.launch({ channel: "chrome" });
+}
+
+// src/converter.ts
+import { readFile as readFile2, writeFile as writeFile2, mkdir as mkdir2 } from "node:fs/promises";
+import { join as join2 } from "node:path";
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+var turndown = new TurndownService({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  bulletListMarker: "-"
+});
+var GENERIC_REMOVE = [
+  "style",
+  "script",
+  "noscript",
+  "iframe",
+  "svg"
+];
+function cleanMarkdown(md, textPatterns) {
+  let cleaned = md.replace(/^(#+)\s*$/gm, "").replace(/\n{3,}/g, "\n\n");
+  if (textPatterns) {
+    for (const pattern of textPatterns) {
+      cleaned = cleaned.replace(new RegExp(pattern, "gm"), "");
+    }
+    cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
+  }
+  return cleaned.trim();
+}
+function extractContent(html, contentSelector, removeSelectors, removeTextPatterns) {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+  const contentEl = doc.querySelector(contentSelector);
+  if (!contentEl) {
+    return cleanMarkdown(turndown.turndown(doc.body.innerHTML), removeTextPatterns);
+  }
+  const allSelectors = [...GENERIC_REMOVE, ...removeSelectors ?? []];
+  for (const selector of allSelectors) {
+    for (const el of contentEl.querySelectorAll(selector)) {
+      el.remove();
+    }
+  }
+  return cleanMarkdown(turndown.turndown(contentEl.innerHTML), removeTextPatterns);
+}
+function extractTitle(html) {
+  const dom = new JSDOM(html);
+  const titleEl = dom.window.document.querySelector("title");
+  if (!titleEl) return "Untitled";
+  return titleEl.textContent?.replace(/\s*[|–—-]\s*.+$/, "").trim() ?? "Untitled";
+}
+function buildFrontmatter(source, url, title) {
+  return [
+    "---",
+    `source: ${source}`,
+    `url: "${url}"`,
+    `title: "${title.replace(/"/g, '\\"')}"`,
+    `fetched_at: "${(/* @__PURE__ */ new Date()).toISOString()}"`,
+    "---"
+  ].join("\n");
+}
+function convertPage(html, source, url, contentSelector, removeSelectors, removeTextPatterns) {
+  const title = extractTitle(html);
+  const content = extractContent(html, contentSelector, removeSelectors, removeTextPatterns);
+  const frontmatter = buildFrontmatter(source, url, title);
+  const markdown = `${frontmatter}
+
+${content}`;
+  return { source, url, title, markdown };
+}
+var DEFAULT_CONCURRENCY2 = 10;
+async function convertSource(sourceName, urls, contentSelector, removeSelectors, removeTextPatterns, dataDir, concurrency = DEFAULT_CONCURRENCY2, onProgress) {
+  const rawDir = join2(dataDir, "raw", sourceName);
+  const mdDir = join2(dataDir, "markdown", sourceName);
+  await mkdir2(mdDir, { recursive: true });
+  const pages = new Array(urls.length);
+  let completed = 0;
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < urls.length) {
+      const i = nextIndex++;
+      const url = urls[i];
+      const slug = slugifyUrl(url);
+      const htmlPath = join2(rawDir, `${slug}.html`);
+      const html = await readFile2(htmlPath, "utf-8");
+      const page = convertPage(html, sourceName, url, contentSelector, removeSelectors, removeTextPatterns);
+      await writeFile2(join2(mdDir, `${slug}.md`), page.markdown, "utf-8");
+      pages[i] = page;
+      completed++;
+      onProgress?.(completed, urls.length, url);
+    }
+  }
+  const workers = Array.from(
+    { length: Math.min(concurrency, urls.length) },
+    () => worker()
+  );
+  await Promise.all(workers);
+  return pages;
+}
+
+// src/chunker.ts
+var MAX_TOKENS = 500;
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function slugifyHeading(heading) {
+  return heading.toLowerCase().replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
+}
+function buildChunkId(source, url, headingSlug, index) {
+  const urlSlug = slugifyUrl(url);
+  const prefix = `${source}::${urlSlug}::`;
+  const maxSlugBytes = 1500 - Buffer.byteLength(prefix) - 10;
+  const truncatedSlug = Buffer.byteLength(headingSlug) > maxSlugBytes ? Buffer.from(headingSlug).subarray(0, maxSlugBytes).toString() : headingSlug;
+  const base = `${prefix}${truncatedSlug}`;
+  return index !== void 0 ? `${base}-${index}` : base;
+}
+function parseHeadingSections(markdown) {
+  const lines = markdown.split("\n");
+  const sections = [];
+  const headingStack = [];
+  const levelStack = [];
+  let currentSection = {
+    level: 0,
+    heading: "",
+    headingPath: [],
+    lines: []
+  };
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+    if (headingMatch) {
+      if (currentSection.lines.length > 0 || currentSection.heading !== "") {
+        sections.push(currentSection);
+      }
+      const level = headingMatch[1].length;
+      const heading = headingMatch[2].trim();
+      while (levelStack.length > 0 && levelStack[levelStack.length - 1] >= level) {
+        levelStack.pop();
+        headingStack.pop();
+      }
+      headingStack.push(heading);
+      levelStack.push(level);
+      currentSection = {
+        level,
+        heading,
+        headingPath: [...headingStack],
+        lines: []
+      };
+    } else {
+      currentSection.lines.push(line);
+    }
+  }
+  if (currentSection.lines.length > 0 || currentSection.heading !== "") {
+    sections.push(currentSection);
+  }
+  return sections;
+}
+function splitAtParagraphBoundaries(text, maxTokens) {
+  const paragraphs = text.split(/\n\n+/);
+  const parts = [];
+  let current = [];
+  let currentTokens = 0;
+  for (const para of paragraphs) {
+    const paraTokens = estimateTokens(para);
+    if (currentTokens + paraTokens > maxTokens && current.length > 0) {
+      parts.push(current.join("\n\n"));
+      current = [para];
+      currentTokens = paraTokens;
+    } else {
+      current.push(para);
+      currentTokens += paraTokens;
+    }
+  }
+  if (current.length > 0) {
+    parts.push(current.join("\n\n"));
+  }
+  return parts;
+}
+function stripFrontmatter(markdown) {
+  if (markdown.startsWith("---")) {
+    const endIndex = markdown.indexOf("---", 3);
+    if (endIndex !== -1) {
+      return markdown.slice(endIndex + 3).trim();
+    }
+  }
+  return markdown;
+}
+function chunkMarkdown(markdown, source, url, title) {
+  const stripped = stripFrontmatter(markdown);
+  const sections = parseHeadingSections(stripped);
+  const chunks = [];
+  const usedIds = /* @__PURE__ */ new Set();
+  function uniqueId(baseSlug) {
+    let id = buildChunkId(source, url, baseSlug);
+    if (!usedIds.has(id)) {
+      usedIds.add(id);
+      return id;
+    }
+    let counter = 1;
+    while (usedIds.has(buildChunkId(source, url, baseSlug, counter))) {
+      counter++;
+    }
+    id = buildChunkId(source, url, baseSlug, counter);
+    usedIds.add(id);
+    return id;
+  }
+  for (const section of sections) {
+    const headingLine = section.heading ? `${"#".repeat(section.level)} ${section.heading}
+
+` : "";
+    const content = headingLine + section.lines.join("\n").trim();
+    if (!content.trim()) continue;
+    const headingSlug = section.heading ? slugifyHeading(section.heading) : "intro";
+    const tokens = estimateTokens(content);
+    if (tokens <= MAX_TOKENS) {
+      chunks.push({
+        id: uniqueId(headingSlug),
+        source,
+        url,
+        title,
+        heading_path: section.headingPath,
+        content,
+        token_count: tokens
+      });
+    } else {
+      const parts = splitAtParagraphBoundaries(content, MAX_TOKENS);
+      for (let i = 0; i < parts.length; i++) {
+        const partContent = parts[i].trim();
+        if (!partContent) continue;
+        const partSlug = parts.length > 1 ? `${headingSlug}-${i}` : headingSlug;
+        chunks.push({
+          id: uniqueId(partSlug),
+          source,
+          url,
+          title,
+          heading_path: section.headingPath,
+          content: partContent,
+          token_count: estimateTokens(partContent)
+        });
+      }
+    }
+  }
+  return chunks;
+}
+
+// src/embedder.ts
+import { GoogleGenerativeAI } from "@google/generative-ai";
+var BATCH_SIZE = 50;
+var MODEL = "gemini-embedding-001";
+var OUTPUT_DIMENSIONALITY = 768;
+var MAX_RETRIES = 5;
+var RATE_LIMIT_BASE_DELAY_MS = 6e4;
+var NETWORK_BASE_DELAY_MS = 1e4;
+var BATCH_DELAY_MS = 2500;
+var DEFAULT_CHECKPOINT_EVERY_BATCHES = 20;
+var NETWORK_ERROR_PATTERNS = [
+  "fetch failed",
+  "ECONNRESET",
+  "ETIMEDOUT",
+  "ECONNREFUSED",
+  "EAI_AGAIN",
+  "ENOTFOUND",
+  "socket hang up",
+  "UND_ERR_"
+];
+var genAI;
+function getClient() {
+  if (!genAI) {
+    const apiKey = process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+      throw new Error("GEMINI_API_KEY environment variable is not set");
+    }
+    genAI = new GoogleGenerativeAI(apiKey);
+  }
+  return genAI;
+}
+function classifyError(message) {
+  if (message.includes("429") || message.includes("503")) {
+    return "rate_limit";
+  }
+  if (NETWORK_ERROR_PATTERNS.some((p) => message.includes(p))) {
+    return "network";
+  }
+  return "other";
+}
+async function embedTexts(texts, options = {}) {
+  const client = getClient();
+  const model = client.getGenerativeModel({ model: MODEL });
+  const { onProgress, onCheckpoint, resumeFrom } = options;
+  const checkpointEveryBatches = options.checkpointEveryBatches ?? DEFAULT_CHECKPOINT_EVERY_BATCHES;
+  const embeddings = resumeFrom ? [...resumeFrom] : [];
+  const startIndex = Math.floor(embeddings.length / BATCH_SIZE) * BATCH_SIZE;
+  if (embeddings.length > startIndex) {
+    embeddings.length = startIndex;
+  }
+  if (startIndex > 0) {
+    console.log(`  Resuming from chunk ${startIndex} of ${texts.length} (${embeddings.length} cached).`);
+  }
+  if (startIndex >= texts.length) {
+    return embeddings.slice(0, texts.length);
+  }
+  let batchesSinceCheckpoint = 0;
+  for (let i = startIndex; i < texts.length; i += BATCH_SIZE) {
+    const batch = texts.slice(i, i + BATCH_SIZE);
+    const batchNumber = i / BATCH_SIZE + 1;
+    let result;
+    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+      try {
+        result = await model.batchEmbedContents({
+          requests: batch.map((text) => ({
+            content: { role: "user", parts: [{ text }] },
+            outputDimensionality: OUTPUT_DIMENSIONALITY
+          }))
+        });
+        break;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        const kind = classifyError(message);
+        if (kind !== "other" && attempt < MAX_RETRIES - 1) {
+          const baseDelay = kind === "rate_limit" ? RATE_LIMIT_BASE_DELAY_MS : NETWORK_BASE_DELAY_MS;
+          const delay = baseDelay * Math.pow(2, attempt);
+          const label = kind === "rate_limit" ? "Rate limited" : "Network error";
+          console.log(`  ${label} (batch ${batchNumber}), retrying in ${delay / 1e3}s...`);
+          await new Promise((resolve2) => setTimeout(resolve2, delay));
+          continue;
+        }
+        console.error(`  Embedding failed at batch ${batchNumber} (chunks ${i + 1}-${i + batch.length}): ${message}`);
+        throw err;
+      }
+    }
+    for (const embedding of result.embeddings) {
+      embeddings.push(embedding.values);
+    }
+    onProgress?.(Math.min(i + BATCH_SIZE, texts.length), texts.length);
+    batchesSinceCheckpoint++;
+    if (onCheckpoint && batchesSinceCheckpoint >= checkpointEveryBatches && i + BATCH_SIZE < texts.length) {
+      await onCheckpoint(embeddings);
+      batchesSinceCheckpoint = 0;
+    }
+    if (i + BATCH_SIZE < texts.length) {
+      await new Promise((resolve2) => setTimeout(resolve2, BATCH_DELAY_MS));
+    }
+  }
+  if (onCheckpoint) {
+    await onCheckpoint(embeddings);
+  }
+  return embeddings;
+}
+async function embedText(text) {
+  const [embedding] = await embedTexts([text]);
+  return embedding;
+}
+
+// src/store.ts
+import { initializeApp, applicationDefault, getApps } from "firebase-admin/app";
+import {
+  getFirestore,
+  FieldValue
+} from "firebase-admin/firestore";
+var BATCH_SIZE2 = 500;
+var MAX_RETRIES2 = 5;
+var BASE_DELAY_MS = 5e3;
+async function retryOnQuota(fn) {
+  for (let attempt = 0; attempt < MAX_RETRIES2; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      if ((message.includes("RESOURCE_EXHAUSTED") || message.includes("Quota exceeded")) && attempt < MAX_RETRIES2 - 1) {
+        const delay = BASE_DELAY_MS * Math.pow(2, attempt);
+        console.log(`  Quota exceeded, retrying in ${delay / 1e3}s...`);
+        await new Promise((resolve2) => setTimeout(resolve2, delay));
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw new Error("Max retries exceeded");
+}
+var db;
+function getDb() {
+  if (!db) {
+    if (getApps().length === 0) {
+      initializeApp({ credential: applicationDefault() });
+    }
+    db = getFirestore();
+  }
+  return db;
+}
+function chunksCol() {
+  return getDb().collection("grimoire_chunks");
+}
+function sourcesCol() {
+  return getDb().collection("grimoire_sources");
+}
+async function storeChunks(chunks, embeddings, onProgress) {
+  const database = getDb();
+  const col = chunksCol();
+  for (let i = 0; i < chunks.length; i += BATCH_SIZE2) {
+    const batch = database.batch();
+    const slice = chunks.slice(i, i + BATCH_SIZE2);
+    const embSlice = embeddings.slice(i, i + BATCH_SIZE2);
+    for (let j = 0; j < slice.length; j++) {
+      const chunk = slice[j];
+      batch.set(col.doc(chunk.id), {
+        source: chunk.source,
+        url: chunk.url,
+        title: chunk.title,
+        heading_path: chunk.heading_path,
+        content: chunk.content,
+        token_count: chunk.token_count,
+        embedded_at: (/* @__PURE__ */ new Date()).toISOString(),
+        embedding: FieldValue.vector(embSlice[j])
+      });
+    }
+    await retryOnQuota(() => batch.commit());
+    onProgress?.(Math.min(i + BATCH_SIZE2, chunks.length), chunks.length);
+  }
+}
+async function purgeSource(sourceName) {
+  const database = getDb();
+  const col = chunksCol();
+  const snapshot = await col.where("source", "==", sourceName).get();
+  if (snapshot.empty) return 0;
+  let batch = database.batch();
+  let count = 0;
+  for (const doc of snapshot.docs) {
+    batch.delete(doc.ref);
+    count++;
+    if (count % BATCH_SIZE2 === 0) {
+      await retryOnQuota(() => batch.commit());
+      batch = database.batch();
+    }
+  }
+  if (count % BATCH_SIZE2 !== 0) {
+    await retryOnQuota(() => batch.commit());
+  }
+  return count;
+}
+async function updateSourceMeta(sourceName, chunkCount, urlCount, version) {
+  await sourcesCol().doc(sourceName).set({
+    source: sourceName,
+    ...version ? { version } : {},
+    last_refreshed: (/* @__PURE__ */ new Date()).toISOString(),
+    chunk_count: chunkCount,
+    url_count: urlCount
+  });
+}
+async function getSourceMeta(sourceName) {
+  const doc = await sourcesCol().doc(sourceName).get();
+  if (!doc.exists) return null;
+  return doc.data();
+}
+async function getAllSourcesMeta() {
+  const snapshot = await sourcesCol().get();
+  return snapshot.docs.map((doc) => doc.data());
+}
+async function deleteSourceMeta(sourceName) {
+  await sourcesCol().doc(sourceName).delete();
+}
+async function deleteChunksByIds(ids, onProgress) {
+  if (ids.length === 0) return;
+  const database = getDb();
+  const col = chunksCol();
+  for (let i = 0; i < ids.length; i += BATCH_SIZE2) {
+    const batch = database.batch();
+    const slice = ids.slice(i, i + BATCH_SIZE2);
+    for (const id of slice) {
+      batch.delete(col.doc(id));
+    }
+    await retryOnQuota(() => batch.commit());
+    onProgress?.(Math.min(i + BATCH_SIZE2, ids.length), ids.length);
+  }
+}
+async function getSourceChunkIds(sourceName) {
+  const col = chunksCol();
+  const snapshot = await col.where("source", "==", sourceName).select().get();
+  return new Set(snapshot.docs.map((doc) => doc.id));
+}
+async function vectorSearch(queryEmbedding, limit, source) {
+  const col = chunksCol();
+  let query = col;
+  if (source) {
+    query = query.where("source", "==", source);
+  }
+  const snapshot = await query.findNearest({
+    vectorField: "embedding",
+    queryVector: FieldValue.vector(queryEmbedding),
+    limit,
+    distanceMeasure: "COSINE",
+    distanceResultField: "_distance"
+  }).get();
+  return snapshot.docs.map((doc) => {
+    const data = doc.data();
+    const distance = data._distance ?? 0;
+    delete data._distance;
+    return { id: doc.id, data, distance };
+  });
+}
+
+// src/reranker.ts
+function getRerankerUrl() {
+  const url = process.env.RERANKER_URL;
+  if (!url) {
+    throw new Error("RERANKER_URL environment variable is not set");
+  }
+  return url;
+}
+async function rerank(query, documents, topN = 5) {
+  const baseUrl = getRerankerUrl();
+  const response = await fetch(`${baseUrl}/v1/rerank`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ query, documents, top_n: topN })
+  });
+  if (!response.ok) {
+    throw new Error(`Reranker request failed: ${response.status} ${response.statusText}`);
+  }
+  const data = await response.json();
+  return data.results;
+}
+
+// src/search.ts
+function hasReranker() {
+  return !!process.env.RERANKER_URL;
+}
+async function search(query, options = {}) {
+  const { source, candidates = 20, topN = 5 } = options;
+  const queryEmbedding = await embedText(query);
+  const rawResults = await vectorSearch(queryEmbedding, candidates, source);
+  if (rawResults.length === 0) return [];
+  if (hasReranker()) {
+    const documents = rawResults.map((r) => r.data.content);
+    const reranked = await rerank(query, documents, topN);
+    return reranked.map((r) => {
+      const original = rawResults[r.index];
+      const data = original.data;
+      return {
+        id: original.id,
+        source: data.source,
+        url: data.url,
+        title: data.title,
+        heading_path: data.heading_path,
+        content: data.content,
+        relevance_score: r.relevance_score
+      };
+    });
+  }
+  return rawResults.slice(0, topN).map((r) => {
+    const data = r.data;
+    return {
+      id: r.id,
+      source: data.source,
+      url: data.url,
+      title: data.title,
+      heading_path: data.heading_path,
+      content: data.content,
+      relevance_score: Math.max(0, 1 - r.distance / 2)
+    };
+  });
+}
+
+// src/apikey.ts
+import { randomBytes, createHash } from "node:crypto";
+import {
+  getFirestore as getFirestore2
+} from "firebase-admin/firestore";
+import { initializeApp as initializeApp2, applicationDefault as applicationDefault2, getApps as getApps2 } from "firebase-admin/app";
+var db2;
+function getDb2() {
+  if (!db2) {
+    if (getApps2().length === 0) {
+      initializeApp2({ credential: applicationDefault2() });
+    }
+    db2 = getFirestore2();
+  }
+  return db2;
+}
+function hashKey(key) {
+  return createHash("sha256").update(key).digest("hex");
+}
+function apiKeysCol() {
+  return getDb2().collection("grimoire_api_keys");
+}
+async function createApiKey(name) {
+  const raw = `grim_${randomBytes(32).toString("base64url")}`;
+  const hash = hashKey(raw);
+  await apiKeysCol().doc(hash).set({
+    name,
+    created_at: (/* @__PURE__ */ new Date()).toISOString(),
+    last_used_at: null
+  });
+  return raw;
+}
+async function listApiKeys() {
+  const snapshot = await apiKeysCol().get();
+  if (snapshot.empty) {
+    console.log("No API keys found.");
+    return;
+  }
+  console.log("\nAPI Keys:\n");
+  for (const doc of snapshot.docs) {
+    const data = doc.data();
+    const lastUsed = data.last_used_at ?? "never";
+    console.log(`  ${bold(data.name)}`);
+    console.log(`    Created: ${data.created_at}`);
+    console.log(`    Last used: ${lastUsed}`);
+  }
+}
+async function deleteApiKey(name) {
+  const snapshot = await apiKeysCol().where("name", "==", name).get();
+  if (snapshot.empty) {
+    throw new Error(`No API key found with name "${name}".`);
+  }
+  for (const doc of snapshot.docs) {
+    await doc.ref.delete();
+  }
+  console.log(`API key "${name}" deleted.`);
+}
+async function cmdApiKey() {
+  const subcommand = process.argv[3];
+  const arg = process.argv[4];
+  if (subcommand === "create") {
+    if (!arg) {
+      console.error("Usage: grimoire apikey create <name>");
+      process.exit(1);
+    }
+    const key = await createApiKey(arg);
+    console.log(`
+API key created for "${arg}":
+`);
+    console.log(`  ${key}
+`);
+    console.log("Save this key \u2014 it will not be shown again.");
+  } else if (subcommand === "list") {
+    await listApiKeys();
+  } else if (subcommand === "delete") {
+    if (!arg) {
+      console.error("Usage: grimoire apikey delete <name>");
+      process.exit(1);
+    }
+    await deleteApiKey(arg);
+  } else {
+    console.error("Usage: grimoire apikey <create|list|delete> [name]");
+    process.exit(1);
+  }
+}
+
+// src/llms-ingest.ts
+import { writeFile as writeFile3, mkdir as mkdir3 } from "node:fs/promises";
+import { join as join3 } from "node:path";
+var BOILERPLATE_PATTERNS = [
+  /^\[Skip to content\]\([^)]*\)\s*$/gm,
+  /^Was this helpful\?\s*$/gm,
+  /^YesNo\s*$/gm,
+  /^\[ Edit page \]\([^)]+\) \[ Report issue \]\([^)]+\)\s*$/gm,
+  /^Copy page\s*$/gm,
+  /^```json\n\{"@context":"https:\/\/schema\.org","@type":"BreadcrumbList"[^`]*```\s*$/gm
+];
+function splitPages(content) {
+  const pages = [];
+  const frontmatterPattern = /^---\ntitle: (.+)\n/gm;
+  const boundaries = [];
+  let match;
+  while ((match = frontmatterPattern.exec(content)) !== null) {
+    boundaries.push({ index: match.index, title: match[1] });
+  }
+  for (let i = 0; i < boundaries.length; i++) {
+    const start = boundaries[i].index;
+    const end = i + 1 < boundaries.length ? boundaries[i + 1].index : content.length;
+    const raw = content.slice(start, end).trimEnd();
+    const url = extractUrl(raw);
+    if (!url) continue;
+    const bodyStart = raw.indexOf("---", 3);
+    if (bodyStart === -1) continue;
+    const body = raw.slice(raw.indexOf("\n", bodyStart) + 1);
+    let cleaned = body;
+    for (const pattern of BOILERPLATE_PATTERNS) {
+      cleaned = cleaned.replace(pattern, "");
+    }
+    cleaned = cleaned.replace(/\n{3,}/g, "\n\n").trim();
+    if (!cleaned) continue;
+    pages.push({
+      title: boundaries[i].title,
+      url,
+      markdown: cleaned
+    });
+  }
+  return pages;
+}
+function extractUrl(pageContent) {
+  const match = pageContent.match(
+    /```json\n\{"@context":"https:\/\/schema\.org","@type":"BreadcrumbList","itemListElement":\[(.+?)\]\}\n```/
+  );
+  if (!match) return null;
+  const items = JSON.parse(`[${match[1]}]`);
+  const last = items[items.length - 1];
+  if (!last?.item?.["@id"]) return null;
+  return `https://developers.cloudflare.com${last.item["@id"]}`;
+}
+async function ingestLlmsFull(llmsFullUrl, sourceName, baseUrl, dataDir, onProgress) {
+  const response = await fetch(llmsFullUrl);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${llmsFullUrl}: ${response.status} ${response.statusText}`);
+  }
+  const content = await response.text();
+  const pages = splitPages(content);
+  const mdDir = join3(dataDir, "markdown", sourceName);
+  await mkdir3(mdDir, { recursive: true });
+  const results = [];
+  for (let i = 0; i < pages.length; i++) {
+    const page = pages[i];
+    const frontmatter = buildFrontmatter(sourceName, page.url, page.title);
+    const fullMarkdown = `${frontmatter}
+
+${page.markdown}`;
+    const slug = slugifyUrl(page.url);
+    await writeFile3(join3(mdDir, `${slug}.md`), fullMarkdown, "utf-8");
+    results.push({
+      source: sourceName,
+      url: page.url,
+      title: page.title,
+      markdown: fullMarkdown
+    });
+    if (onProgress && ((i + 1) % 100 === 0 || i + 1 === pages.length)) {
+      onProgress(i + 1, pages.length);
+    }
+  }
+  return results;
+}
+
+// src/admin.ts
+var PROJECT_ROOT = resolve(import.meta.dirname, "..");
+var CONFIG_PATH = join4(PROJECT_ROOT, "config", "sources.yaml");
+var DATA_DIR = join4(PROJECT_ROOT, "data");
+function prompt(rl, question) {
+  return new Promise((resolve2) => rl.question(question, resolve2));
+}
+async function cmdAdd() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      url: { type: "string" }
+    },
+    allowPositionals: true
+  });
+  const name = args.positionals[0];
+  const url = args.values.url;
+  if (!name || !url) {
+    console.error("Usage: grimoire add <name> --url <start_url>");
+    process.exit(1);
+  }
+  console.log("Scanning page...\n");
+  const browser = await createBrowser();
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded" });
+    const navCandidates = await page.evaluate(() => {
+      const selectors = ["nav", "[role='navigation']"];
+      const results = [];
+      const seen = /* @__PURE__ */ new Set();
+      for (const sel of selectors) {
+        for (const el of document.querySelectorAll(sel)) {
+          if (seen.has(el)) continue;
+          seen.add(el);
+          const links = el.querySelectorAll("a[href]");
+          const label = el.getAttribute("aria-label") || el.getAttribute("class") || el.tagName.toLowerCase();
+          results.push({
+            selector: sel,
+            label,
+            linkCount: links.length
+          });
+        }
+      }
+      return results.sort((a, b) => b.linkCount - a.linkCount);
+    });
+    if (navCandidates.length === 0) {
+      console.error("No navigation elements found on this page.");
+      process.exit(1);
+    }
+    console.log("Navigation candidates:");
+    for (let i = 0; i < navCandidates.length; i++) {
+      const c = navCandidates[i];
+      console.log(`  [${i + 1}] ${c.selector} (${c.label}) \u2014 ${c.linkCount} links`);
+    }
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    const navChoice = await prompt(rl, "\nSelect navigation: ");
+    const navIndex = parseInt(navChoice, 10) - 1;
+    if (isNaN(navIndex) || navIndex < 0 || navIndex >= navCandidates.length) {
+      console.error("Invalid selection.");
+      rl.close();
+      process.exit(1);
+    }
+    const selectedNav = navCandidates[navIndex];
+    const parsedUrl = new URL(url);
+    const defaultPattern = parsedUrl.pathname.replace(/\/$/, "");
+    const allLinks = await page.$$eval(
+      `${selectedNav.selector} a[href]`,
+      (links, pattern) => {
+        return [...new Set(
+          links.map((a) => a.href).filter((h) => h.startsWith("http") && !h.includes("?hl=") && !h.endsWith("#") && h.includes(pattern))
+        )];
+      },
+      defaultPattern
+    );
+    console.log(`
+Found ${allLinks.length} links matching ${defaultPattern}`);
+    const patternInput = await prompt(rl, `Include pattern [default: ${defaultPattern}]: `);
+    const includePattern = patternInput.trim() || defaultPattern;
+    const excludeInput = await prompt(rl, "Exclude patterns (comma-separated, optional): ");
+    const excludePatterns = excludeInput.trim() ? excludeInput.split(",").map((p) => p.trim()) : void 0;
+    rl.close();
+    const contentSelector = await page.evaluate(() => {
+      if (document.querySelector("article")) return "article";
+      if (document.querySelector("main")) return "main";
+      return "body";
+    });
+    const removeSelectors = await page.evaluate(() => {
+      const candidates = [
+        { selector: "nav", label: "nav" },
+        { selector: "footer", label: "footer" },
+        { selector: "[role='complementary']", label: "[role='complementary']" },
+        { selector: "[role='banner']", label: "[role='banner']" },
+        { selector: ".breadcrumbs, .breadcrumb", label: ".breadcrumbs" },
+        { selector: ".pagination-nav, .pagination", label: ".pagination-nav" }
+      ];
+      return candidates.filter((c) => document.querySelector(c.selector) !== null).map((c) => c.label);
+    });
+    if (removeSelectors.length > 0) {
+      console.log(`
+Detected removable elements: ${removeSelectors.join(", ")}`);
+    }
+    const parsedUrlForSitemap = new URL(url);
+    let sitemapUrl;
+    try {
+      const sitemapCheck = await page.goto(`${parsedUrlForSitemap.origin}/sitemap.xml`, { waitUntil: "domcontentloaded", timeout: 1e4 });
+      if (sitemapCheck && sitemapCheck.status() === 200) {
+        const body = await page.textContent("body");
+        if (body && (body.includes("<urlset") || body.includes("<sitemapindex"))) {
+          sitemapUrl = `${parsedUrlForSitemap.origin}/sitemap.xml`;
+          console.log(`
+Sitemap found: ${sitemapUrl}`);
+        }
+      }
+    } catch {
+    }
+    const source = {
+      name: name.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
+      start_url: url,
+      ...sitemapUrl ? { sitemap_url: sitemapUrl } : {},
+      nav_selector: selectedNav.selector,
+      content_selector: contentSelector,
+      include_patterns: [includePattern],
+      ...excludePatterns ? { exclude_patterns: excludePatterns } : {},
+      ...removeSelectors.length > 0 ? { remove_selectors: removeSelectors } : {}
+    };
+    let existingContent = "";
+    try {
+      existingContent = await readFile3(CONFIG_PATH, "utf-8");
+    } catch {
+      existingContent = "sources:\n";
+    }
+    const newEntry = stringify({ [name]: source }, { indent: 2 });
+    const indented = newEntry.split("\n").map((line) => line.trim() ? `  ${line}` : "").join("\n");
+    await writeFile4(CONFIG_PATH, existingContent.trimEnd() + "\n" + indented, "utf-8");
+    console.log(`
+Source "${name}" added to config/sources.yaml`);
+    console.log(`Run "grimoire refresh ${name}" to start scraping.`);
+  } finally {
+    await browser.close();
+  }
+}
+async function loadEmbeddingsCache(cachePath) {
+  try {
+    const data = await readFile3(cachePath, "utf-8");
+    return JSON.parse(data);
+  } catch {
+    return null;
+  }
+}
+async function embedWithCheckpoint(texts, rawDir, embeddingsCachePath) {
+  await mkdir4(rawDir, { recursive: true });
+  const partialCache = await loadEmbeddingsCache(embeddingsCachePath);
+  const resumeFrom = partialCache && partialCache.length > 0 && partialCache.length < texts.length ? partialCache : void 0;
+  return embedTexts(texts, {
+    onProgress: (done, total) => {
+      console.log(`  [${done}/${total}] embedded`);
+    },
+    onCheckpoint: async (current) => {
+      await writeFile4(embeddingsCachePath, JSON.stringify(current), "utf-8");
+    },
+    resumeFrom
+  });
+}
+async function storeWithStrategy(sourceName, allChunks, embeddings, urlCount, version, diff) {
+  if (diff) {
+    console.log("  Computing diff...");
+    const existingIds = await getSourceChunkIds(sourceName);
+    const newIds = new Set(allChunks.map((c) => c.id));
+    const toDelete = [...existingIds].filter((id) => !newIds.has(id));
+    console.log(`  Diff: ${toDelete.length} to delete, ${allChunks.length} to upsert (${existingIds.size} existing)`);
+    if (toDelete.length > 0) {
+      console.log("  Deleting removed chunks...");
+      await deleteChunksByIds(toDelete, (cur, total) => {
+        console.log(`  [${cur}/${total}] deleted`);
+      });
+    }
+    console.log("  Upserting chunks...");
+    await storeChunks(allChunks, embeddings, (cur, total) => {
+      console.log(`  [${cur}/${total}] stored`);
+    });
+  } else {
+    console.log("  Purging old chunks...");
+    await purgeSource(sourceName);
+    console.log("  Storing in Firestore...");
+    await storeChunks(allChunks, embeddings, (cur, total) => {
+      console.log(`  [${cur}/${total}] stored`);
+    });
+  }
+  await updateSourceMeta(sourceName, allChunks.length, urlCount, version);
+  console.log(`  Done. ${allChunks.length} chunks stored for "${sourceName}".`);
+}
+async function cmdRefresh() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      full: { type: "boolean", default: false },
+      all: { type: "boolean", default: false },
+      diff: { type: "boolean", default: false },
+      concurrency: { type: "string" },
+      limit: { type: "string" },
+      "from-html": { type: "boolean", default: false },
+      "from-markdown": { type: "boolean", default: false },
+      "from-embeddings": { type: "boolean", default: false },
+      "skip-store": { type: "boolean", default: false }
+    },
+    allowPositionals: true
+  });
+  const config = await loadConfig(CONFIG_PATH);
+  const sourcesToRefresh = args.values.all ? Object.keys(config.sources) : [args.positionals[0]];
+  if (!args.values.all && !sourcesToRefresh[0]) {
+    console.error("Usage: grimoire refresh <source> [--full] [--from-html] [--from-markdown] [--from-embeddings] [--skip-store] [--limit <n>] [--concurrency <n>]");
+    process.exit(1);
+  }
+  const concurrencyOverride = args.values.concurrency ? parseInt(args.values.concurrency, 10) : void 0;
+  const urlLimit = args.values.limit ? parseInt(args.values.limit, 10) : void 0;
+  for (const sourceName of sourcesToRefresh) {
+    const source = config.sources[sourceName];
+    if (!source) {
+      console.error(`Source "${sourceName}" not found in config.`);
+      process.exit(1);
+    }
+    if (concurrencyOverride) {
+      source.concurrency = concurrencyOverride;
+    }
+    const rawDir = join4(DATA_DIR, "raw", sourceName);
+    const mdDir = join4(DATA_DIR, "markdown", sourceName);
+    const embeddingsCachePath = join4(rawDir, "embeddings.json");
+    console.log(`
+Refreshing "${sourceName}"...`);
+    if (args.values.full) {
+      console.log("  Purging existing chunks...");
+      const deleted = await purgeSource(sourceName);
+      console.log(`  Deleted ${deleted} chunks.`);
+      await rm(rawDir, { recursive: true, force: true });
+      await rm(mdDir, { recursive: true, force: true });
+    }
+    let urls;
+    if (args.values["from-embeddings"]) {
+      console.log("  Loading cached embeddings...");
+      const cached = await loadEmbeddingsCache(embeddingsCachePath);
+      if (!cached) {
+        console.error("  No cached embeddings found. Run without --from-embeddings first.");
+        process.exit(1);
+      }
+      const mdFiles = await readdir(mdDir);
+      const allPages = [];
+      for (const f of mdFiles.filter((f2) => f2.endsWith(".md"))) {
+        const content = await readFile3(join4(mdDir, f), "utf-8");
+        const urlMatch = content.match(/^url: "(.+)"$/m);
+        const titleMatch = content.match(/^title: "(.+)"$/m);
+        allPages.push({
+          markdown: content,
+          url: urlMatch?.[1] ?? "",
+          title: titleMatch?.[1] ?? "Untitled"
+        });
+      }
+      console.log("  Chunking...");
+      const allChunks2 = allPages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
+      console.log(`  Created ${allChunks2.length} chunks.`);
+      if (cached.length !== allChunks2.length) {
+        console.error(`  Embeddings cache (${cached.length}) doesn't match chunk count (${allChunks2.length}). Re-embed with --from-html.`);
+        process.exit(1);
+      }
+      if (args.values["skip-store"]) {
+        console.log(`  Done. ${allChunks2.length} chunks ready (skipped Firestore).`);
+        continue;
+      }
+      await storeWithStrategy(sourceName, allChunks2, cached, allPages.length, source.version, args.values.diff);
+      continue;
+    }
+    if (args.values["from-markdown"]) {
+      console.log("  Reading cached markdown...");
+      const mdFiles = await readdir(mdDir).catch(() => []);
+      const markdownFiles = mdFiles.filter((f) => f.endsWith(".md"));
+      if (markdownFiles.length === 0) {
+        console.error("  No cached markdown found. Run with --from-html first.");
+        process.exit(1);
+      }
+      const pages2 = [];
+      for (const f of markdownFiles) {
+        const content = await readFile3(join4(mdDir, f), "utf-8");
+        const urlMatch = content.match(/^url: "(.+)"$/m);
+        const titleMatch = content.match(/^title: "(.+)"$/m);
+        pages2.push({
+          markdown: content,
+          url: urlMatch?.[1] ?? "",
+          title: titleMatch?.[1] ?? "Untitled"
+        });
+      }
+      console.log(`  Found ${pages2.length} cached pages.`);
+      console.log("  Chunking...");
+      const allChunks2 = pages2.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
+      console.log(`  Created ${allChunks2.length} chunks.`);
+      console.log("  Embedding chunks...");
+      const texts2 = allChunks2.map((c) => c.content);
+      const embeddings2 = await embedWithCheckpoint(texts2, rawDir, embeddingsCachePath);
+      if (args.values["skip-store"]) {
+        console.log(`  Done. ${allChunks2.length} chunks ready (skipped Firestore).`);
+        continue;
+      }
+      await storeWithStrategy(sourceName, allChunks2, embeddings2, pages2.length, source.version, args.values.diff);
+      continue;
+    }
+    let pages;
+    if (source.llms_full_url && !args.values["from-html"]) {
+      console.log(`  Fetching llms-full.txt from ${source.llms_full_url}...`);
+      pages = await ingestLlmsFull(
+        source.llms_full_url,
+        sourceName,
+        source.start_url,
+        DATA_DIR,
+        (cur, total) => {
+          console.log(`  [${cur}/${total}] pages processed`);
+        }
+      );
+      console.log(`  Extracted ${pages.length} pages.`);
+    } else {
+      if (args.values["from-html"]) {
+        console.log("  Reading URLs from cached HTML...");
+        const urlsJsonPath = join4(rawDir, "urls.json");
+        try {
+          urls = JSON.parse(await readFile3(urlsJsonPath, "utf-8"));
+        } catch {
+          const rawFiles = await readdir(rawDir);
+          const htmlFiles = rawFiles.filter((f) => f.endsWith(".html"));
+          urls = [];
+          for (const f of htmlFiles) {
+            const fileSlug = f.replace(/\.html$/, "");
+            const htmlPath = join4(rawDir, f);
+            const html = await readFile3(htmlPath, "utf-8");
+            const match = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/);
+            if (match && slugifyUrl(match[1]) === fileSlug) {
+              urls.push(match[1]);
+              continue;
+            }
+            const ogMatch = html.match(/<meta[^>]+property="og:url"[^>]+content="([^"]+)"/);
+            if (ogMatch && slugifyUrl(ogMatch[1]) === fileSlug) {
+              urls.push(ogMatch[1]);
+              continue;
+            }
+            urls.push(`https://recovered/${fileSlug}`);
+          }
+        }
+        console.log(`  Found ${urls.length} cached pages.`);
+      } else {
+        console.log("  Scraping URLs...");
+        urls = await scrapeSource(source, sourceName, DATA_DIR, (cur, total, url) => {
+          console.log(`  [${cur}/${total}] ${url}`);
+        });
+        console.log(`  Found ${urls.length} pages.`);
+      }
+      if (urlLimit && urls.length > urlLimit) {
+        urls = urls.slice(0, urlLimit);
+        console.log(`  Limited to ${urlLimit} pages.`);
+      }
+      console.log("  Converting to markdown...");
+      pages = await convertSource(
+        sourceName,
+        urls,
+        source.content_selector,
+        source.remove_selectors,
+        source.remove_text_patterns,
+        DATA_DIR,
+        source.concurrency,
+        (cur, total) => {
+          if (cur % 10 === 0 || cur === total) console.log(`  [${cur}/${total}] converted`);
+        }
+      );
+    }
+    console.log("  Chunking...");
+    const allChunks = pages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
+    console.log(`  Created ${allChunks.length} chunks.`);
+    console.log("  Embedding chunks...");
+    const texts = allChunks.map((c) => c.content);
+    const embeddings = await embedWithCheckpoint(texts, rawDir, embeddingsCachePath);
+    if (args.values["skip-store"]) {
+      console.log(`  Done. ${allChunks.length} chunks ready (skipped Firestore).`);
+      continue;
+    }
+    await storeWithStrategy(sourceName, allChunks, embeddings, pages.length, source.version, args.values.diff);
+  }
+}
+async function cmdSearch() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      source: { type: "string" },
+      top: { type: "string" },
+      compact: { type: "boolean", default: false }
+    },
+    allowPositionals: true
+  });
+  const query = args.positionals.join(" ");
+  if (!query) {
+    console.error('Usage: grimoire search "<query>" [--source <name>] [--top <n>] [--compact]');
+    process.exit(1);
+  }
+  const topN = args.values.top ? parseInt(args.values.top, 10) : void 0;
+  const results = await search(query, { source: args.values.source, topN });
+  if (results.length === 0) {
+    console.log("No results found.");
+    return;
+  }
+  if (args.values.compact) {
+    for (const r of results) {
+      console.log(`${r.relevance_score.toFixed(4)} | ${r.source} | ${r.title} | ${r.heading_path.join(" > ")} | ${r.url}`);
+    }
+    return;
+  }
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    console.log(`
+${bold(`[${i + 1}] ${r.title}`)} (${r.relevance_score.toFixed(4)})`);
+    console.log(`    ${cyan(r.url)}`);
+    console.log(`    ${yellow(r.heading_path.join(" > "))}`);
+    console.log(`    ${r.content.replace(/\n/g, " ")}`);
+  }
+}
+async function cmdList() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      names: { type: "boolean", default: false }
+    },
+    allowPositionals: true
+  });
+  const metas = await getAllSourcesMeta();
+  if (metas.length === 0) {
+    console.log("No sources have been refreshed yet.");
+    return;
+  }
+  if (args.values.names) {
+    for (const meta of metas) {
+      console.log(meta.source);
+    }
+    return;
+  }
+  console.log("\nSources:\n");
+  for (const meta of metas) {
+    const ver = meta.version ? ` v${meta.version}` : "";
+    console.log(`  ${bold(meta.source)}${ver}`);
+    console.log(`    ${meta.chunk_count} chunks, ${meta.url_count} URLs, last refreshed ${meta.last_refreshed}`);
+  }
+}
+async function cmdStats() {
+  const metas = await getAllSourcesMeta();
+  if (metas.length === 0) {
+    console.log("No sources have been refreshed yet.");
+    return;
+  }
+  let totalChunks = 0;
+  let totalUrls = 0;
+  console.log("\nSource Statistics:\n");
+  for (const meta of metas) {
+    const ver = meta.version ? ` v${meta.version}` : "";
+    console.log(`  ${bold(meta.source)}${ver}`);
+    console.log(`    Chunks: ${meta.chunk_count}`);
+    console.log(`    URLs: ${meta.url_count}`);
+    console.log(`    Last refreshed: ${meta.last_refreshed}`);
+    totalChunks += meta.chunk_count;
+    totalUrls += meta.url_count;
+  }
+  console.log(`
+  Total: ${totalChunks} chunks across ${totalUrls} URLs from ${metas.length} sources`);
+}
+async function cmdExport() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      format: { type: "string", default: "json" }
+    },
+    allowPositionals: true
+  });
+  const sourceName = args.positionals[0];
+  if (!sourceName) {
+    console.error("Usage: grimoire export <source> [--format json]");
+    process.exit(1);
+  }
+  const mdDir = join4(DATA_DIR, "markdown", sourceName);
+  let files;
+  try {
+    files = await readdir(mdDir);
+  } catch {
+    console.error(`No markdown data found for source "${sourceName}".`);
+    process.exit(1);
+  }
+  const pages = [];
+  for (const file of files.filter((f) => f.endsWith(".md"))) {
+    const content = await readFile3(join4(mdDir, file), "utf-8");
+    pages.push({ file, content });
+  }
+  console.log(JSON.stringify(pages, null, 2));
+}
+async function cmdDelete() {
+  const sourceName = process.argv[3];
+  if (!sourceName) {
+    console.error("Usage: grimoire delete <source>");
+    process.exit(1);
+  }
+  const meta = await getSourceMeta(sourceName);
+  if (!meta) {
+    console.error(`Source "${sourceName}" not found in Firestore.`);
+    process.exit(1);
+  }
+  console.log(`Deleting "${sourceName}" (${meta.chunk_count} chunks)...`);
+  const deleted = await purgeSource(sourceName);
+  await deleteSourceMeta(sourceName);
+  console.log(`Deleted ${deleted} chunks and source metadata for "${sourceName}".`);
+}
+async function cmdScrapeUrls() {
+  const args = parseArgs({
+    args: process.argv.slice(3),
+    options: {
+      concurrency: { type: "string" }
+    },
+    allowPositionals: true
+  });
+  const sourceName = args.positionals[0];
+  if (!sourceName) {
+    console.error("Usage: grimoire scrape-urls <source> [--concurrency <n>]");
+    process.exit(1);
+  }
+  const config = await loadConfig(CONFIG_PATH);
+  const source = config.sources[sourceName];
+  if (!source) {
+    console.error(`Source "${sourceName}" not found in config.`);
+    process.exit(1);
+  }
+  const rawDir = join4(DATA_DIR, "raw", sourceName);
+  const urlsPath = join4(rawDir, "urls.json");
+  let urls;
+  try {
+    urls = JSON.parse(await readFile3(urlsPath, "utf-8"));
+  } catch {
+    console.error(`No urls.json found for "${sourceName}". Run 'grimoire refresh ${sourceName} --skip-store' first.`);
+    process.exit(1);
+  }
+  const missing = urls.filter((url) => !existsSync(join4(rawDir, `${slugifyUrl(url)}.html`)));
+  console.log(`
+Total: ${urls.length}, Cached: ${urls.length - missing.length}, Missing: ${missing.length}`);
+  if (missing.length === 0) {
+    console.log("Nothing to scrape.");
+    return;
+  }
+  const concurrency = args.values.concurrency ? parseInt(args.values.concurrency, 10) : source.concurrency ?? 20;
+  const browser = await createBrowser();
+  const context = await browser.newContext(source.user_agent ? { userAgent: source.user_agent } : {});
+  let done = 0;
+  for (let i = 0; i < missing.length; i += concurrency) {
+    const batch = missing.slice(i, i + concurrency);
+    await Promise.all(batch.map(async (url) => {
+      const page = await context.newPage();
+      try {
+        await page.goto(url, { waitUntil: source.headed ? "networkidle" : "domcontentloaded", timeout: 3e4 });
+        const html = await page.content();
+        await writeFile4(join4(rawDir, `${slugifyUrl(url)}.html`), html, "utf-8");
+        done++;
+        if (done % 10 === 0 || done === missing.length) console.log(`  [${done}/${missing.length}]`);
+      } catch (e) {
+        console.error(`  FAILED: ${url} - ${e instanceof Error ? e.message : String(e)}`);
+      } finally {
+        await page.close();
+      }
+    }));
+  }
+  console.log(`Done. Fetched ${done} pages.`);
+  await browser.close();
+}
+var ADMIN_COMMANDS = {
+  add: cmdAdd,
+  refresh: cmdRefresh,
+  delete: cmdDelete,
+  "scrape-urls": cmdScrapeUrls,
+  search: cmdSearch,
+  list: cmdList,
+  stats: cmdStats,
+  export: cmdExport,
+  apikey: cmdApiKey
+};
+export {
+  ADMIN_COMMANDS
+};
+//# sourceMappingURL=admin-HA6FNUV4.js.map