@astrofoundry/grimoire 3.12.3 → 3.14.0

package/dist/cli.js

@@ -1,656 +1,232 @@
 #!/usr/bin/env node
+import {
+  bold,
+  cyan,
+  yellow
+} from "./chunk-BRS6X3AE.js";
+
+// src/cli.ts
 import { parseArgs } from "node:util";
-import { readFile, writeFile, readdir, rm, mkdir } from "node:fs/promises";
 import { readFileSync, existsSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { join as join2, resolve } from "node:path";
+
+// src/consumer-config.ts
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { homedir } from "node:os";
 import { createInterface } from "node:readline";
-import { stringify } from "yaml";
-import { bold, cyan, yellow } from "./format.js";
-import { detectConsumerMode, resolveConsumerConfig, cmdInit } from "./consumer-config.js";
-import { cmdConsumerSearch, cmdConsumerList, cmdConsumerStats } from "./consumer.js";
-import { loadConfig } from "./config.js";
-import { scrapeSource, createBrowser, slugifyUrl } from "./scraper.js";
-import { convertSource } from "./converter.js";
-import { chunkMarkdown } from "./chunker.js";
-import { embedTexts } from "./embedder.js";
-import { storeChunks, purgeSource, updateSourceMeta, getAllSourcesMeta, getSourceMeta, deleteSourceMeta, deleteChunksByIds, getSourceChunkIds } from "./store.js";
-import { search } from "./search.js";
-import { cmdApiKey } from "./apikey.js";
-import { ingestLlmsFull } from "./llms-ingest.js";
-const PROJECT_ROOT = resolve(import.meta.dirname, "..");
-const CONFIG_PATH = join(PROJECT_ROOT, "config", "sources.yaml");
-const DATA_DIR = join(PROJECT_ROOT, "data");
-const envPath = join(PROJECT_ROOT, ".env");
-if (existsSync(envPath)) {
-    for (const line of readFileSync(envPath, "utf-8").split("\n")) {
-        const trimmed = line.trim();
-        if (!trimmed || trimmed.startsWith("#"))
-            continue;
-        const eqIndex = trimmed.indexOf("=");
-        if (eqIndex === -1)
-            continue;
-        const key = trimmed.slice(0, eqIndex);
-        const value = trimmed.slice(eqIndex + 1);
-        if (!process.env[key]) {
-            process.env[key] = value;
-        }
-    }
-}
-function prompt(rl, question) {
-    return new Promise((resolve) => rl.question(question, resolve));
-}
-async function cmdAdd() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            url: { type: "string" },
-        },
-        allowPositionals: true,
-    });
-    const name = args.positionals[0];
-    const url = args.values.url;
-    if (!name || !url) {
-        console.error("Usage: grimoire add <name> --url <start_url>");
-        process.exit(1);
-    }
-    console.log("Scanning page...\n");
-    const browser = await createBrowser();
-    const context = await browser.newContext();
-    const page = await context.newPage();
-    try {
-        await page.goto(url, { waitUntil: "domcontentloaded" });
-        const navCandidates = await page.evaluate(() => {
-            const selectors = ["nav", "[role='navigation']"];
-            const results = [];
-            const seen = new Set();
-            for (const sel of selectors) {
-                for (const el of document.querySelectorAll(sel)) {
-                    if (seen.has(el))
-                        continue;
-                    seen.add(el);
-                    const links = el.querySelectorAll("a[href]");
-                    const label = el.getAttribute("aria-label") ||
-                        el.getAttribute("class") ||
-                        el.tagName.toLowerCase();
-                    results.push({
-                        selector: sel,
-                        label,
-                        linkCount: links.length,
-                    });
-                }
-            }
-            return results.sort((a, b) => b.linkCount - a.linkCount);
-        });
-        if (navCandidates.length === 0) {
-            console.error("No navigation elements found on this page.");
-            process.exit(1);
-        }
-        console.log("Navigation candidates:");
-        for (let i = 0; i < navCandidates.length; i++) {
-            const c = navCandidates[i];
-            console.log(`  [${i + 1}] ${c.selector} (${c.label}) — ${c.linkCount} links`);
-        }
-        const rl = createInterface({ input: process.stdin, output: process.stdout });
-        const navChoice = await prompt(rl, "\nSelect navigation: ");
-        const navIndex = parseInt(navChoice, 10) - 1;
-        if (isNaN(navIndex) || navIndex < 0 || navIndex >= navCandidates.length) {
-            console.error("Invalid selection.");
-            rl.close();
-            process.exit(1);
-        }
-        const selectedNav = navCandidates[navIndex];
-        const parsedUrl = new URL(url);
-        const defaultPattern = parsedUrl.pathname.replace(/\/$/, "");
-        const allLinks = await page.$$eval(`${selectedNav.selector} a[href]`, (links, pattern) => {
-            return [...new Set(links
-                    .map((a) => a.href)
-                    .filter((h) => h.startsWith("http") && !h.includes("?hl=") && !h.endsWith("#") && h.includes(pattern)))];
-        }, defaultPattern);
-        console.log(`\nFound ${allLinks.length} links matching ${defaultPattern}`);
-        const patternInput = await prompt(rl, `Include pattern [default: ${defaultPattern}]: `);
-        const includePattern = patternInput.trim() || defaultPattern;
-        const excludeInput = await prompt(rl, "Exclude patterns (comma-separated, optional): ");
-        const excludePatterns = excludeInput.trim()
-            ? excludeInput.split(",").map((p) => p.trim())
-            : undefined;
-        rl.close();
-        const contentSelector = await page.evaluate(() => {
-            if (document.querySelector("article"))
-                return "article";
-            if (document.querySelector("main"))
-                return "main";
-            return "body";
-        });
-        const removeSelectors = await page.evaluate(() => {
-            const candidates = [
-                { selector: "nav", label: "nav" },
-                { selector: "footer", label: "footer" },
-                { selector: "[role='complementary']", label: "[role='complementary']" },
-                { selector: "[role='banner']", label: "[role='banner']" },
-                { selector: ".breadcrumbs, .breadcrumb", label: ".breadcrumbs" },
-                { selector: ".pagination-nav, .pagination", label: ".pagination-nav" },
-            ];
-            return candidates
-                .filter((c) => document.querySelector(c.selector) !== null)
-                .map((c) => c.label);
-        });
-        if (removeSelectors.length > 0) {
-            console.log(`\nDetected removable elements: ${removeSelectors.join(", ")}`);
-        }
-        const parsedUrlForSitemap = new URL(url);
-        let sitemapUrl;
-        try {
-            const sitemapCheck = await page.goto(`${parsedUrlForSitemap.origin}/sitemap.xml`, { waitUntil: "domcontentloaded", timeout: 10000 });
-            if (sitemapCheck && sitemapCheck.status() === 200) {
-                const body = await page.textContent("body");
-                if (body && (body.includes("<urlset") || body.includes("<sitemapindex"))) {
-                    sitemapUrl = `${parsedUrlForSitemap.origin}/sitemap.xml`;
-                    console.log(`\nSitemap found: ${sitemapUrl}`);
-                }
-            }
-        }
-        catch {
-            // No sitemap available
-        }
-        const source = {
-            name: name.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
-            start_url: url,
-            ...(sitemapUrl ? { sitemap_url: sitemapUrl } : {}),
-            nav_selector: selectedNav.selector,
-            content_selector: contentSelector,
-            include_patterns: [includePattern],
-            ...(excludePatterns ? { exclude_patterns: excludePatterns } : {}),
-            ...(removeSelectors.length > 0 ? { remove_selectors: removeSelectors } : {}),
-        };
-        let existingContent = "";
-        try {
-            existingContent = await readFile(CONFIG_PATH, "utf-8");
-        }
-        catch {
-            existingContent = "sources:\n";
-        }
-        const newEntry = stringify({ [name]: source }, { indent: 2 });
-        const indented = newEntry
-            .split("\n")
-            .map((line) => (line.trim() ? `  ${line}` : ""))
-            .join("\n");
-        await writeFile(CONFIG_PATH, existingContent.trimEnd() + "\n" + indented, "utf-8");
-        console.log(`\nSource "${name}" added to config/sources.yaml`);
-        console.log(`Run "grimoire refresh ${name}" to start scraping.`);
-    }
-    finally {
-        await browser.close();
-    }
+var CONFIG_DIR = join(homedir(), ".grimoire");
+var CONFIG_FILE = join(CONFIG_DIR, "config.json");
+async function loadConsumerConfig() {
+  const raw = await readFile(CONFIG_FILE, "utf-8").catch(() => null);
+  if (!raw) return null;
+  const data = JSON.parse(raw);
+  if (typeof data.apiUrl === "string" && typeof data.apiKey === "string") {
+    return { apiUrl: data.apiUrl, apiKey: data.apiKey };
+  }
+  return null;
 }
-async function loadEmbeddingsCache(cachePath) {
-    try {
-        const data = await readFile(cachePath, "utf-8");
-        return JSON.parse(data);
-    }
-    catch {
-        return null;
-    }
+async function saveConsumerConfig(config) {
+  await mkdir(CONFIG_DIR, { recursive: true });
+  await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2) + "\n", "utf-8");
 }
-async function embedWithCheckpoint(texts, rawDir, embeddingsCachePath) {
-    await mkdir(rawDir, { recursive: true });
-    const partialCache = await loadEmbeddingsCache(embeddingsCachePath);
-    const resumeFrom = partialCache && partialCache.length > 0 && partialCache.length < texts.length ? partialCache : undefined;
-    return embedTexts(texts, {
-        onProgress: (done, total) => {
-            console.log(`  [${done}/${total}] embedded`);
-        },
-        onCheckpoint: async (current) => {
-            await writeFile(embeddingsCachePath, JSON.stringify(current), "utf-8");
-        },
-        resumeFrom,
-    });
+async function resolveConsumerConfig() {
+  const envUrl = process.env.GRIMOIRE_API_URL;
+  const envKey = process.env.GRIMOIRE_API_KEY;
+  if (envUrl && envKey) {
+    return { apiUrl: envUrl, apiKey: envKey };
+  }
+  const fileConfig = await loadConsumerConfig();
+  if (fileConfig) return fileConfig;
+  throw new Error("Grimoire is not configured. Run 'grimoire init' to set up.");
 }
-async function storeWithStrategy(sourceName, allChunks, embeddings, urlCount, version, diff) {
-    if (diff) {
-        console.log("  Computing diff...");
-        const existingIds = await getSourceChunkIds(sourceName);
-        const newIds = new Set(allChunks.map((c) => c.id));
-        const toDelete = [...existingIds].filter((id) => !newIds.has(id));
-        console.log(`  Diff: ${toDelete.length} to delete, ${allChunks.length} to upsert (${existingIds.size} existing)`);
-        if (toDelete.length > 0) {
-            console.log("  Deleting removed chunks...");
-            await deleteChunksByIds(toDelete, (cur, total) => {
-                console.log(`  [${cur}/${total}] deleted`);
-            });
-        }
-        console.log("  Upserting chunks...");
-        await storeChunks(allChunks, embeddings, (cur, total) => {
-            console.log(`  [${cur}/${total}] stored`);
-        });
-    }
-    else {
-        console.log("  Purging old chunks...");
-        await purgeSource(sourceName);
-        console.log("  Storing in Firestore...");
-        await storeChunks(allChunks, embeddings, (cur, total) => {
-            console.log(`  [${cur}/${total}] stored`);
-        });
-    }
-    await updateSourceMeta(sourceName, allChunks.length, urlCount, version);
-    console.log(`  Done. ${allChunks.length} chunks stored for "${sourceName}".`);
-}
-async function cmdRefresh() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            full: { type: "boolean", default: false },
-            all: { type: "boolean", default: false },
-            diff: { type: "boolean", default: false },
-            concurrency: { type: "string" },
-            limit: { type: "string" },
-            "from-html": { type: "boolean", default: false },
-            "from-markdown": { type: "boolean", default: false },
-            "from-embeddings": { type: "boolean", default: false },
-            "skip-store": { type: "boolean", default: false },
-        },
-        allowPositionals: true,
-    });
-    const config = await loadConfig(CONFIG_PATH);
-    const sourcesToRefresh = args.values.all
-        ? Object.keys(config.sources)
-        : [args.positionals[0]];
-    if (!args.values.all && !sourcesToRefresh[0]) {
-        console.error("Usage: grimoire refresh <source> [--full] [--from-html] [--from-markdown] [--from-embeddings] [--skip-store] [--limit <n>] [--concurrency <n>]");
-        process.exit(1);
-    }
-    const concurrencyOverride = args.values.concurrency ? parseInt(args.values.concurrency, 10) : undefined;
-    const urlLimit = args.values.limit ? parseInt(args.values.limit, 10) : undefined;
-    for (const sourceName of sourcesToRefresh) {
-        const source = config.sources[sourceName];
-        if (!source) {
-            console.error(`Source "${sourceName}" not found in config.`);
-            process.exit(1);
-        }
-        if (concurrencyOverride) {
-            source.concurrency = concurrencyOverride;
-        }
-        const rawDir = join(DATA_DIR, "raw", sourceName);
-        const mdDir = join(DATA_DIR, "markdown", sourceName);
-        const embeddingsCachePath = join(rawDir, "embeddings.json");
-        console.log(`\nRefreshing "${sourceName}"...`);
-        if (args.values.full) {
-            console.log("  Purging existing chunks...");
-            const deleted = await purgeSource(sourceName);
-            console.log(`  Deleted ${deleted} chunks.`);
-            await rm(rawDir, { recursive: true, force: true });
-            await rm(mdDir, { recursive: true, force: true });
-        }
-        let urls;
-        if (args.values["from-embeddings"]) {
-            console.log("  Loading cached embeddings...");
-            const cached = await loadEmbeddingsCache(embeddingsCachePath);
-            if (!cached) {
-                console.error("  No cached embeddings found. Run without --from-embeddings first.");
-                process.exit(1);
-            }
-            const mdFiles = await readdir(mdDir);
-            const allPages = [];
-            for (const f of mdFiles.filter((f) => f.endsWith(".md"))) {
-                const content = await readFile(join(mdDir, f), "utf-8");
-                const urlMatch = content.match(/^url: "(.+)"$/m);
-                const titleMatch = content.match(/^title: "(.+)"$/m);
-                allPages.push({
-                    markdown: content,
-                    url: urlMatch?.[1] ?? "",
-                    title: titleMatch?.[1] ?? "Untitled",
-                });
-            }
-            console.log("  Chunking...");
-            const allChunks = allPages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
-            console.log(`  Created ${allChunks.length} chunks.`);
-            if (cached.length !== allChunks.length) {
-                console.error(`  Embeddings cache (${cached.length}) doesn't match chunk count (${allChunks.length}). Re-embed with --from-html.`);
-                process.exit(1);
-            }
-            if (args.values["skip-store"]) {
-                console.log(`  Done. ${allChunks.length} chunks ready (skipped Firestore).`);
-                continue;
-            }
-            await storeWithStrategy(sourceName, allChunks, cached, allPages.length, source.version, args.values.diff);
-            continue;
-        }
-        if (args.values["from-markdown"]) {
-            console.log("  Reading cached markdown...");
-            const mdFiles = await readdir(mdDir).catch(() => []);
-            const markdownFiles = mdFiles.filter((f) => f.endsWith(".md"));
-            if (markdownFiles.length === 0) {
-                console.error("  No cached markdown found. Run with --from-html first.");
-                process.exit(1);
-            }
-            const pages = [];
-            for (const f of markdownFiles) {
-                const content = await readFile(join(mdDir, f), "utf-8");
-                const urlMatch = content.match(/^url: "(.+)"$/m);
-                const titleMatch = content.match(/^title: "(.+)"$/m);
-                pages.push({
-                    markdown: content,
-                    url: urlMatch?.[1] ?? "",
-                    title: titleMatch?.[1] ?? "Untitled",
-                });
-            }
-            console.log(`  Found ${pages.length} cached pages.`);
-            console.log("  Chunking...");
-            const allChunks = pages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
-            console.log(`  Created ${allChunks.length} chunks.`);
-            console.log("  Embedding chunks...");
-            const texts = allChunks.map((c) => c.content);
-            const embeddings = await embedWithCheckpoint(texts, rawDir, embeddingsCachePath);
-            if (args.values["skip-store"]) {
-                console.log(`  Done. ${allChunks.length} chunks ready (skipped Firestore).`);
-                continue;
-            }
-            await storeWithStrategy(sourceName, allChunks, embeddings, pages.length, source.version, args.values.diff);
-            continue;
-        }
-        let pages;
-        if (source.llms_full_url && !args.values["from-html"]) {
-            console.log(`  Fetching llms-full.txt from ${source.llms_full_url}...`);
-            pages = await ingestLlmsFull(source.llms_full_url, sourceName, source.start_url, DATA_DIR, (cur, total) => {
-                console.log(`  [${cur}/${total}] pages processed`);
-            });
-            console.log(`  Extracted ${pages.length} pages.`);
-        }
-        else {
-            if (args.values["from-html"]) {
-                console.log("  Reading URLs from cached HTML...");
-                const urlsJsonPath = join(rawDir, "urls.json");
-                try {
-                    urls = JSON.parse(await readFile(urlsJsonPath, "utf-8"));
-                }
-                catch {
-                    const rawFiles = await readdir(rawDir);
-                    const htmlFiles = rawFiles.filter((f) => f.endsWith(".html"));
-                    urls = [];
-                    for (const f of htmlFiles) {
-                        const fileSlug = f.replace(/\.html$/, "");
-                        const htmlPath = join(rawDir, f);
-                        const html = await readFile(htmlPath, "utf-8");
-                        const match = html.match(/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/);
-                        if (match && slugifyUrl(match[1]) === fileSlug) {
-                            urls.push(match[1]);
-                            continue;
-                        }
-                        const ogMatch = html.match(/<meta[^>]+property="og:url"[^>]+content="([^"]+)"/);
-                        if (ogMatch && slugifyUrl(ogMatch[1]) === fileSlug) {
-                            urls.push(ogMatch[1]);
-                            continue;
-                        }
-                        urls.push(`https://recovered/${fileSlug}`);
-                    }
-                }
-                console.log(`  Found ${urls.length} cached pages.`);
-            }
-            else {
-                console.log("  Scraping URLs...");
-                urls = await scrapeSource(source, sourceName, DATA_DIR, (cur, total, url) => {
-                    console.log(`  [${cur}/${total}] ${url}`);
-                });
-                console.log(`  Found ${urls.length} pages.`);
-            }
-            if (urlLimit && urls.length > urlLimit) {
-                urls = urls.slice(0, urlLimit);
-                console.log(`  Limited to ${urlLimit} pages.`);
-            }
-            console.log("  Converting to markdown...");
-            pages = await convertSource(sourceName, urls, source.content_selector, source.remove_selectors, source.remove_text_patterns, DATA_DIR, source.concurrency, (cur, total) => {
-                if (cur % 10 === 0 || cur === total)
-                    console.log(`  [${cur}/${total}] converted`);
-            });
-        }
-        console.log("  Chunking...");
-        const allChunks = pages.flatMap((p) => chunkMarkdown(p.markdown, sourceName, p.url, p.title));
-        console.log(`  Created ${allChunks.length} chunks.`);
-        console.log("  Embedding chunks...");
-        const texts = allChunks.map((c) => c.content);
-        const embeddings = await embedWithCheckpoint(texts, rawDir, embeddingsCachePath);
-        if (args.values["skip-store"]) {
-            console.log(`  Done. ${allChunks.length} chunks ready (skipped Firestore).`);
-            continue;
-        }
-        await storeWithStrategy(sourceName, allChunks, embeddings, pages.length, source.version, args.values.diff);
-    }
+async function detectConsumerMode() {
+  if (process.env.GOOGLE_APPLICATION_CREDENTIALS) return false;
+  if (process.env.GRIMOIRE_API_URL) return true;
+  const config = await loadConsumerConfig();
+  return config !== null;
 }
-async function cmdSearch() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            source: { type: "string" },
-            top: { type: "string" },
-            compact: { type: "boolean", default: false },
-        },
-        allowPositionals: true,
-    });
-    const query = args.positionals.join(" ");
-    if (!query) {
-        console.error("Usage: grimoire search \"<query>\" [--source <name>] [--top <n>] [--compact]");
-        process.exit(1);
-    }
-    const topN = args.values.top ? parseInt(args.values.top, 10) : undefined;
-    const results = await search(query, { source: args.values.source, topN });
-    if (results.length === 0) {
-        console.log("No results found.");
-        return;
-    }
-    if (args.values.compact) {
-        for (const r of results) {
-            console.log(`${r.relevance_score.toFixed(4)} | ${r.source} | ${r.title} | ${r.heading_path.join(" > ")} | ${r.url}`);
-        }
-        return;
-    }
-    for (let i = 0; i < results.length; i++) {
-        const r = results[i];
-        console.log(`\n${bold(`[${i + 1}] ${r.title}`)} (${r.relevance_score.toFixed(4)})`);
-        console.log(`    ${cyan(r.url)}`);
-        console.log(`    ${yellow(r.heading_path.join(" > "))}`);
-        console.log(`    ${r.content.replace(/\n/g, " ")}`);
-    }
+async function cmdInit() {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  const ask = (q) => new Promise((resolve2) => rl.question(q, resolve2));
+  const existing = await loadConsumerConfig();
+  const apiUrl = await ask(`API URL${existing ? ` [${existing.apiUrl}]` : ""}: `);
+  const apiKey = await ask(`API Key${existing ? " [****]" : ""}: `);
+  const config = {
+    apiUrl: apiUrl.trim() || existing?.apiUrl || "",
+    apiKey: apiKey.trim() || existing?.apiKey || ""
+  };
+  rl.close();
+  if (!config.apiUrl || !config.apiKey) {
+    throw new Error("Both API URL and API Key are required.");
+  }
+  await saveConsumerConfig(config);
+  console.log(`
+Saved to ${CONFIG_FILE}`);
 }
-async function cmdList() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            names: { type: "boolean", default: false },
-        },
-        allowPositionals: true,
+
+// src/consumer.ts
+async function apiRequest(config, path, options) {
+  const url = `${config.apiUrl.replace(/\/$/, "")}${path}`;
+  let response;
+  try {
+    response = await fetch(url, {
+      ...options,
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": config.apiKey,
+        ...options?.headers
+      }
     });
-    const metas = await getAllSourcesMeta();
-    if (metas.length === 0) {
-        console.log("No sources have been refreshed yet.");
-        return;
-    }
-    if (args.values.names) {
-        for (const meta of metas) {
-            console.log(meta.source);
-        }
-        return;
-    }
-    console.log("\nSources:\n");
-    for (const meta of metas) {
-        const ver = meta.version ? ` v${meta.version}` : "";
-        console.log(`  ${bold(meta.source)}${ver}`);
-        console.log(`    ${meta.chunk_count} chunks, ${meta.url_count} URLs, last refreshed ${meta.last_refreshed}`);
-    }
+  } catch {
+    throw new Error(`Cannot reach Grimoire API at ${config.apiUrl}. Check your GRIMOIRE_API_URL.`);
+  }
+  if (response.status === 401 || response.status === 403) {
+    throw new Error("Invalid API key. Check your GRIMOIRE_API_KEY or run 'grimoire init'.");
+  }
+  if (!response.ok) {
+    throw new Error(`API error: ${response.status} ${response.statusText}`);
+  }
+  return response.json();
 }
-async function cmdStats() {
-    const metas = await getAllSourcesMeta();
-    if (metas.length === 0) {
-        console.log("No sources have been refreshed yet.");
-        return;
-    }
-    let totalChunks = 0;
-    let totalUrls = 0;
-    console.log("\nSource Statistics:\n");
-    for (const meta of metas) {
-        const ver = meta.version ? ` v${meta.version}` : "";
-        console.log(`  ${bold(meta.source)}${ver}`);
-        console.log(`    Chunks: ${meta.chunk_count}`);
-        console.log(`    URLs: ${meta.url_count}`);
-        console.log(`    Last refreshed: ${meta.last_refreshed}`);
-        totalChunks += meta.chunk_count;
-        totalUrls += meta.url_count;
-    }
-    console.log(`\n  Total: ${totalChunks} chunks across ${totalUrls} URLs from ${metas.length} sources`);
+async function cmdConsumerSearch(config, query, options) {
+  const data = await apiRequest(config, "/search", {
+    method: "POST",
+    body: JSON.stringify({ query, source: options.source, topN: options.topN })
+  });
+  if (data.results.length === 0) {
+    console.log("No results found.");
+    return;
+  }
+  if (options.compact) {
+    for (const r of data.results) {
+      console.log(`${r.relevance_score.toFixed(4)} | ${r.source} | ${r.title} | ${r.heading_path.join(" > ")} | ${r.url}`);
+    }
+    return;
+  }
+  for (let i = 0; i < data.results.length; i++) {
+    const r = data.results[i];
+    console.log(`
+${bold(`[${i + 1}] ${r.title}`)} (${r.relevance_score.toFixed(4)})`);
+    console.log(`    ${cyan(r.url)}`);
+    console.log(`    ${yellow(r.heading_path.join(" > "))}`);
+    console.log(`    ${r.content.replace(/\n/g, " ")}`);
+  }
 }
-async function cmdExport() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            format: { type: "string", default: "json" },
-        },
-        allowPositionals: true,
-    });
-    const sourceName = args.positionals[0];
-    if (!sourceName) {
-        console.error("Usage: grimoire export <source> [--format json]");
-        process.exit(1);
-    }
-    const mdDir = join(DATA_DIR, "markdown", sourceName);
-    let files;
-    try {
-        files = await readdir(mdDir);
-    }
-    catch {
-        console.error(`No markdown data found for source "${sourceName}".`);
-        process.exit(1);
-    }
-    const pages = [];
-    for (const file of files.filter((f) => f.endsWith(".md"))) {
-        const content = await readFile(join(mdDir, file), "utf-8");
-        pages.push({ file, content });
-    }
-    console.log(JSON.stringify(pages, null, 2));
+async function cmdConsumerList(config, options) {
+  const data = await apiRequest(config, "/list");
+  if (data.sources.length === 0) {
+    console.log("No sources available.");
+    return;
+  }
+  if (options?.names) {
+    for (const s of data.sources) {
+      console.log(s.source);
+    }
+    return;
+  }
+  console.log("\nSources:\n");
+  for (const s of data.sources) {
+    const ver = s.version ? ` v${s.version}` : "";
+    console.log(`  ${bold(s.source)}${ver}`);
+    console.log(`    ${s.chunk_count} chunks, ${s.url_count} URLs, last refreshed ${s.last_refreshed}`);
+  }
 }
-async function cmdDelete() {
-    const sourceName = process.argv[3];
-    if (!sourceName) {
-        console.error("Usage: grimoire delete <source>");
-        process.exit(1);
-    }
-    const meta = await getSourceMeta(sourceName);
-    if (!meta) {
-        console.error(`Source "${sourceName}" not found in Firestore.`);
-        process.exit(1);
-    }
-    console.log(`Deleting "${sourceName}" (${meta.chunk_count} chunks)...`);
-    const deleted = await purgeSource(sourceName);
-    await deleteSourceMeta(sourceName);
-    console.log(`Deleted ${deleted} chunks and source metadata for "${sourceName}".`);
+async function cmdConsumerStats(config) {
+  const data = await apiRequest(config, "/stats");
+  if (data.sources.length === 0) {
+    console.log("No sources have been refreshed yet.");
+    return;
+  }
+  console.log("\nSource Statistics:\n");
+  for (const s of data.sources) {
+    const ver = s.version ? ` v${s.version}` : "";
+    console.log(`  ${bold(s.source)}${ver}`);
+    console.log(`    Chunks: ${s.chunk_count}`);
+    console.log(`    URLs: ${s.url_count}`);
+    console.log(`    Last refreshed: ${s.last_refreshed}`);
+  }
+  console.log(`
+  Total: ${data.totalChunks} chunks across ${data.totalUrls} URLs from ${data.sources.length} sources`);
 }
-async function cmdScrapeUrls() {
-    const args = parseArgs({
-        args: process.argv.slice(3),
-        options: {
-            concurrency: { type: "string" },
-        },
-        allowPositionals: true,
-    });
-    const sourceName = args.positionals[0];
-    if (!sourceName) {
-        console.error("Usage: grimoire scrape-urls <source> [--concurrency <n>]");
-        process.exit(1);
-    }
-    const config = await loadConfig(CONFIG_PATH);
-    const source = config.sources[sourceName];
-    if (!source) {
-        console.error(`Source "${sourceName}" not found in config.`);
-        process.exit(1);
-    }
-    const rawDir = join(DATA_DIR, "raw", sourceName);
-    const urlsPath = join(rawDir, "urls.json");
-    let urls;
-    try {
-        urls = JSON.parse(await readFile(urlsPath, "utf-8"));
-    }
-    catch {
-        console.error(`No urls.json found for "${sourceName}". Run 'grimoire refresh ${sourceName} --skip-store' first.`);
-        process.exit(1);
-    }
-    const missing = urls.filter((url) => !existsSync(join(rawDir, `${slugifyUrl(url)}.html`)));
-    console.log(`\nTotal: ${urls.length}, Cached: ${urls.length - missing.length}, Missing: ${missing.length}`);
-    if (missing.length === 0) {
-        console.log("Nothing to scrape.");
-        return;
-    }
-    const concurrency = args.values.concurrency ? parseInt(args.values.concurrency, 10) : source.concurrency ?? 20;
-    const browser = await createBrowser();
-    const context = await browser.newContext(source.user_agent ? { userAgent: source.user_agent } : {});
-    let done = 0;
-    for (let i = 0; i < missing.length; i += concurrency) {
-        const batch = missing.slice(i, i + concurrency);
-        await Promise.all(batch.map(async (url) => {
-            const page = await context.newPage();
-            try {
-                await page.goto(url, { waitUntil: source.headed ? "networkidle" : "domcontentloaded", timeout: 30000 });
-                const html = await page.content();
-                await writeFile(join(rawDir, `${slugifyUrl(url)}.html`), html, "utf-8");
-                done++;
-                if (done % 10 === 0 || done === missing.length)
-                    console.log(`  [${done}/${missing.length}]`);
-            }
-            catch (e) {
-                console.error(`  FAILED: ${url} - ${e instanceof Error ? e.message : String(e)}`);
-            }
-            finally {
-                await page.close();
-            }
-        }));
-    }
-    console.log(`Done. Fetched ${done} pages.`);
-    await browser.close();
+
+// src/cli.ts
+var PROJECT_ROOT = resolve(import.meta.dirname, "..");
+var envPath = join2(PROJECT_ROOT, ".env");
+if (existsSync(envPath)) {
+  for (const line of readFileSync(envPath, "utf-8").split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIndex = trimmed.indexOf("=");
+    if (eqIndex === -1) continue;
+    const key = trimmed.slice(0, eqIndex);
+    const value = trimmed.slice(eqIndex + 1);
+    if (!process.env[key]) {
+      process.env[key] = value;
+    }
+  }
 }
+var ADMIN_ONLY_COMMANDS = ["add", "refresh", "delete", "scrape-urls", "export", "apikey"];
 async function cmdUpdate() {
-    const { execSync } = await import("node:child_process");
-    const pkg = JSON.parse(readFileSync(join(PROJECT_ROOT, "package.json"), "utf-8"));
-    console.log(`Current version: ${pkg.version}`);
-    console.log("Checking for updates...");
-    execSync("pnpm add -g @astrofoundry/grimoire@latest", { stdio: "inherit" });
-    const updated = JSON.parse(readFileSync(join(PROJECT_ROOT, "package.json"), "utf-8"));
-    if (updated.version === pkg.version) {
-        console.log("Already on the latest version.");
-    }
-    else {
-        console.log(`Updated to ${updated.version}.`);
-    }
+  const { execSync } = await import("node:child_process");
+  const pkg = JSON.parse(readFileSync(join2(PROJECT_ROOT, "package.json"), "utf-8"));
+  console.log(`Current version: ${pkg.version}`);
+  console.log("Checking for updates...");
+  execSync("npm install -g @astrofoundry/grimoire@latest", { stdio: "inherit" });
+  const updated = JSON.parse(readFileSync(join2(PROJECT_ROOT, "package.json"), "utf-8"));
+  if (updated.version === pkg.version) {
+    console.log("Already on the latest version.");
+  } else {
+    console.log(`Updated to ${updated.version}.`);
+  }
 }
-const ADMIN_COMMANDS = {
-    add: cmdAdd,
-    refresh: cmdRefresh,
-    delete: cmdDelete,
-    "scrape-urls": cmdScrapeUrls,
-    update: cmdUpdate,
-    search: cmdSearch,
-    list: cmdList,
-    stats: cmdStats,
-    export: cmdExport,
-    apikey: cmdApiKey,
-};
-const ADMIN_ONLY_COMMANDS = ["add", "refresh", "delete", "scrape-urls", "export", "apikey"];
 function showHelp(isConsumer) {
-    if (isConsumer) {
-        console.log(`
-grimoire — Documentation Search
+  if (isConsumer) {
+    console.log(`
+grimoire \u2014 Documentation RAG
 
-Commands:
-  search "<query>" [--source <n>] [--top <n>] [--compact]
-                                   Search documentation
-  list [--names]                   List available sources
-  stats                            Show statistics
-  init                             Configure API connection
-  update                           Update grimoire to the latest version
+USAGE
+  grimoire search "<query>" [--source <name>] [--top <n>] [--compact]
+
+QUERY TIPS
+  - Use the library's own terminology ("Firestore batched writes"
+    not "how do I do multi-write in firestore")
+  - Scope with --source when you know the area
+  - Rephrase if first search misses
+  - Prefer --compact for scanning; omit for full snippets
+  - Cite the URL in your answer when precision matters
+
+EXAMPLES
+  grimoire search "Firestore batched writes" --source firebase-firestore --compact
+  grimoire search "react server components" --top 3
+  grimoire list --names
+
+FLAGS
+  --source <name>   Scope to one indexed source. Run \`grimoire list --names\`
+                    for the full list (sources are added regularly).
+  --top <n>         Max results. Default: 5.
+  --compact         One line per result: score | source | title | heading | url
+                    Default (non-compact): multi-line block with title, URL,
+                    heading path, and content snippet.
 
-Environment:
-  GRIMOIRE_API_URL                 API endpoint URL
-  GRIMOIRE_API_KEY                 API key
+RELEVANCE SCORES
+  Range 0\u20131 (higher = better). >0.85 strong match, 0.6\u20130.85 relevant,
+  <0.6 usually too weak to cite. "No results found." + exit 0 = clean miss.
+
+MANAGEMENT
+  grimoire list [--names]   Show indexed sources
+  grimoire stats            Index statistics
+  grimoire init             Configure API connection (first-time setup)
+  grimoire update           Update grimoire itself
+  grimoire --version        Print CLI version
+
+ENVIRONMENT
+  GRIMOIRE_API_URL          API endpoint URL
+  GRIMOIRE_API_KEY          API key
 `);
-    }
-    else {
-        console.log(`
-grimoire — Documentation RAG System (admin)
+  } else {
+    console.log(`
+grimoire \u2014 Documentation RAG System (admin)
 
 Commands:
   add <name> --url <url>           Add a new documentation source
@@ -674,103 +250,96 @@ Commands:
   apikey list                      List API keys
   apikey delete <name>             Delete an API key
 `);
-    }
+  }
 }
 async function main() {
-    const command = process.argv[2];
-    const isConsumer = await detectConsumerMode();
-    if (command === "--version" || command === "-v") {
-        const pkg = JSON.parse(readFileSync(join(PROJECT_ROOT, "package.json"), "utf-8"));
-        console.log(pkg.version);
-        process.exit(0);
-    }
-    if (!command || command === "--help" || command === "-h") {
-        showHelp(isConsumer);
-        process.exit(0);
-    }
-    if (command === "update") {
-        await cmdUpdate();
-        return;
-    }
-    if (isConsumer) {
-        if (command === "init") {
-            await cmdInit();
-            return;
-        }
-        if (ADMIN_ONLY_COMMANDS.includes(command)) {
-            console.error(`The '${command}' command is only available in admin mode.`);
-            process.exit(1);
-        }
-        const config = await resolveConsumerConfig().catch(() => {
-            console.error("Grimoire is not configured yet. Run 'grimoire init' to set up your API connection.");
-            process.exit(1);
-        });
-        if (command === "search") {
-            const args = parseArgs({
-                args: process.argv.slice(3),
-                options: {
-                    source: { type: "string" },
-                    top: { type: "string" },
-                    compact: { type: "boolean", default: false },
-                },
-                allowPositionals: true,
-            });
-            const query = args.positionals[0];
-            if (!query) {
-                console.error("Usage: grimoire search \"<query>\" [--source <name>] [--top <n>] [--compact]");
-                process.exit(1);
-            }
-            const topN = args.values.top ? parseInt(args.values.top, 10) : undefined;
-            await cmdConsumerSearch(config, query, { source: args.values.source, topN, compact: args.values.compact });
-        }
-        else if (command === "list") {
-            const args = parseArgs({
-                args: process.argv.slice(3),
-                options: { names: { type: "boolean", default: false } },
-                allowPositionals: true,
-            });
-            await cmdConsumerList(config, { names: args.values.names });
-        }
-        else if (command === "stats") {
-            await cmdConsumerStats(config);
-        }
-        else {
-            console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
-            process.exit(1);
-        }
-        return;
-    }
+  const command = process.argv[2];
+  if (command === "--version" || command === "-v") {
+    const pkg = JSON.parse(readFileSync(join2(PROJECT_ROOT, "package.json"), "utf-8"));
+    console.log(pkg.version);
+    process.exit(0);
+  }
+  const isConsumer = await detectConsumerMode();
+  if (!command || command === "--help" || command === "-h") {
+    showHelp(isConsumer);
+    process.exit(0);
+  }
+  if (command === "update") {
+    await cmdUpdate();
+    return;
+  }
+  if (isConsumer) {
     if (command === "init") {
-        await cmdInit();
-        return;
+      await cmdInit();
+      return;
     }
-    if (!existsSync(CONFIG_PATH)) {
-        console.error("Grimoire is not configured. Run 'grimoire init' to set up your API key.");
-        process.exit(1);
+    if (ADMIN_ONLY_COMMANDS.includes(command)) {
+      console.error(`The '${command}' command is only available in admin mode.`);
+      process.exit(1);
     }
-    const handler = ADMIN_COMMANDS[command];
-    if (!handler) {
-        console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
+    const config = await resolveConsumerConfig().catch(() => {
+      console.error("Grimoire is not configured yet. Run 'grimoire init' to set up your API connection.");
+      process.exit(1);
+    });
+    if (command === "search") {
+      const args = parseArgs({
+        args: process.argv.slice(3),
+        options: {
+          source: { type: "string" },
+          top: { type: "string" },
+          compact: { type: "boolean", default: false }
+        },
+        allowPositionals: true
+      });
+      const query = args.positionals[0];
+      if (!query) {
+        console.error('Usage: grimoire search "<query>" [--source <name>] [--top <n>] [--compact]');
         process.exit(1);
-    }
-    await handler();
+      }
+      const topN = args.values.top ? parseInt(args.values.top, 10) : void 0;
+      await cmdConsumerSearch(config, query, { source: args.values.source, topN, compact: args.values.compact });
+    } else if (command === "list") {
+      const args = parseArgs({
+        args: process.argv.slice(3),
+        options: { names: { type: "boolean", default: false } },
+        allowPositionals: true
+      });
+      await cmdConsumerList(config, { names: args.values.names });
+    } else if (command === "stats") {
+      await cmdConsumerStats(config);
+    } else {
+      console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
+      process.exit(1);
+    }
+    return;
+  }
+  if (command === "init") {
+    await cmdInit();
+    return;
+  }
+  const { ADMIN_COMMANDS } = await import("./admin-HA6FNUV4.js");
+  const handler = ADMIN_COMMANDS[command];
+  if (!handler) {
+    console.error(`Unknown command: ${command}. Run "grimoire --help" for usage.`);
+    process.exit(1);
+  }
+  await handler();
 }
-const GCP_AUTH_PATTERNS = [
-    "Unable to detect a Project Id",
-    "Could not load the default credentials",
-    "invalid_grant",
-    "invalid_rapt",
-    "UNAUTHENTICATED",
-    "Getting metadata from plugin failed",
+var GCP_AUTH_PATTERNS = [
+  "Unable to detect a Project Id",
+  "Could not load the default credentials",
+  "invalid_grant",
+  "invalid_rapt",
+  "UNAUTHENTICATED",
+  "Getting metadata from plugin failed"
 ];
 main().catch((err) => {
-    const msg = err.message ?? String(err);
-    if (GCP_AUTH_PATTERNS.some((p) => msg.includes(p))) {
-        console.error("Google Cloud authentication failed. Re-authenticate with:\n\n  gcloud auth application-default login\n");
-    }
-    else {
-        console.error(`Error: ${msg}`);
-    }
-    process.exit(1);
+  const msg = err.message ?? String(err);
+  if (GCP_AUTH_PATTERNS.some((p) => msg.includes(p))) {
+    console.error("Google Cloud authentication failed. Re-authenticate with:\n\n  gcloud auth application-default login\n");
+  } else {
+    console.error(`Error: ${msg}`);
+  }
+  process.exit(1);
 });
-//# sourceMappingURL=cli.js.map
+//# sourceMappingURL=cli.js.map