@astrofoundry/grimoire 3.13.0 → 3.14.0

package/dist/scraper.js

@@ -1,98 +0,0 @@
-import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { chromium } from "playwright";
-export function slugifyUrl(url) {
-    const parsed = new URL(url);
-    return parsed.pathname
-        .replace(/^\//, "")
-        .replace(/\/$/, "")
-        .replace(/\//g, "-")
-        .replace(/[^a-zA-Z0-9-]/g, "");
-}
-export function filterUrls(urls, includePatterns, excludePatterns) {
-    let filtered = urls.filter((url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#"));
-    if (includePatterns && includePatterns.length > 0) {
-        filtered = filtered.filter((url) => includePatterns.some((pattern) => url.includes(pattern)));
-    }
-    if (excludePatterns && excludePatterns.length > 0) {
-        filtered = filtered.filter((url) => !excludePatterns.some((pattern) => url.includes(pattern)));
-    }
-    return [...new Set(filtered)].sort();
-}
-async function fetchSitemapUrls(sitemapUrl) {
-    const response = await fetch(sitemapUrl);
-    const xml = await response.text();
-    const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
-    if (xml.includes("<sitemapindex")) {
-        const nested = await Promise.all(locs.map((loc) => fetchSitemapUrls(loc)));
-        return nested.flat();
-    }
-    return locs;
-}
-async function discoverFromSitemap(sitemapUrl, source) {
-    const urls = await fetchSitemapUrls(sitemapUrl);
-    return filterUrls(urls, source.include_patterns, source.exclude_patterns);
-}
-export async function discoverUrls(page, source) {
-    if (source.sitemap_url) {
-        return discoverFromSitemap(source.sitemap_url, source);
-    }
-    await page.goto(source.start_url, { waitUntil: source.headed ? "networkidle" : "domcontentloaded" });
-    const rawUrls = await page.$$eval(`${source.nav_selector} a[href]`, (links) => links.map((a) => a.href));
-    const discovered = filterUrls(rawUrls, source.include_patterns, source.exclude_patterns);
-    if (!discovered.includes(source.start_url)) {
-        discovered.unshift(source.start_url);
-    }
-    return discovered;
-}
-export async function fetchPage(page, url, headed) {
-    await page.goto(url, { waitUntil: headed ? "networkidle" : "domcontentloaded" });
-    return page.content();
-}
-const DEFAULT_CONCURRENCY = 50;
-async function runPool(items, concurrency, fn) {
-    let nextIndex = 0;
-    async function worker() {
-        while (nextIndex < items.length) {
-            const index = nextIndex++;
-            await fn(items[index], index);
-        }
-    }
-    const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
-    await Promise.all(workers);
-}
-export async function scrapeSource(source, sourceName, dataDir, onProgress) {
-    const rawDir = join(dataDir, "raw", sourceName);
-    await mkdir(rawDir, { recursive: true });
-    const concurrency = source.concurrency ?? DEFAULT_CONCURRENCY;
-    const browser = await chromium.launch({ channel: "chrome", headless: !source.headed });
-    const context = await browser.newContext(source.user_agent ? { userAgent: source.user_agent } : {});
-    const discoveryPage = await context.newPage();
-    const urls = await discoverUrls(discoveryPage, source);
-    await discoveryPage.close();
-    let completed = 0;
-    try {
-        await runPool(urls, concurrency, async (url) => {
-            const page = await context.newPage();
-            try {
-                const html = await fetchPage(page, url, source.headed);
-                const slug = slugifyUrl(url);
-                await writeFile(join(rawDir, `${slug}.html`), html, "utf-8");
-                completed++;
-                onProgress?.(completed, urls.length, url);
-            }
-            finally {
-                await page.close();
-            }
-        });
-        await writeFile(join(rawDir, "urls.json"), JSON.stringify(urls), "utf-8");
-        return urls;
-    }
-    finally {
-        await browser.close();
-    }
-}
-export async function createBrowser() {
-    return chromium.launch({ channel: "chrome" });
-}
-//# sourceMappingURL=scraper.js.map

package/dist/scraper.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAG/D,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,QAAQ;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAc,EACd,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC/E,CAAC;IAEF,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,UAAkB;IAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,UAAkB,EAAE,MAAoB;IACzE,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAU,EACV,MAAoB;IAEpB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,OAAO,mBAAmB,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAErG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,GAAG,MAAM,CAAC,YAAY,UAAU,EAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,CAAC,CAC3D,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACzF,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,GAAW,EACX,MAAgB;IAEhB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACjF,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,KAAK,UAAU,OAAO,CACpB,KAAU,EACV,WAAmB,EACnB,EAA6C;IAE7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAoB,EACpB,UAAkB,EAClB,OAAe,EACf,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEpG,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvD,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC5C,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1E,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AAChD,CAAC"}

package/dist/search.d.ts

@@ -1,8 +0,0 @@
-import type { SearchResult } from "./types.js";
-export type { SearchResult };
-export declare function search(query: string, options?: {
-    source?: string;
-    candidates?: number;
-    topN?: number;
-}): Promise<SearchResult[]>;
-//# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,YAAY,EAAE,YAAY,EAAE,CAAC;AAM7B,wBAAsB,MAAM,CAC1B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GACpE,OAAO,CAAC,YAAY,EAAE,CAAC,CAuCzB"}

package/dist/search.js

@@ -1,43 +0,0 @@
-import { embedText } from "./embedder.js";
-import { vectorSearch } from "./store.js";
-import { rerank } from "./reranker.js";
-function hasReranker() {
-    return !!process.env.RERANKER_URL;
-}
-export async function search(query, options = {}) {
-    const { source, candidates = 20, topN = 5 } = options;
-    const queryEmbedding = await embedText(query);
-    const rawResults = await vectorSearch(queryEmbedding, candidates, source);
-    if (rawResults.length === 0)
-        return [];
-    if (hasReranker()) {
-        const documents = rawResults.map((r) => r.data.content);
-        const reranked = await rerank(query, documents, topN);
-        return reranked.map((r) => {
-            const original = rawResults[r.index];
-            const data = original.data;
-            return {
-                id: original.id,
-                source: data.source,
-                url: data.url,
-                title: data.title,
-                heading_path: data.heading_path,
-                content: data.content,
-                relevance_score: r.relevance_score,
-            };
-        });
-    }
-    return rawResults.slice(0, topN).map((r) => {
-        const data = r.data;
-        return {
-            id: r.id,
-            source: data.source,
-            url: data.url,
-            title: data.title,
-            heading_path: data.heading_path,
-            content: data.content,
-            relevance_score: Math.max(0, 1 - r.distance / 2),
-        };
-    });
-}
-//# sourceMappingURL=search.js.map

package/dist/search.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAKvC,SAAS,WAAW;IAClB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,UAAmE,EAAE;IAErE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAEtD,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAE1E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,WAAW,EAAE,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAiB,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAEtD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACxB,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,OAAO;gBACL,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAgB;gBAC7B,GAAG,EAAE,IAAI,CAAC,GAAa;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAe;gBAC3B,YAAY,EAAE,IAAI,CAAC,YAAwB;gBAC3C,OAAO,EAAE,IAAI,CAAC,OAAiB;gBAC/B,eAAe,EAAE,CAAC,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACzC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;QACpB,OAAO;YACL,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,IAAI,CAAC,MAAgB;YAC7B,GAAG,EAAE,IAAI,CAAC,GAAa;YACvB,KAAK,EAAE,IAAI,CAAC,KAAe;YAC3B,YAAY,EAAE,IAAI,CAAC,YAAwB;YAC3C,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/store.d.ts

@@ -1,15 +0,0 @@
-import type { Chunk, SourceMeta } from "./types.js";
-export declare function storeChunks(chunks: Chunk[], embeddings: number[][], onProgress?: (current: number, total: number) => void): Promise<void>;
-export declare function purgeSource(sourceName: string): Promise<number>;
-export declare function updateSourceMeta(sourceName: string, chunkCount: number, urlCount: number, version?: string): Promise<void>;
-export declare function getSourceMeta(sourceName: string): Promise<SourceMeta | null>;
-export declare function getAllSourcesMeta(): Promise<SourceMeta[]>;
-export declare function deleteSourceMeta(sourceName: string): Promise<void>;
-export declare function deleteChunksByIds(ids: string[], onProgress?: (current: number, total: number) => void): Promise<void>;
-export declare function getSourceChunkIds(sourceName: string): Promise<Set<string>>;
-export declare function vectorSearch(queryEmbedding: number[], limit: number, source?: string): Promise<{
-    id: string;
-    data: Record<string, unknown>;
-    distance: number;
-}[]>;
-//# sourceMappingURL=store.d.ts.map

package/dist/store.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA4CpD,wBAAsB,WAAW,CAC/B,MAAM,EAAE,KAAK,EAAE,EACf,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAwBrE;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAQf;AAED,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAIlF;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAG/D;AAED,wBAAsB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAExE;AAED,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EAAE,EACb,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAIhF;AAED,wBAAsB,YAAY,CAChC,cAAc,EAAE,MAAM,EAAE,EACxB,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC,CAwB5E"}

package/dist/store.js

@@ -1,149 +0,0 @@
-import { initializeApp, applicationDefault, getApps } from "firebase-admin/app";
-import { getFirestore, FieldValue, } from "firebase-admin/firestore";
-const BATCH_SIZE = 500;
-const MAX_RETRIES = 5;
-const BASE_DELAY_MS = 5000;
-async function retryOnQuota(fn) {
-    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
-        try {
-            return await fn();
-        }
-        catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            if ((message.includes("RESOURCE_EXHAUSTED") || message.includes("Quota exceeded")) && attempt < MAX_RETRIES - 1) {
-                const delay = BASE_DELAY_MS * Math.pow(2, attempt);
-                console.log(`  Quota exceeded, retrying in ${delay / 1000}s...`);
-                await new Promise((resolve) => setTimeout(resolve, delay));
-                continue;
-            }
-            throw err;
-        }
-    }
-    throw new Error("Max retries exceeded");
-}
-let db;
-function getDb() {
-    if (!db) {
-        if (getApps().length === 0) {
-            initializeApp({ credential: applicationDefault() });
-        }
-        db = getFirestore();
-    }
-    return db;
-}
-function chunksCol() {
-    return getDb().collection("grimoire_chunks");
-}
-function sourcesCol() {
-    return getDb().collection("grimoire_sources");
-}
-export async function storeChunks(chunks, embeddings, onProgress) {
-    const database = getDb();
-    const col = chunksCol();
-    for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
-        const batch = database.batch();
-        const slice = chunks.slice(i, i + BATCH_SIZE);
-        const embSlice = embeddings.slice(i, i + BATCH_SIZE);
-        for (let j = 0; j < slice.length; j++) {
-            const chunk = slice[j];
-            batch.set(col.doc(chunk.id), {
-                source: chunk.source,
-                url: chunk.url,
-                title: chunk.title,
-                heading_path: chunk.heading_path,
-                content: chunk.content,
-                token_count: chunk.token_count,
-                embedded_at: new Date().toISOString(),
-                embedding: FieldValue.vector(embSlice[j]),
-            });
-        }
-        await retryOnQuota(() => batch.commit());
-        onProgress?.(Math.min(i + BATCH_SIZE, chunks.length), chunks.length);
-    }
-}
-export async function purgeSource(sourceName) {
-    const database = getDb();
-    const col = chunksCol();
-    const snapshot = await col.where("source", "==", sourceName).get();
-    if (snapshot.empty)
-        return 0;
-    let batch = database.batch();
-    let count = 0;
-    for (const doc of snapshot.docs) {
-        batch.delete(doc.ref);
-        count++;
-        if (count % BATCH_SIZE === 0) {
-            await retryOnQuota(() => batch.commit());
-            batch = database.batch();
-        }
-    }
-    if (count % BATCH_SIZE !== 0) {
-        await retryOnQuota(() => batch.commit());
-    }
-    return count;
-}
-export async function updateSourceMeta(sourceName, chunkCount, urlCount, version) {
-    await sourcesCol().doc(sourceName).set({
-        source: sourceName,
-        ...(version ? { version } : {}),
-        last_refreshed: new Date().toISOString(),
-        chunk_count: chunkCount,
-        url_count: urlCount,
-    });
-}
-export async function getSourceMeta(sourceName) {
-    const doc = await sourcesCol().doc(sourceName).get();
-    if (!doc.exists)
-        return null;
-    return doc.data();
-}
-export async function getAllSourcesMeta() {
-    const snapshot = await sourcesCol().get();
-    return snapshot.docs.map((doc) => doc.data());
-}
-export async function deleteSourceMeta(sourceName) {
-    await sourcesCol().doc(sourceName).delete();
-}
-export async function deleteChunksByIds(ids, onProgress) {
-    if (ids.length === 0)
-        return;
-    const database = getDb();
-    const col = chunksCol();
-    for (let i = 0; i < ids.length; i += BATCH_SIZE) {
-        const batch = database.batch();
-        const slice = ids.slice(i, i + BATCH_SIZE);
-        for (const id of slice) {
-            batch.delete(col.doc(id));
-        }
-        await retryOnQuota(() => batch.commit());
-        onProgress?.(Math.min(i + BATCH_SIZE, ids.length), ids.length);
-    }
-}
-export async function getSourceChunkIds(sourceName) {
-    const col = chunksCol();
-    const snapshot = await col.where("source", "==", sourceName).select().get();
-    return new Set(snapshot.docs.map((doc) => doc.id));
-}
-export async function vectorSearch(queryEmbedding, limit, source) {
-    const col = chunksCol();
-    let query = col;
-    if (source) {
-        query = query.where("source", "==", source);
-    }
-    const snapshot = await query
-        .findNearest({
-        vectorField: "embedding",
-        queryVector: FieldValue.vector(queryEmbedding),
-        limit,
-        distanceMeasure: "COSINE",
-        distanceResultField: "_distance",
-    })
-        .get();
-    return snapshot.docs.map((doc) => {
-        const data = doc.data();
-        const distance = data._distance ?? 0;
-        delete data._distance;
-        return { id: doc.id, data, distance };
-    });
-}
-//# sourceMappingURL=store.js.map

package/dist/store.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,YAAY,EACZ,UAAU,GAEX,MAAM,0BAA0B,CAAC;AAGlC,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,KAAK,UAAU,YAAY,CAAI,EAAoB;IACjD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,IAAI,OAAO,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;gBAChH,MAAM,KAAK,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,iCAAiC,KAAK,GAAG,IAAI,MAAM,CAAC,CAAC;gBACjE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3D,SAAS;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;AAC1C,CAAC;AAED,IAAI,EAAyB,CAAC;AAE9B,SAAS,KAAK;IACZ,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,IAAI,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,aAAa,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,EAAE,GAAG,YAAY,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAe,EACf,UAAsB,EACtB,UAAqD;IAErD,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC1C,CAAC,CAAC;QACL,CAAC;QAED,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,GAAG,EAAE,CAAC;IACnE,IAAI,QAAQ,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IAE7B,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,KAAK,EAAE,CAAC;QACR,IAAI,KAAK,GAAG,UAAU,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,UAAU,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAkB,EAClB,UAAkB,EAClB,QAAgB,EAChB,OAAgB;IAEhB,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;QACrC,MAAM,EAAE,UAAU;QAClB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxC,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,UAAkB;IACpD,MAAM,GAAG,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAC;IACrD,IAAI,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC7B,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,EAAE,CAAC;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAgB,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAAkB;IACvD,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAa,EACb,UAAqD;IAErD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC7B,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC3C,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;YACvB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,YAAY,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,UAAkB;IACxD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;IAC5E,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,cAAwB,EACxB,KAAa,EACb,MAAe;IAEf,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,IAAI,KAAK,GAAG,GAA8B,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,KAAK;SACzB,WAAW,CAAC;QACX,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC;QAC9C,KAAK;QACL,eAAe,EAAE,QAAQ;QACzB,mBAAmB,EAAE,WAAW;KACjC,CAAC;SACD,GAAG,EAAE,CAAC;IAET,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAI,IAAI,CAAC,SAAoB,IAAI,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC,SAAS,CAAC;QACtB,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/types.d.ts

@@ -1,26 +0,0 @@
-export interface Chunk {
-    id: string;
-    source: string;
-    url: string;
-    title: string;
-    heading_path: string[];
-    content: string;
-    token_count: number;
-}
-export interface SearchResult {
-    id: string;
-    source: string;
-    url: string;
-    title: string;
-    heading_path: string[];
-    content: string;
-    relevance_score: number;
-}
-export interface SourceMeta {
-    source: string;
-    version?: string;
-    last_refreshed: string;
-    chunk_count: number;
-    url_count: number;
-}
-//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB"}

package/dist/types.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}