@astrofoundry/grimoire 1.2.2

package/dist/store.js

@@ -0,0 +1,102 @@
+import { initializeApp, applicationDefault, getApps } from "firebase-admin/app";
+import { getFirestore, FieldValue, } from "firebase-admin/firestore";
+const BATCH_SIZE = 500;
+let db;
+function getDb() {
+    if (!db) {
+        if (getApps().length === 0) {
+            initializeApp({ credential: applicationDefault() });
+        }
+        db = getFirestore();
+    }
+    return db;
+}
+function chunksCol() {
+    return getDb().collection("grimoire_chunks");
+}
+function sourcesCol() {
+    return getDb().collection("grimoire_sources");
+}
+export async function storeChunks(chunks, embeddings, onProgress) {
+    const database = getDb();
+    const col = chunksCol();
+    for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
+        const batch = database.batch();
+        const slice = chunks.slice(i, i + BATCH_SIZE);
+        const embSlice = embeddings.slice(i, i + BATCH_SIZE);
+        for (let j = 0; j < slice.length; j++) {
+            const chunk = slice[j];
+            batch.set(col.doc(chunk.id), {
+                source: chunk.source,
+                url: chunk.url,
+                title: chunk.title,
+                heading_path: chunk.heading_path,
+                content: chunk.content,
+                token_count: chunk.token_count,
+                embedded_at: new Date().toISOString(),
+                embedding: FieldValue.vector(embSlice[j]),
+            });
+        }
+        await batch.commit();
+        onProgress?.(Math.min(i + BATCH_SIZE, chunks.length), chunks.length);
+    }
+}
+export async function purgeSource(sourceName) {
+    const database = getDb();
+    const col = chunksCol();
+    const snapshot = await col.where("source", "==", sourceName).get();
+    if (snapshot.empty)
+        return 0;
+    const batches = [];
+    let batch = database.batch();
+    let count = 0;
+    for (const doc of snapshot.docs) {
+        batch.delete(doc.ref);
+        count++;
+        if (count % BATCH_SIZE === 0) {
+            batches.push(batch.commit());
+            batch = database.batch();
+        }
+    }
+    if (count % BATCH_SIZE !== 0) {
+        batches.push(batch.commit());
+    }
+    await Promise.all(batches);
+    return count;
+}
+export async function updateSourceMeta(sourceName, chunkCount, urlCount) {
+    await sourcesCol().doc(sourceName).set({
+        source: sourceName,
+        last_refreshed: new Date().toISOString(),
+        chunk_count: chunkCount,
+        url_count: urlCount,
+    });
+}
+export async function getSourceMeta(sourceName) {
+    const doc = await sourcesCol().doc(sourceName).get();
+    if (!doc.exists)
+        return null;
+    return doc.data();
+}
+export async function getAllSourcesMeta() {
+    const snapshot = await sourcesCol().get();
+    return snapshot.docs.map((doc) => doc.data());
+}
+export async function vectorSearch(queryEmbedding, limit, source) {
+    const col = chunksCol();
+    let query = col;
+    if (source) {
+        query = query.where("source", "==", source);
+    }
+    const snapshot = await query
+        .findNearest("embedding", FieldValue.vector(queryEmbedding), {
+        limit,
+        distanceMeasure: "COSINE",
+    })
+        .get();
+    return snapshot.docs.map((doc) => ({
+        id: doc.id,
+        data: doc.data(),
+    }));
+}
+//# sourceMappingURL=store.js.map

package/dist/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,YAAY,EACZ,UAAU,GAEX,MAAM,0BAA0B,CAAC;AAGlC,MAAM,UAAU,GAAG,GAAG,CAAC;AAEvB,IAAI,EAAyB,CAAC;AAE9B,SAAS,KAAK;IACZ,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,IAAI,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,aAAa,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,EAAE,GAAG,YAAY,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAe,EACf,UAAsB,EACtB,UAAqD;IAErD,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE;gBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC1C,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QACrB,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,GAAG,EAAE,CAAC;IACnE,IAAI,QAAQ,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IAE7B,MAAM,OAAO,GAA+C,EAAE,CAAC;IAC/D,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,KAAK,EAAE,CAAC;QACR,IAAI,KAAK,GAAG,UAAU,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7B,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAG,UAAU,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAkB,EAClB,UAAkB,EAClB,QAAgB;IAEhB,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;QACrC,MAAM,EAAE,UAAU;QAClB,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACxC,WAAW,EAAE,UAAU;QACvB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,UAAkB;IACpD,MAAM,GAAG,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,CAAC;IACrD,IAAI,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC7B,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,EAAE,CAAC;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAgB,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,cAAwB,EACxB,KAAa,EACb,MAAe;IAEf,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IAExB,IAAI,KAAK,GAAG,GAA8B,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,KAAK;SACzB,WAAW,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE;QAC3D,KAAK;QACL,eAAe,EAAE,QAAQ;KAC1B,CAAC;SACD,GAAG,EAAE,CAAC;IAET,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE;KACjB,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,25 @@
+export interface Chunk {
+    id: string;
+    source: string;
+    url: string;
+    title: string;
+    heading_path: string[];
+    content: string;
+    token_count: number;
+}
+export interface SearchResult {
+    id: string;
+    source: string;
+    url: string;
+    title: string;
+    heading_path: string[];
+    content: string;
+    relevance_score: number;
+}
+export interface SourceMeta {
+    source: string;
+    last_refreshed: string;
+    chunk_count: number;
+    url_count: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@astrofoundry/grimoire",
+  "version": "1.2.2",
+  "description": "Documentation RAG System",
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "module",
+  "bin": {
+    "grimoire": "dist/cli.js"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/jsdom": "^28.0.1",
+    "@types/node": "^25.5.0",
+    "@types/turndown": "^5.0.6",
+    "eslint": "^10.1.0",
+    "globals": "^17.4.0",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.57.2",
+    "vitest": "^4.1.1"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "optionalDependencies": {
+    "@google/generative-ai": "^0.24.1",
+    "cohere-ai": "^7.20.0",
+    "firebase-admin": "^13.7.0",
+    "jsdom": "^29.0.1",
+    "playwright": "^1.58.2",
+    "turndown": "^7.2.2",
+    "yaml": "^2.8.3"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "check": "tsc --noEmit && eslint . && vitest run",
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "release:patch": "v=$(pnpm version patch --no-git-tag-version) && git add -A && git commit -m \"Release $v\" && git tag $v && git push && git push origin $v",
+    "release:minor": "v=$(pnpm version minor --no-git-tag-version) && git add -A && git commit -m \"Release $v\" && git tag $v && git push && git push origin $v",
+    "release:major": "v=$(pnpm version major --no-git-tag-version) && git add -A && git commit -m \"Release $v\" && git tag $v && git push && git push origin $v"
+  }
+}