@deepagents/retrieval 0.1.0

package/dist/lib/ingest.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/lib/ingest.ts"],
+  "sourcesContent": ["import {\n  MarkdownTextSplitter,\n  RecursiveCharacterTextSplitter,\n} from 'langchain/text_splitter';\n\nimport type { Connector } from './connectors/connector.js';\nimport type { Splitter } from './pipeline.js';\nimport { cid } from './stores/cid.js';\nimport type { Embedder, Store } from './stores/store.js';\n\nexport interface IngestionConfig {\n  connector: Connector;\n  store: Store;\n  splitter?: Splitter;\n  embedder: Embedder;\n}\n\nexport async function ingest(\n  config: IngestionConfig,\n  callback?: (it: string) => void,\n) {\n  const splitter = config.splitter ?? split;\n  const embedder = config.embedder;\n  const corpuses = config.connector.sources();\n\n  for await (const it of corpuses) {\n    callback?.(it.id);\n    const content = await it.content();\n    if (!content.trim()) {\n      // skip empty files\n      continue;\n    }\n    await config.store.index(config.connector.sourceId, {\n      id: it.id,\n      cid: cid(content),\n      metadata: it.metadata,\n      chunker: async function* () {\n        // Embed in small batches to control memory usage\n        const values = await splitter(it.id, content);\n        const batchSize = 40;\n        for (let i = 0; i < values.length; i += batchSize) {\n          const batch = values.slice(i, i + batchSize);\n          const { embeddings } = await embedder(batch);\n          for (let j = 0; j < embeddings.length; j++) {\n            yield {\n              content: batch[j],\n              embedding: embeddings[j],\n            };\n          }\n        }\n      },\n    });\n  }\n}\n\nfunction split(id: string, content: string) {\n  const splitter = new MarkdownTextSplitter();\n  return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n  startLine: number;\n  startColumn: number;\n  endLine: number;\n  endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n  content: string;\n  position: ChunkPosition | null;\n  index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n  return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n  originalContent: string,\n  chunks: string[],\n): Array<ChunkPosition | null> {\n  if (!chunks.length) {\n    return [];\n  }\n\n  const normalizedContent = normalizeNewlines(originalContent);\n  const positions: Array<ChunkPosition | null> = [];\n  let searchOffset = 0;\n\n  for (const chunk of chunks) {\n    const normalizedChunk = normalizeNewlines(chunk);\n    const trimmedChunk = normalizedChunk.trim();\n\n    const seek = (needle: string, fromIndex: number) =>\n      needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n    let matchIndex = seek(normalizedChunk, searchOffset);\n    let matchValue = normalizedChunk;\n\n    if (matchIndex === -1 && trimmedChunk) {\n      matchIndex = seek(trimmedChunk, searchOffset);\n      matchValue = trimmedChunk;\n    }\n\n    if (matchIndex === -1) {\n      matchIndex = seek(normalizedChunk, 0);\n      matchValue = normalizedChunk;\n    }\n\n    if (matchIndex === -1 && trimmedChunk) {\n      matchIndex = seek(trimmedChunk, 0);\n      matchValue = trimmedChunk;\n    }\n\n    if (matchIndex === -1) {\n      positions.push(null);\n      continue;\n    }\n\n    const before = normalizedContent.slice(0, matchIndex);\n    const beforeLines = before.split('\\n');\n    const startLine = beforeLines.length;\n    const startColumn = beforeLines[beforeLines.length - 1].length + 1;\n\n    const lines = matchValue.split('\\n');\n    const endLine = startLine + lines.length - 1;\n    const endColumn =\n      lines.length === 1\n        ? startColumn + lines[0].length\n        : lines[lines.length - 1].length + 1;\n\n    positions.push({ startLine, startColumn, endLine, endColumn });\n    searchOffset = matchIndex + matchValue.length;\n  }\n\n  return positions;\n}\n\nfunction buildChunksWithPositions(\n  originalContent: string,\n  chunks: string[],\n): SplitChunkWithPosition[] {\n  const positions = computePositions(originalContent, chunks);\n  return chunks.map((content, index) => ({\n    content,\n    index,\n    position: positions[index] ?? null,\n  }));\n}\n\nexport async function splitTypeScriptWithPositions(\n  id: string,\n  content: string,\n): Promise<SplitChunkWithPosition[]> {\n  const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n    chunkSize: 512,\n    chunkOverlap: 100,\n  });\n  const docs = await splitter.createDocuments([content]);\n  const chunks = docs.map((d) => d.pageContent);\n  return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n  const chunks = await splitTypeScriptWithPositions(id, content);\n  return chunks.map((chunk) => chunk.content);\n}\n"],
+  "mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AAIP,SAAS,WAAW;AAUpB,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAE1C,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM,MAAM,OAAO,UAAU,UAAU;AAAA,MAClD,IAAI,GAAG;AAAA,MACP,KAAK,IAAI,OAAO;AAAA,MAChB,UAAU,GAAG;AAAA,MACb,SAAS,mBAAmB;AAE1B,cAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,cAAM,YAAY;AAClB,iBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,gBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,gBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,mBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,kBAAM;AAAA,cACJ,SAAS,MAAM,CAAC;AAAA,cAChB,WAAW,WAAW,CAAC;AAAA,YACzB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,YAAY,YAAY,SAAS,CAAC,EAAE,SAAS;AAEjE,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,MAAM,MAAM,SAAS,CAAC,EAAE,SAAS;AAEvC,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;",
+  "names": []
+}

package/dist/lib/pipeline.d.ts

@@ -0,0 +1,2 @@
+export type Splitter = (id: string, content: string) => Promise<string[]>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/lib/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/pipeline.ts"],"names":[],"mappings":"AAUA,MAAM,MAAM,QAAQ,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC"}

package/dist/lib/pipeline.js

@@ -0,0 +1 @@
+//# sourceMappingURL=pipeline.js.map

package/dist/lib/pipeline.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/sidecar.d.ts

@@ -0,0 +1 @@
+//# sourceMappingURL=sidecar.d.ts.map

package/dist/lib/sidecar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sidecar.d.ts","sourceRoot":"","sources":["../../src/lib/sidecar.ts"],"names":[],"mappings":""}

package/dist/lib/sidecar.js

@@ -0,0 +1 @@
+//# sourceMappingURL=sidecar.js.map

package/dist/lib/sidecar.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/similiarty-search.d.ts

@@ -0,0 +1,3 @@
+import { type IngestionConfig } from './ingest.js';
+export declare function similaritySearch(query: string, config: Omit<IngestionConfig, 'splitter'>): Promise<any[]>;
+//# sourceMappingURL=similiarty-search.d.ts.map

package/dist/lib/similiarty-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC,kBAiD1C"}

package/dist/lib/similiarty-search.js

@@ -0,0 +1,43 @@
+import { embedMany } from "ai";
+import { ingest } from "./ingest.js";
+async function similaritySearch(query, config) {
+  const mode = config.connector.ingestWhen ?? "contentChanged";
+  let shouldIngest = true;
+  if (mode === "never") {
+    if (await config.store.sourceExists(config.connector.sourceId)) {
+      console.log(
+        `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`
+      );
+      shouldIngest = false;
+    }
+  } else if (mode === "expired") {
+    const sourceExists = await config.store.sourceExists(
+      config.connector.sourceId
+    );
+    if (sourceExists && !await config.store.sourceExpired(config.connector.sourceId)) {
+      console.log(
+        `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`
+      );
+      shouldIngest = false;
+    }
+  }
+  const expiryDate = config.connector.expiresAfter ? new Date(Date.now() + config.connector.expiresAfter) : void 0;
+  if (shouldIngest) {
+    await ingest(config);
+  }
+  return config.store.search(
+    query,
+    { sourceId: config.connector.sourceId, topN: 50 },
+    config.embedder
+  ).then(
+    (results) => results.map((it) => ({
+      ...it,
+      similarity: 1 - it.distance,
+      distance: it.distance
+    }))
+  );
+}
+export {
+  similaritySearch
+};
+//# sourceMappingURL=similiarty-search.js.map

package/dist/lib/similiarty-search.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/lib/similiarty-search.ts"],
+  "sourcesContent": ["import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.js';\n\nexport async function similaritySearch(\n  query: string,\n  config: Omit<IngestionConfig, 'splitter'>,\n) {\n  // Ingest if needed, then perform vector similarity search via the configured store\n  const mode = config.connector.ingestWhen ?? 'contentChanged';\n  let shouldIngest = true;\n\n  if (mode === 'never') {\n    if (await config.store.sourceExists(config.connector.sourceId)) {\n      console.log(\n        `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n      );\n      shouldIngest = false;\n    }\n  } else if (mode === 'expired') {\n    const sourceExists = await config.store.sourceExists(\n      config.connector.sourceId,\n    );\n    if (\n      sourceExists &&\n      !(await config.store.sourceExpired(config.connector.sourceId))\n    ) {\n      console.log(\n        `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n      );\n      shouldIngest = false;\n    }\n  }\n\n  // Calculate expiry date if connector specifies expiresAfter\n  const expiryDate = config.connector.expiresAfter\n    ? new Date(Date.now() + config.connector.expiresAfter)\n    : undefined;\n  if (shouldIngest) {\n    await ingest(config);\n  }\n  return config.store\n    .search(\n      query,\n      { sourceId: config.connector.sourceId, topN: 50 },\n      config.embedder,\n    )\n    .then(\n      (results) =>\n        results.map((it) => ({\n          ...it,\n          similarity: 1 - it.distance,\n          distance: it.distance,\n        })) as any[],\n    );\n}\n"],
+  "mappings": "AAAA,SAAS,iBAAiB;AAE1B,SAA+B,cAAc;AAE7C,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAGA,QAAM,aAAa,OAAO,UAAU,eAChC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,UAAU,YAAY,IACnD;AACJ,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IACC,CAAC,YACC,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACN;AACJ;",
+  "names": []
+}

package/dist/lib/stores/cid.d.ts

@@ -0,0 +1,2 @@
+export declare function cid(content: string): string;
+//# sourceMappingURL=cid.d.ts.map

package/dist/lib/stores/cid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cid.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/cid.ts"],"names":[],"mappings":"AAEA,wBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,UAElC"}

package/dist/lib/stores/cid.js

@@ -0,0 +1,8 @@
+import { createHash } from "node:crypto";
+function cid(content) {
+  return createHash("sha256").update(content).digest("hex");
+}
+export {
+  cid
+};
+//# sourceMappingURL=cid.js.map

package/dist/lib/stores/cid.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/stores/cid.ts"],
+  "sourcesContent": ["import { createHash } from 'node:crypto';\n\nexport function cid(content: string) {\n  return createHash('sha256').update(content).digest('hex');\n}\n"],
+  "mappings": "AAAA,SAAS,kBAAkB;AAEpB,SAAS,IAAI,SAAiB;AACnC,SAAO,WAAW,QAAQ,EAAE,OAAO,OAAO,EAAE,OAAO,KAAK;AAC1D;",
+  "names": []
+}

package/dist/lib/stores/sqlite/bun-sqlite.d.ts

@@ -0,0 +1 @@
+//# sourceMappingURL=bun-sqlite.d.ts.map

package/dist/lib/stores/sqlite/bun-sqlite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bun-sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/bun-sqlite.ts"],"names":[],"mappings":""}

package/dist/lib/stores/sqlite/bun-sqlite.js

@@ -0,0 +1 @@
+//# sourceMappingURL=bun-sqlite.js.map

package/dist/lib/stores/sqlite/bun-sqlite.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/stores/sqlite/node-sqlite.d.ts

@@ -0,0 +1,3 @@
+import { SQLiteStore } from './sqlite.js';
+export declare function nodeSQLite(dbName: string, dimension: number): SQLiteStore;
+//# sourceMappingURL=node-sqlite.d.ts.map

package/dist/lib/stores/sqlite/node-sqlite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"node-sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/node-sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAO3D"}

package/dist/lib/stores/sqlite/node-sqlite.js

@@ -0,0 +1,14 @@
+import { DatabaseSync } from "node:sqlite";
+import * as sqliteVec from "sqlite-vec";
+import { SQLiteStore } from "./sqlite.js";
+function nodeSQLite(dbName, dimension) {
+  const db = new DatabaseSync(dbName, {
+    allowExtension: true
+  });
+  db.loadExtension(sqliteVec.getLoadablePath());
+  return new SQLiteStore(db, dimension);
+}
+export {
+  nodeSQLite
+};
+//# sourceMappingURL=node-sqlite.js.map

package/dist/lib/stores/sqlite/node-sqlite.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/lib/stores/sqlite/node-sqlite.ts"],
+  "sourcesContent": ["import { DatabaseSync } from 'node:sqlite';\nimport * as sqliteVec from 'sqlite-vec';\n\nimport { SQLiteStore } from './sqlite.js';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n  const db = new DatabaseSync(dbName, {\n    allowExtension: true,\n  });\n\n  db.loadExtension(sqliteVec.getLoadablePath());\n  return new SQLiteStore(db, dimension);\n}\n"],
+  "mappings": "AAAA,SAAS,oBAAoB;AAC7B,YAAY,eAAe;AAE3B,SAAS,mBAAmB;AAErB,SAAS,WAAW,QAAgB,WAAmB;AAC5D,QAAM,KAAK,IAAI,aAAa,QAAQ;AAAA,IAClC,gBAAgB;AAAA,EAClB,CAAC;AAED,KAAG,cAAc,UAAU,gBAAgB,CAAC;AAC5C,SAAO,IAAI,YAAY,IAAI,SAAS;AACtC;",
+  "names": []
+}

package/dist/lib/stores/sqlite/sqlite.d.ts

@@ -0,0 +1,35 @@
+import type { Chunk, Corpus, Embedder, SearchOptions, Store } from '../store.js';
+interface DB {
+    prepare: (sql: string) => {
+        run: (...args: any[]) => any;
+        all: (...args: any[]) => any[];
+        get: (...args: any[]) => any;
+    };
+    exec: (sql: string) => void;
+}
+export declare class SQLiteStore implements Store {
+    #private;
+    constructor(db: DB, dimension: number);
+    search(query: string, options: SearchOptions, embedder: Embedder): Promise<any[]>;
+    upsertDoc(inputs: {
+        documentId: string;
+        sourceId: string;
+        cid: string;
+        metadata?: Record<string, any>;
+    }): any;
+    insertDoc(inputs: {
+        sourceId: string;
+        documentId: string;
+    }): (chunk: Chunk) => void;
+    delete(inputs: {
+        sourceId: string;
+        documentId: string;
+    }): any;
+    sourceExists(sourceId: string): boolean;
+    sourceExpired(sourceId: string): boolean;
+    setSourceExpiry(sourceId: string, expiryDate: Date): any;
+    index(sourceId: string, corpus: Corpus, expiryDate?: Date): Promise<void>;
+}
+export declare function vectorToBlob(vector: number[] | Float32Array): Buffer;
+export {};
+//# sourceMappingURL=sqlite.d.ts.map

package/dist/lib/stores/sqlite/sqlite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,KAAK,EACN,MAAM,aAAa,CAAC;AAKrB,UAAU,EAAE;IACV,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;QACxB,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;QAC7B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;KAC9B,CAAC;IACF,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;IAwD/B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IA+CtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM;IAW9B,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CA+ChE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,MAAM,CASpE"}

package/dist/lib/stores/sqlite/sqlite.js

@@ -0,0 +1,223 @@
+import dedent from "dedent";
+import { template } from "lodash-es";
+import sql from "./sqlite.sql.js";
+const DEFAULT_TOP_N = 10;
+class SQLiteStore {
+  #db;
+  constructor(db, dimension) {
+    this.#db = db;
+    const compiled = template(sql);
+    this.#db.exec(compiled({ DIMENSION: dimension }));
+  }
+  #transaction(callback) {
+    try {
+      this.#db.exec("BEGIN IMMEDIATE");
+      callback();
+      this.#db.exec("COMMIT");
+    } catch (error) {
+      this.#db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+  #searchByDocument(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				SELECT v.content, v.distance, v.document_id, d.metadata
+				FROM vec_chunks v
+				JOIN documents d ON d.id = v.document_id
+				WHERE v.source_id = ?
+					AND v.document_id = ?
+					AND v.embedding MATCH vec_normalize(vec_f32(?))
+					AND v.k = ?
+				ORDER BY v.distance ASC
+			`);
+    return stmt.all(
+      inputs.sourceId,
+      inputs.documentId,
+      inputs.embedding,
+      inputs.k
+    );
+  }
+  #searchBySource(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				SELECT v.content, v.distance, v.document_id, d.metadata
+				FROM vec_chunks v
+				JOIN documents d ON d.id = v.document_id
+				WHERE v.source_id = ?
+					AND v.embedding MATCH vec_normalize(vec_f32(?))
+					AND v.k = ?
+				ORDER BY v.distance ASC
+			`);
+    return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);
+  }
+  async search(query, options, embedder) {
+    const { embeddings } = await embedder([query]);
+    if (!embeddings.length) {
+      return [];
+    }
+    const vectorBlob = vectorToBlob(embeddings[0]);
+    const topN = options.topN;
+    if (options.documentId) {
+      const rows2 = this.#searchByDocument({
+        sourceId: options.sourceId,
+        documentId: options.documentId,
+        embedding: vectorBlob,
+        k: topN ?? DEFAULT_TOP_N
+      });
+      return rows2.map((r) => ({
+        ...r,
+        metadata: safeParseMetadata(r.metadata)
+      }));
+    }
+    const rows = this.#searchBySource({
+      sourceId: options.sourceId,
+      embedding: vectorBlob,
+      k: topN ?? DEFAULT_TOP_N
+    });
+    return rows.map((r) => ({
+      ...r,
+      metadata: safeParseMetadata(r.metadata)
+    }));
+  }
+  #upsertSource(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				INSERT INTO sources (source_id) VALUES (?)
+				ON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')
+			`);
+    return stmt.run(inputs.sourceId);
+  }
+  #upsertSourceWithExpiry(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				INSERT INTO sources (source_id, expires_at) VALUES (?, ?)
+				ON CONFLICT(source_id) DO UPDATE SET
+					updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now',
+					expires_at=excluded.expires_at
+			`);
+    return stmt.run(inputs.sourceId, inputs.expiresAt);
+  }
+  upsertDoc(inputs) {
+    const stmt = this.#db.prepare(dedent`
+        INSERT INTO documents (id, source_id, cid, metadata)
+        VALUES (?, ?, ?, json(?))
+        ON CONFLICT(id) DO UPDATE SET
+          cid=excluded.cid,
+          metadata=COALESCE(excluded.metadata, documents.metadata),
+          updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')
+        WHERE documents.cid != excluded.cid;
+      `);
+    return stmt.run(
+      inputs.documentId,
+      inputs.sourceId,
+      inputs.cid,
+      inputs.metadata ? JSON.stringify(inputs.metadata) : null
+    );
+  }
+  insertDoc(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				INSERT INTO vec_chunks (source_id, document_id, content, embedding)
+				VALUES (?, ?, ?, vec_normalize(vec_f32(?)))
+			`);
+    return (chunk) => {
+      stmt.run(
+        inputs.sourceId,
+        inputs.documentId,
+        chunk.content,
+        vectorToBlob(chunk.embedding)
+      );
+    };
+  }
+  delete(inputs) {
+    const stmt = this.#db.prepare(dedent`
+				DELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?
+			`);
+    return stmt.run(inputs.sourceId, inputs.documentId);
+  }
+  sourceExists(sourceId) {
+    const stmt = this.#db.prepare(dedent`
+				SELECT 1 FROM sources WHERE source_id = ? LIMIT 1
+			`);
+    const row = stmt.get(sourceId);
+    return Boolean(row);
+  }
+  sourceExpired(sourceId) {
+    const stmt = this.#db.prepare(dedent`
+				SELECT 1 FROM sources
+				WHERE source_id = ?
+					AND expires_at IS NOT NULL
+					AND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
+				LIMIT 1
+			`);
+    const row = stmt.run(sourceId);
+    return Boolean(row);
+  }
+  setSourceExpiry(sourceId, expiryDate) {
+    const stmt = this.#db.prepare(dedent`
+				UPDATE sources
+				SET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
+				WHERE source_id = ?
+			`);
+    return stmt.run(expiryDate.toISOString(), sourceId);
+  }
+  async index(sourceId, corpus, expiryDate) {
+    if (expiryDate) {
+      this.#upsertSourceWithExpiry({
+        expiresAt: expiryDate.toISOString(),
+        sourceId
+      });
+    } else {
+      this.#upsertSource({ sourceId });
+    }
+    const info = this.upsertDoc({
+      documentId: corpus.id,
+      sourceId,
+      cid: corpus.cid,
+      metadata: corpus.metadata
+    });
+    const changed = info.changes > 0;
+    if (!changed) {
+      return;
+    }
+    const insert = this.insertDoc({ sourceId, documentId: corpus.id });
+    this.#transaction(() => {
+      this.delete({ sourceId, documentId: corpus.id });
+    });
+    const batchSize = 32;
+    let batch = [];
+    const flush = () => {
+      if (!batch.length) return;
+      this.#transaction(() => {
+        for (let i = 0; i < batch.length; i++) {
+          insert(batch[i]);
+        }
+      });
+      batch = [];
+    };
+    for await (const chunk of corpus.chunker()) {
+      batch.push(chunk);
+      if (batch.length >= batchSize) flush();
+    }
+    flush();
+  }
+}
+function vectorToBlob(vector) {
+  if (vector instanceof Float32Array) {
+    const copied = new Float32Array(vector.length);
+    copied.set(vector);
+    return Buffer.from(copied.buffer);
+  }
+  const floatArray = new Float32Array(vector);
+  return Buffer.from(floatArray.buffer);
+}
+function safeParseMetadata(value) {
+  if (value == null) return null;
+  if (typeof value === "object") return value;
+  try {
+    return JSON.parse(String(value));
+  } catch {
+    return null;
+  }
+}
+export {
+  SQLiteStore,
+  vectorToBlob
+};
+//# sourceMappingURL=sqlite.js.map

package/dist/lib/stores/sqlite/sqlite.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/lib/stores/sqlite/sqlite.ts"],
+  "sourcesContent": ["import dedent from 'dedent';\nimport { template } from 'lodash-es';\n\nimport type {\n  Chunk,\n  Corpus,\n  Embedder,\n  SearchOptions,\n  Store,\n} from '../store.js';\nimport sql from './sqlite.sql.js';\n\nconst DEFAULT_TOP_N = 10;\n\ninterface DB {\n  prepare: (sql: string) => {\n    run: (...args: any[]) => any;\n    all: (...args: any[]) => any[];\n    get: (...args: any[]) => any;\n  };\n  exec: (sql: string) => void;\n}\n\nexport class SQLiteStore implements Store {\n  readonly #db: DB;\n  constructor(db: DB, dimension: number) {\n    this.#db = db;\n    const compiled = template(sql);\n    this.#db.exec(compiled({ DIMENSION: dimension }));\n  }\n\n  #transaction(callback: () => void) {\n    try {\n      this.#db.exec('BEGIN IMMEDIATE');\n      callback();\n      this.#db.exec('COMMIT');\n    } catch (error) {\n      this.#db.exec('ROLLBACK');\n      throw error;\n    }\n  }\n\n  #searchByDocument(inputs: {\n    sourceId: string;\n    documentId: string;\n    embedding: Buffer<ArrayBufferLike>;\n    k: number;\n  }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n    return stmt.all(\n      inputs.sourceId,\n      inputs.documentId,\n      inputs.embedding,\n      inputs.k,\n    );\n  }\n  #searchBySource(inputs: {\n    sourceId: string;\n    embedding: Buffer<ArrayBufferLike>;\n    k: number;\n  }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n    return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n  }\n  async search(query: string, options: SearchOptions, embedder: Embedder) {\n    const { embeddings } = await embedder([query]);\n    if (!embeddings.length) {\n      return [];\n    }\n    const vectorBlob = vectorToBlob(embeddings[0]);\n    const topN = options.topN;\n\n    if (options.documentId) {\n      const rows = this.#searchByDocument({\n        sourceId: options.sourceId,\n        documentId: options.documentId,\n        embedding: vectorBlob,\n        k: topN ?? DEFAULT_TOP_N,\n      });\n      return rows.map((r: any) => ({\n        ...r,\n        metadata: safeParseMetadata(r.metadata),\n      }));\n    }\n\n    const rows = this.#searchBySource({\n      sourceId: options.sourceId,\n      embedding: vectorBlob,\n      k: topN ?? DEFAULT_TOP_N,\n    });\n    return rows.map((r: any) => ({\n      ...r,\n      metadata: safeParseMetadata(r.metadata),\n    }));\n  }\n  #upsertSource(inputs: { sourceId: string }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n    return stmt.run(inputs.sourceId);\n  }\n  #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now',\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n    return stmt.run(inputs.sourceId, inputs.expiresAt);\n  }\n  upsertDoc(inputs: {\n    documentId: string;\n    sourceId: string;\n    cid: string;\n    metadata?: Record<string, any>;\n  }) {\n    const stmt = this.#db.prepare(dedent`\n        INSERT INTO documents (id, source_id, cid, metadata)\n        VALUES (?, ?, ?, json(?))\n        ON CONFLICT(id) DO UPDATE SET\n          cid=excluded.cid,\n          metadata=COALESCE(excluded.metadata, documents.metadata),\n          updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n        WHERE documents.cid != excluded.cid;\n      `);\n    return stmt.run(\n      inputs.documentId,\n      inputs.sourceId,\n      inputs.cid,\n      inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n    );\n  }\n  insertDoc(inputs: { sourceId: string; documentId: string }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n    return (chunk: Chunk) => {\n      stmt.run(\n        inputs.sourceId,\n        inputs.documentId,\n        chunk.content,\n        vectorToBlob(chunk.embedding),\n      );\n    };\n  }\n  delete(inputs: { sourceId: string; documentId: string }) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n    return stmt.run(inputs.sourceId, inputs.documentId);\n  }\n  sourceExists(sourceId: string) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n    const row = stmt.get(sourceId);\n    return Boolean(row);\n  }\n  sourceExpired(sourceId: string) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tLIMIT 1\n\t\t\t`);\n    const row = stmt.run(sourceId);\n    return Boolean(row);\n  }\n  setSourceExpiry(sourceId: string, expiryDate: Date) {\n    const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n    return stmt.run(expiryDate.toISOString(), sourceId);\n  }\n  async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n    if (expiryDate) {\n      this.#upsertSourceWithExpiry({\n        expiresAt: expiryDate.toISOString(),\n        sourceId,\n      });\n    } else {\n      this.#upsertSource({ sourceId });\n    }\n\n    const info = this.upsertDoc({\n      documentId: corpus.id,\n      sourceId,\n      cid: corpus.cid,\n      metadata: corpus.metadata,\n    });\n    const changed = info.changes > 0;\n\n    if (!changed) {\n      return;\n    }\n\n    const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n    // Delete previous rows once before inserting\n    this.#transaction(() => {\n      this.delete({ sourceId, documentId: corpus.id });\n    });\n\n    const batchSize = 32;\n    let batch: Chunk[] = [];\n    const flush = () => {\n      if (!batch.length) return;\n      this.#transaction(() => {\n        for (let i = 0; i < batch.length; i++) {\n          insert(batch[i]);\n        }\n      });\n      batch = [];\n    };\n\n    for await (const chunk of corpus.chunker()) {\n      batch.push(chunk);\n      if (batch.length >= batchSize) flush();\n    }\n    // flush any remaining\n    flush();\n  }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n  if (vector instanceof Float32Array) {\n    // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n    const copied = new Float32Array(vector.length);\n    copied.set(vector);\n    return Buffer.from(copied.buffer);\n  }\n  const floatArray = new Float32Array(vector);\n  return Buffer.from(floatArray.buffer);\n}\n\nfunction safeParseMetadata(value: any) {\n  if (value == null) return null;\n  if (typeof value === 'object') return value;\n  try {\n    return JSON.parse(String(value));\n  } catch {\n    return null;\n  }\n}\n"],
+  "mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,gBAAgB;AASzB,OAAO,SAAS;AAEhB,MAAM,gBAAgB;AAWf,MAAM,YAA6B;AAAA,EAC/B;AAAA,EACT,YAAY,IAAQ,WAAmB;AACrC,SAAK,MAAM;AACX,UAAM,WAAW,SAAS,GAAG;AAC7B,SAAK,IAAI,KAAK,SAAS,EAAE,WAAW,UAAU,CAAC,CAAC;AAAA,EAClD;AAAA,EAEA,aAAa,UAAsB;AACjC,QAAI;AACF,WAAK,IAAI,KAAK,iBAAiB;AAC/B,eAAS;AACT,WAAK,IAAI,KAAK,QAAQ;AAAA,IACxB,SAAS,OAAO;AACd,WAAK,IAAI,KAAK,UAAU;AACxB,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,CAAC,OAAY;AAAA,QAC3B,GAAG;AAAA,QACH,UAAU,kBAAkB,EAAE,QAAQ;AAAA,MACxC,EAAE;AAAA,IACJ;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,CAAC,OAAY;AAAA,MAC3B,GAAG;AAAA,MACH,UAAU,kBAAkB,EAAE,QAAQ;AAAA,IACxC,EAAE;AAAA,EACJ;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB;AAC9B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,WAAO,KAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EACpD;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE,SAAK,aAAa,MAAM;AACtB,WAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,IACjD,CAAC;AAED,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB,WAAK,aAAa,MAAM;AACtB,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,iBAAO,MAAM,CAAC,CAAC;AAAA,QACjB;AAAA,MACF,CAAC;AACD,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,kBAAkB,OAAY;AACrC,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI;AACF,WAAO,KAAK,MAAM,OAAO,KAAK,CAAC;AAAA,EACjC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;",
+  "names": ["rows"]
+}

package/dist/lib/stores/sqlite/sqlite.sql.d.ts

@@ -0,0 +1,3 @@
+declare const _default: "-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n  source_id TEXT PARTITION KEY,\n  document_id TEXT,\n  embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n  +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n  source_id TEXT PRIMARY KEY,\n  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n  expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n  id TEXT PRIMARY KEY,\n  source_id TEXT NOT NULL,\n  cid TEXT NOT NULL,\n  metadata TEXT, -- JSON blob with arbitrary document metadata\n  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n  FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n  DELETE FROM vec_chunks\n  WHERE source_id = OLD.source_id\n    AND document_id = OLD.id;\nEND;\n";
+export default _default;
+//# sourceMappingURL=sqlite.sql.d.ts.map

package/dist/lib/stores/sqlite/sqlite.sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlite.sql.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],"names":[],"mappings":";AAAA,wBAiDC"}

package/dist/lib/stores/sqlite/sqlite.sql.js

@@ -0,0 +1,54 @@
+var sqlite_sql_default = `-- Embedding store schema
+-- Use <%= DIMENSION %> placeholder replaced at runtime.
+
+PRAGMA page_size = 32768;
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA temp_store = MEMORY;
+PRAGMA foreign_keys = ON;
+PRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads
+PRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls
+PRAGMA wal_autocheckpoint = 1000;
+PRAGMA optimize;
+
+CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
+  source_id TEXT PARTITION KEY,
+  document_id TEXT,
+  embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,
+  +content TEXT -- auxiliary payload (not filterable) for reconstruction
+);
+
+CREATE TABLE IF NOT EXISTS sources (
+  source_id TEXT PRIMARY KEY,
+  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+  expires_at TEXT
+) STRICT;
+
+CREATE TABLE IF NOT EXISTS documents (
+  id TEXT PRIMARY KEY,
+  source_id TEXT NOT NULL,
+  cid TEXT NOT NULL,
+  metadata TEXT, -- JSON blob with arbitrary document metadata
+  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+  FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE
+) STRICT;
+
+-- Indexes to accelerate lookups / maintenance
+CREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);
+CREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);
+CREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);
+CREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;
+
+-- emulate cascade for the virtual table
+CREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec
+AFTER DELETE ON documents
+BEGIN
+  DELETE FROM vec_chunks
+  WHERE source_id = OLD.source_id
+    AND document_id = OLD.id;
+END;
+`;
+export {
+  sqlite_sql_default as default
+};
+//# sourceMappingURL=sqlite.sql.js.map

package/dist/lib/stores/sqlite/sqlite.sql.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],
+  "sourcesContent": ["export default `-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n  source_id TEXT PARTITION KEY,\n  document_id TEXT,\n  embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n  +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n  source_id TEXT PRIMARY KEY,\n  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n  expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n  id TEXT PRIMARY KEY,\n  source_id TEXT NOT NULL,\n  cid TEXT NOT NULL,\n  metadata TEXT, -- JSON blob with arbitrary document metadata\n  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n  FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n  DELETE FROM vec_chunks\n  WHERE source_id = OLD.source_id\n    AND document_id = OLD.id;\nEND;\n`"],
+  "mappings": "AAAA,IAAO,qBAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;",
+  "names": []
+}

package/dist/lib/stores/store.d.ts

@@ -0,0 +1,28 @@
+import type { Embedding } from 'ai';
+export type Chunk = {
+    content: string;
+    embedding: Embedding | Float32Array;
+};
+export type Embedder = (documents: string[]) => Promise<{
+    embeddings: (Embedding | Float32Array)[];
+    dimensions: number;
+}>;
+export interface SearchOptions {
+    sourceId: string;
+    documentId?: string;
+    topN?: number;
+}
+export type Corpus = {
+    id: string;
+    cid: string;
+    chunker: () => AsyncGenerator<Chunk>;
+    metadata?: Record<string, any> | undefined;
+};
+export interface Store {
+    search: (query: string, options: SearchOptions, embedder: Embedder) => Promise<any[]>;
+    sourceExists: (sourceId: string) => Promise<boolean> | boolean;
+    sourceExpired: (sourceId: string) => Promise<boolean> | boolean;
+    setSourceExpiry: (sourceId: string, expiryDate: Date) => Promise<void> | void;
+    index: (sourceId: string, corpus: Corpus, expiryDate?: Date) => Promise<void>;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/lib/stores/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,MAAM,KAAK,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,GAAG,YAAY,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;IACtD,UAAU,EAAE,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,QAAQ,KACf,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpB,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAChE,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC9E,KAAK,EAAE,CACL,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,IAAI,KACd,OAAO,CAAC,IAAI,CAAC,CAAC;CACpB"}

package/dist/lib/stores/store.js

@@ -0,0 +1 @@
+//# sourceMappingURL=store.js.map

package/dist/lib/stores/store.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}