@larkup/vector-stores 0.1.20 → 0.1.21

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @larkup/vector-stores
+
+## 0.1.21
+
+### Patch Changes
+
+- 5caaf2f: feat: support persistent S3-compatible LanceDB storage for serverless deployments

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/vector-stores",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "publishConfig": {
     "access": "public"
   },
@@ -18,7 +18,7 @@
     "@pinecone-database/pinecone": "^7.2.0"
   },
   "peerDependencies": {
-    "@larkup/core": "0.1.20"
+    "@larkup/core": ">=0.1.20 <1.0.0"
   },
   "devDependencies": {
     "chromadb": "^1.10.5",

package/src/adapters/base.ts

@@ -7,42 +7,44 @@
  */
 
 export interface VectorRecord {
-  id: string
-  vector: number[]
-  text: string
-  title: string
-  url?: string
-  source: string
-  documentId: string
-  chunkIndex: number
-  metadata?: Record<string, any>
+  id: string;
+  vector: number[];
+  text: string;
+  title: string;
+  url?: string;
+  source: string;
+  documentId: string;
+  chunkIndex: number;
+  metadata?: Record<string, any>;
 }
 
 export interface QueryHit {
-  id: string
-  score: number
-  text: string
-  title: string
-  url?: string
-  documentId: string
-  metadata?: Record<string, any>
+  id: string;
+  score: number;
+  text: string;
+  title: string;
+  url?: string;
+  documentId: string;
+  metadata?: Record<string, any>;
 }
 
 export interface VectorStoreAdapter {
   /** Create / connect to the index, sized for `dimensions`. */
-  init(dimensions: number): Promise<void>
+  init(dimensions: number): Promise<void>;
   /** Remove all existing vectors (full re-index). */
-  reset(): Promise<void>
+  reset(): Promise<void>;
   /** Upsert a batch of records. */
-  upsert(records: VectorRecord[]): Promise<void>
+  upsert(records: VectorRecord[]): Promise<void>;
   /** Number of vectors currently stored, if cheaply knowable. */
-  count(): Promise<number | null>
+  count(): Promise<number | null>;
   /**
    * Nearest-neighbour search for a query vector.
    * Pass `queryText` for hybrid/lexical stores so they can also generate a
    * sparse vector and merge dense + sparse results (RRF).
    */
-  query(vector: number[], topK: number, queryText?: string): Promise<QueryHit[]>
+  query(vector: number[], topK: number, queryText?: string): Promise<QueryHit[]>;
   /** Test connection to the vector store. Throws if invalid. */
-  testConnection(dimensions: number): Promise<void>
+  testConnection(dimensions: number): Promise<void>;
+  /** Delete all chunks associated with the given document IDs. */
+  deleteByDocumentIds(documentIds: string[]): Promise<void>;
 }

package/src/adapters/chroma.ts

@@ -1,4 +1,4 @@
-import type { QueryHit, VectorRecord, VectorStoreAdapter } from "./base";
+import type { QueryHit, VectorRecord, VectorStoreAdapter } from './base';
 
 /**
  * Chroma adapter — self-hosted (client-server) or Chroma Cloud.
@@ -47,7 +47,7 @@ export class ChromaAdapter implements VectorStoreAdapter {
   private readonly collectionName: string;
 
   constructor(private readonly config: ChromaConfig) {
-    this.collectionName = config.collectionName?.trim() || "documents";
+    this.collectionName = config.collectionName?.trim() || 'documents';
   }
 
   /**
@@ -60,37 +60,37 @@ export class ChromaAdapter implements VectorStoreAdapter {
 
     let chromadb: any;
     try {
-      chromadb = await import("chromadb");
+      chromadb = await import('chromadb');
     } catch {
       throw new Error(
         'The "chromadb" package is not installed. Please install it first via the Configure page or run: pnpm add chromadb --filter @larkup/vector-stores',
       );
     }
 
-    if (this.config.mode === "cloud") {
+    if (this.config.mode === 'cloud') {
       if (!this.config.apiKey) {
-        throw new Error("Chroma Cloud requires an API key.");
+        throw new Error('Chroma Cloud requires an API key.');
       }
       const CloudClientClass = chromadb.CloudClient ?? chromadb.default?.CloudClient;
       if (!CloudClientClass) {
-        throw new Error("Could not find CloudClient in the chromadb package.");
+        throw new Error('Could not find CloudClient in the chromadb package.');
       }
       this.client = new CloudClientClass({
         apiKey: this.config.apiKey,
-        tenant: this.config.tenant || "default_tenant",
-        database: this.config.database || "default_database",
+        tenant: this.config.tenant || 'default_tenant',
+        database: this.config.database || 'default_database',
       });
     } else {
       // Server mode
-      const host = this.config.host?.trim() || "http://localhost:8000";
+      const host = this.config.host?.trim() || 'http://localhost:8000';
       const ClientClass = chromadb.ChromaClient ?? chromadb.default?.ChromaClient;
       if (!ClientClass) {
-        throw new Error("Could not find ChromaClient in the chromadb package.");
+        throw new Error('Could not find ChromaClient in the chromadb package.');
       }
       const opts: any = { path: host };
       if (this.config.authToken) {
         opts.auth = {
-          provider: "token",
+          provider: 'token',
           credentials: this.config.authToken,
         };
       }
@@ -105,7 +105,7 @@ export class ChromaAdapter implements VectorStoreAdapter {
     const client = await this.getClient();
     this.collection = await client.getOrCreateCollection({
       name: this.collectionName,
-      metadata: { "hnsw:space": "cosine" },
+      metadata: { 'hnsw:space': 'cosine' },
     });
     return this.collection;
   }
@@ -126,6 +126,16 @@ export class ChromaAdapter implements VectorStoreAdapter {
     this.collection = null;
   }
 
+  async deleteByDocumentIds(documentIds: string[]): Promise<void> {
+    if (documentIds.length === 0) return;
+    try {
+      const collection = await this.getCollection();
+      await collection.delete({ where: { documentId: { $in: documentIds } } });
+    } catch (err) {
+      console.error('Failed to delete vectors by documentIds in Chroma:', err);
+    }
+  }
+
   // ── Upsert ─────────────────────────────────────────────────────────────────
 
   async upsert(records: VectorRecord[]): Promise<void> {
@@ -145,7 +155,6 @@ export class ChromaAdapter implements VectorStoreAdapter {
       metadatas.push(this.buildMetadata(r));
     }
 
-    // Chroma has a max batch size of ~5000 for upsert, process in chunks
     const BATCH = 4096;
     for (let i = 0; i < ids.length; i += BATCH) {
       await collection.upsert({
@@ -169,7 +178,7 @@ export class ChromaAdapter implements VectorStoreAdapter {
     if (r.metadata) {
       for (const [k, v] of Object.entries(r.metadata)) {
         if (v === null || v === undefined) continue;
-        if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
+        if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') {
           meta[k] = v;
         } else {
           meta[k] = JSON.stringify(v);
@@ -192,14 +201,10 @@ export class ChromaAdapter implements VectorStoreAdapter {
 
   // ── Query ──────────────────────────────────────────────────────────────────
 
-  async query(
-    vector: number[],
-    topK: number,
-    queryText?: string,
-  ): Promise<QueryHit[]> {
+  async query(vector: number[], topK: number, queryText?: string): Promise<QueryHit[]> {
     const collection = await this.getCollection();
-    const isSemantic = this.config.indexType === "semantic";
-    const isLexical = this.config.indexType === "lexical";
+    const isSemantic = this.config.indexType === 'semantic';
+    const isLexical = this.config.indexType === 'lexical';
 
     // Semantic only
     if (isSemantic || !queryText?.trim()) {
@@ -226,15 +231,11 @@ export class ChromaAdapter implements VectorStoreAdapter {
     return denseHits;
   }
 
-  private async denseQuery(
-    collection: any,
-    vector: number[],
-    topK: number,
-  ): Promise<QueryHit[]> {
+  private async denseQuery(collection: any, vector: number[], topK: number): Promise<QueryHit[]> {
     const result = await collection.query({
       queryEmbeddings: [vector],
       nResults: topK,
-      include: ["documents", "metadatas", "distances"],
+      include: ['documents', 'metadatas', 'distances'],
     });
 
     return this.parseQueryResult(result);
@@ -245,11 +246,10 @@ export class ChromaAdapter implements VectorStoreAdapter {
     queryText: string,
     topK: number,
   ): Promise<QueryHit[]> {
-    // Chroma supports substring filtering natively via get() with whereDocument
     const result = await collection.get({
-      whereDocument: { "$contains": queryText },
+      whereDocument: { $contains: queryText },
       limit: topK,
-      include: ["documents", "metadatas"],
+      include: ['documents', 'metadatas'],
     });
 
     return this.parseGetResult(result);
@@ -266,24 +266,16 @@ export class ChromaAdapter implements VectorStoreAdapter {
       const meta = (metadatas[i] ?? {}) as Record<string, any>;
       const score = 1.0; // Exact/lexical matches get max base score before RRF
 
-      const {
-        title,
-        url,
-        source,
-        documentId,
-        chunkIndex,
-        ...customMetadata
-      } = meta;
+      const { title, url, source, documentId, chunkIndex, ...customMetadata } = meta;
 
       return {
         id,
         score,
-        text: (documents[i] as string) ?? "",
-        title: (title as string) ?? "Untitled",
+        text: (documents[i] as string) ?? '',
+        title: (title as string) ?? 'Untitled',
         url: (url as string) || undefined,
-        documentId: (documentId as string) ?? "",
-        metadata:
-          Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
+        documentId: (documentId as string) ?? '',
+        metadata: Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
       };
     });
   }
@@ -299,28 +291,19 @@ export class ChromaAdapter implements VectorStoreAdapter {
     return ids.map((id, i) => {
       const meta = (metadatas[i] ?? {}) as Record<string, any>;
       const distance = distances[i] ?? 0;
-      // Chroma cosine distance: 0 = identical, 2 = opposite
       // Convert to a similarity score
       const score = 1 / (1 + distance);
 
-      const {
-        title,
-        url,
-        source,
-        documentId,
-        chunkIndex,
-        ...customMetadata
-      } = meta;
+      const { title, url, source, documentId, chunkIndex, ...customMetadata } = meta;
 
       return {
         id,
         score,
-        text: (documents[i] as string) ?? "",
-        title: (title as string) ?? "Untitled",
+        text: (documents[i] as string) ?? '',
+        title: (title as string) ?? 'Untitled',
         url: (url as string) || undefined,
-        documentId: (documentId as string) ?? "",
-        metadata:
-          Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
+        documentId: (documentId as string) ?? '',
+        metadata: Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
       };
     });
   }
@@ -339,29 +322,26 @@ export class ChromaAdapter implements VectorStoreAdapter {
     try {
       await client.heartbeat();
     } catch (err: any) {
-      const msg: string = err.message ?? "";
+      const msg: string = err.message ?? '';
 
       if (
-        msg.includes("401") ||
-        msg.includes("Unauthorized") ||
-        msg.includes("403") ||
-        msg.includes("Forbidden")
+        msg.includes('401') ||
+        msg.includes('Unauthorized') ||
+        msg.includes('403') ||
+        msg.includes('Forbidden')
       ) {
-        throw new Error(
-          "Invalid Chroma credentials. Please check your auth token or API key.",
-        );
+        throw new Error('Invalid Chroma credentials. Please check your auth token or API key.');
       }
 
-      if (msg.includes("ECONNREFUSED") || msg.includes("fetch failed")) {
+      if (msg.includes('ECONNREFUSED') || msg.includes('fetch failed')) {
         throw new Error(
-          "Could not reach the Chroma server. Is it running and accessible at the configured URL?",
+          'Could not reach the Chroma server. Is it running and accessible at the configured URL?',
         );
       }
 
       throw new Error(`Failed to reach Chroma server: ${msg}`);
     }
 
-    // Verify we can access the specified tenant/database by listing collections
     try {
       await client.listCollections();
     } catch (err: any) {
@@ -374,11 +354,7 @@ export class ChromaAdapter implements VectorStoreAdapter {
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
-function rrfMerge(
-  denseHits: QueryHit[],
-  sparseHits: QueryHit[],
-  topK: number,
-): QueryHit[] {
+function rrfMerge(denseHits: QueryHit[], sparseHits: QueryHit[], topK: number): QueryHit[] {
   const scores = new Map<string, { hit: QueryHit; score: number }>();
 
   const addList = (hits: QueryHit[]) =>

package/src/adapters/lancedb.ts

@@ -1,10 +1,11 @@
-import path from "node:path";
-import fs from "node:fs/promises";
-import * as lancedb from "@lancedb/lancedb";
-import type { QueryHit, VectorRecord, VectorStoreAdapter } from "./base";
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import * as lancedb from '@lancedb/lancedb';
+import type { QueryHit, VectorRecord, VectorStoreAdapter } from './base';
 
 /**
- * LanceDB adapter — embedded/on-disk for local dev, or LanceDB Cloud.
+ * LanceDB adapter — embedded/on-disk for local dev, LanceDB Cloud, or an
+ * S3-compatible object store (including Cloudflare R2).
  *
  * The table schema is inferred from the first batch of rows we add, so `init`
  * is a no-op connect; the table is (re)created lazily on `reset`/first upsert.
@@ -15,6 +16,11 @@ interface LanceConfig {
   dbPath?: string;
   uri?: string;
   apiKey?: string;
+  s3Uri?: string;
+  s3Endpoint?: string;
+  s3Region?: string;
+  s3AccessKeyId?: string;
+  s3SecretAccessKey?: string;
   tableName?: string;
 }
 
@@ -36,34 +42,49 @@ export class LanceDBAdapter implements VectorStoreAdapter {
   private readonly tableName: string;
 
   constructor(private readonly config: LanceConfig) {
-    this.tableName = config.tableName?.trim() || "documents";
+    this.tableName = config.tableName?.trim() || 'documents';
   }
 
   private async connect(): Promise<lancedb.Connection> {
     if (this.conn) return this.conn;
-    if (this.config.mode === "cloud") {
+    if (this.config.mode === 'cloud') {
       if (!this.config.uri || !this.config.apiKey) {
-        throw new Error("LanceDB Cloud requires both a URI and an API key.");
+        throw new Error('LanceDB Cloud requires both a URI and an API key.');
       }
-      if (!this.config.uri.startsWith("db://")) {
-        throw new Error(
-          "LanceDB Cloud URI must start with 'db://' (e.g. db://my-database).",
-        );
+      if (!this.config.uri.startsWith('db://')) {
+        throw new Error("LanceDB Cloud URI must start with 'db://' (e.g. db://my-database).");
       }
       this.conn = await lancedb.connect(this.config.uri, {
         apiKey: this.config.apiKey,
       });
+    } else if (this.config.mode === 's3') {
+      if (!this.config.s3Uri || !this.config.s3AccessKeyId || !this.config.s3SecretAccessKey) {
+        throw new Error(
+          'S3-compatible LanceDB requires a URI, access key ID, and secret access key.',
+        );
+      }
+      if (!this.config.s3Uri.startsWith('s3://')) {
+        throw new Error(
+          "S3-compatible LanceDB URI must start with 's3://' (e.g. s3://my-bucket/lancedb).",
+        );
+      }
+      this.conn = await lancedb.connect(this.config.s3Uri, {
+        storageOptions: {
+          aws_access_key_id: this.config.s3AccessKeyId,
+          aws_secret_access_key: this.config.s3SecretAccessKey,
+          ...(this.config.s3Endpoint ? { aws_endpoint: this.config.s3Endpoint } : {}),
+          ...(this.config.s3Region ? { aws_region: this.config.s3Region } : {}),
+        },
+      });
     } else {
-      const dir = this.config.dbPath?.trim() || "./.larkup/lancedb";
+      const dir = this.config.dbPath?.trim() || './.larkup/lancedb';
       let abs = path.isAbsolute(dir) ? dir : path.join(process.cwd(), dir);
 
       try {
         await fs.mkdir(abs, { recursive: true });
       } catch (err: any) {
         if (path.isAbsolute(dir)) {
-          // The user likely forgot the dot (e.g. typed "/lancedb" instead of "./lancedb").
-          // Automatically retry by prepending "." to make it relative to the workspace.
-          const fallbackDir = "." + dir;
+          const fallbackDir = '.' + dir;
           abs = path.join(process.cwd(), fallbackDir);
           try {
             await fs.mkdir(abs, { recursive: true });
@@ -73,9 +94,7 @@ export class LanceDBAdapter implements VectorStoreAdapter {
             );
           }
         } else {
-          throw new Error(
-            `Could not create directory for LanceDB at "${abs}": ${err.message}`,
-          );
+          throw new Error(`Could not create directory for LanceDB at "${abs}": ${err.message}`);
         }
       }
 
@@ -101,13 +120,23 @@ export class LanceDBAdapter implements VectorStoreAdapter {
     this.table = null;
   }
 
+  async deleteByDocumentIds(documentIds: string[]): Promise<void> {
+    if (documentIds.length === 0) return;
+    if (!this.table) {
+      await this.init();
+      if (!this.table) return;
+    }
+    const idsString = documentIds.map((id) => `'${id}'`).join(', ');
+    await this.table.delete(`documentId IN (${idsString})`);
+  }
+
   private toRow(r: VectorRecord): LanceRow {
     return {
       id: r.id,
       vector: r.vector,
       text: r.text,
       title: r.title,
-      url: r.url ?? "",
+      url: r.url ?? '',
       source: r.source,
       documentId: r.documentId,
       chunkIndex: r.chunkIndex,
@@ -151,37 +180,33 @@ export class LanceDBAdapter implements VectorStoreAdapter {
       await this.init();
       if (!this.table) return [];
     }
-    const rows = (await this.table
-      .search(vector)
-      .limit(topK)
-      .toArray()) as Array<LanceRow & { _distance?: number }>;
+    const rows = (await this.table.search(vector).limit(topK).toArray()) as Array<
+      LanceRow & { _distance?: number }
+    >;
 
     return rows.map((row) => ({
       id: row.id as string,
-      // Convert L2 distance → a 0..1 similarity-ish score for display.
-      score: typeof row._distance === "number" ? 1 / (1 + row._distance) : 0,
+      score: typeof row._distance === 'number' ? 1 / (1 + row._distance) : 0,
       text: row.text as string,
       title: row.title as string,
       url: (row.url as string) || undefined,
       documentId: row.documentId as string,
-      metadata:
-        typeof row.metadata === "string" ? JSON.parse(row.metadata) : undefined,
+      metadata: typeof row.metadata === 'string' ? JSON.parse(row.metadata) : undefined,
     }));
   }
 
   async testConnection(dimensions: number): Promise<void> {
-    // ── Local mode: just verify the directory is reachable ──────────────────
-    if (this.config.mode !== "cloud") {
+    if (this.config.mode !== 'cloud') {
       try {
         await this.connect();
         return;
       } catch (err: any) {
-        throw new Error(`Failed to connect to local LanceDB: ${err.message}`);
+        const label = this.config.mode === 's3' ? 'S3-compatible LanceDB' : 'local LanceDB';
+        throw new Error(`Failed to connect to ${label}: ${err.message}`);
       }
     }
 
     // ── Cloud mode ───────────────────────────────────────────────────────────
-    // connect() is lazy (never contacts the server), so we call tableNames()
     // to force a real authenticated HTTP round-trip.
     let conn: lancedb.Connection;
     try {
@@ -193,22 +218,21 @@ export class LanceDBAdapter implements VectorStoreAdapter {
     try {
       await conn.tableNames();
     } catch (err: any) {
-      const msg: string = err.message ?? "";
+      const msg: string = err.message ?? '';
 
-      // The SDK puts the HTTP status INSIDE the message string (not as a
       // property). Pattern: "Http error: ... 401 Unauthorized: Unauthorized"
       if (
-        msg.includes("401") ||
-        msg.includes("Unauthorized") ||
-        msg.includes("403") ||
-        msg.includes("Forbidden")
+        msg.includes('401') ||
+        msg.includes('Unauthorized') ||
+        msg.includes('403') ||
+        msg.includes('Forbidden')
       ) {
         throw new Error(
-          "Invalid LanceDB Cloud API key. Please check your credentials in the LanceDB Cloud dashboard.",
+          'Invalid LanceDB Cloud API key. Please check your credentials in the LanceDB Cloud dashboard.',
         );
       }
 
-      if (msg.includes("404") || msg.includes("Not Found")) {
+      if (msg.includes('404') || msg.includes('Not Found')) {
         throw new Error(
           `LanceDB Cloud database not found. Please verify your URI (e.g. "db://my-database").`,
         );