@larkup/vector-stores 0.1.14

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@larkup/vector-stores",
+  "version": "0.1.14",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "exports": {
+    "./adapter": "./src/adapters/base.ts",
+    "./factory": "./src/factory.ts",
+    "./registry": "./src/registry.ts",
+    "./lancedb": "./src/adapters/lancedb.ts",
+    "./pinecone": "./src/adapters/pinecone.ts",
+    "./chroma": "./src/adapters/chroma.ts"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.30.0",
+    "@pinecone-database/pinecone": "^7.2.0"
+  },
+  "peerDependencies": {
+    "@larkup/core": "workspace:*"
+  },
+  "devDependencies": {
+    "chromadb": "^1.10.5",
+    "typescript": "5.7.3"
+  }
+}

package/src/adapters/base.ts

@@ -0,0 +1,48 @@
+/**
+ * Common contract every vector-store adapter implements.
+ *
+ * The indexer talks to this interface only, so swapping LanceDB ⇆ Pinecone is
+ * purely a config change. The same shape is what the Phase 4 generated server
+ * will rely on, keeping local + deployed retrieval behaviour identical.
+ */
+
+export interface VectorRecord {
+  id: string
+  vector: number[]
+  text: string
+  title: string
+  url?: string
+  source: string
+  documentId: string
+  chunkIndex: number
+  metadata?: Record<string, any>
+}
+
+export interface QueryHit {
+  id: string
+  score: number
+  text: string
+  title: string
+  url?: string
+  documentId: string
+  metadata?: Record<string, any>
+}
+
+export interface VectorStoreAdapter {
+  /** Create / connect to the index, sized for `dimensions`. */
+  init(dimensions: number): Promise<void>
+  /** Remove all existing vectors (full re-index). */
+  reset(): Promise<void>
+  /** Upsert a batch of records. */
+  upsert(records: VectorRecord[]): Promise<void>
+  /** Number of vectors currently stored, if cheaply knowable. */
+  count(): Promise<number | null>
+  /**
+   * Nearest-neighbour search for a query vector.
+   * Pass `queryText` for hybrid/lexical stores so they can also generate a
+   * sparse vector and merge dense + sparse results (RRF).
+   */
+  query(vector: number[], topK: number, queryText?: string): Promise<QueryHit[]>
+  /** Test connection to the vector store. Throws if invalid. */
+  testConnection(dimensions: number): Promise<void>
+}

package/src/adapters/chroma.ts

@@ -0,0 +1,402 @@
+import type { QueryHit, VectorRecord, VectorStoreAdapter } from "./base";
+
+/**
+ * Chroma adapter — self-hosted (client-server) or Chroma Cloud.
+ *
+ * ── Server (self-hosted) ──────────────────────────────────────────────────
+ *   Connects to a running Chroma instance via HTTP (default localhost:8000).
+ *   Optionally uses a static auth token for secured servers.
+ *
+ * ── Cloud (Chroma Cloud) ──────────────────────────────────────────────────
+ *   Connects to Chroma Cloud with API key, tenant, and database.
+ *
+ * ── Features ──────────────────────────────────────────────────────────────
+ *   - Collections: getOrCreateCollection with cosine distance
+ *   - Metadata filtering: stores text, title, url, source, documentId, chunkIndex
+ *   - Full-text search: where_document $contains for hybrid mode
+ *   - Distance → score: score = 1 / (1 + distance) for cosine distance
+ *
+ * The `chromadb` package is imported dynamically so it only fails if the user
+ * selects Chroma without installing the optional dependency.
+ */
+
+interface ChromaConfig {
+  mode?: string;
+  /** Server URL — used when mode is "server" */
+  host?: string;
+  /** Static auth token for server mode */
+  authToken?: string;
+  /** Cloud API key */
+  apiKey?: string;
+  /** Cloud tenant */
+  tenant?: string;
+  /** Cloud database */
+  database?: string;
+  /** Collection name */
+  collectionName?: string;
+  /** Index type for determining search behavior */
+  indexType?: string;
+}
+
+/** RRF constant for hybrid merge */
+const RRF_K = 60;
+
+export class ChromaAdapter implements VectorStoreAdapter {
+  private client: any = null;
+  private collection: any = null;
+  private readonly collectionName: string;
+
+  constructor(private readonly config: ChromaConfig) {
+    this.collectionName = config.collectionName?.trim() || "documents";
+  }
+
+  /**
+   * Lazily connect to the Chroma server or cloud.
+   * The `chromadb` package is dynamically imported to avoid module-eval
+   * failures when the package isn't installed.
+   */
+  private async getClient(): Promise<any> {
+    if (this.client) return this.client;
+
+    let chromadb: any;
+    try {
+      chromadb = await import("chromadb");
+    } catch {
+      throw new Error(
+        'The "chromadb" package is not installed. Please install it first via the Configure page or run: pnpm add chromadb --filter @larkup/vector-stores',
+      );
+    }
+
+    if (this.config.mode === "cloud") {
+      if (!this.config.apiKey) {
+        throw new Error("Chroma Cloud requires an API key.");
+      }
+      const CloudClientClass = chromadb.CloudClient ?? chromadb.default?.CloudClient;
+      if (!CloudClientClass) {
+        throw new Error("Could not find CloudClient in the chromadb package.");
+      }
+      this.client = new CloudClientClass({
+        apiKey: this.config.apiKey,
+        tenant: this.config.tenant || "default_tenant",
+        database: this.config.database || "default_database",
+      });
+    } else {
+      // Server mode
+      const host = this.config.host?.trim() || "http://localhost:8000";
+      const ClientClass = chromadb.ChromaClient ?? chromadb.default?.ChromaClient;
+      if (!ClientClass) {
+        throw new Error("Could not find ChromaClient in the chromadb package.");
+      }
+      const opts: any = { path: host };
+      if (this.config.authToken) {
+        opts.auth = {
+          provider: "token",
+          credentials: this.config.authToken,
+        };
+      }
+      this.client = new ClientClass(opts);
+    }
+
+    return this.client;
+  }
+
+  private async getCollection(): Promise<any> {
+    if (this.collection) return this.collection;
+    const client = await this.getClient();
+    this.collection = await client.getOrCreateCollection({
+      name: this.collectionName,
+      metadata: { "hnsw:space": "cosine" },
+    });
+    return this.collection;
+  }
+
+  // ── Lifecycle ──────────────────────────────────────────────────────────────
+
+  async init(_dimensions: number): Promise<void> {
+    await this.getCollection();
+  }
+
+  async reset(): Promise<void> {
+    const client = await this.getClient();
+    try {
+      await client.deleteCollection({ name: this.collectionName });
+    } catch {
+      /* collection may not exist yet — that's fine */
+    }
+    this.collection = null;
+  }
+
+  // ── Upsert ─────────────────────────────────────────────────────────────────
+
+  async upsert(records: VectorRecord[]): Promise<void> {
+    if (records.length === 0) return;
+    const collection = await this.getCollection();
+
+    // Chroma's upsert accepts parallel arrays
+    const ids: string[] = [];
+    const embeddings: number[][] = [];
+    const documents: string[] = [];
+    const metadatas: Record<string, any>[] = [];
+
+    for (const r of records) {
+      ids.push(r.id);
+      embeddings.push(r.vector);
+      documents.push(r.text);
+      metadatas.push(this.buildMetadata(r));
+    }
+
+    // Chroma has a max batch size of ~5000 for upsert, process in chunks
+    const BATCH = 4096;
+    for (let i = 0; i < ids.length; i += BATCH) {
+      await collection.upsert({
+        ids: ids.slice(i, i + BATCH),
+        embeddings: embeddings.slice(i, i + BATCH),
+        documents: documents.slice(i, i + BATCH),
+        metadatas: metadatas.slice(i, i + BATCH),
+      });
+    }
+  }
+
+  private buildMetadata(r: VectorRecord): Record<string, any> {
+    const meta: Record<string, any> = {
+      title: r.title,
+      source: r.source,
+      documentId: r.documentId,
+      chunkIndex: r.chunkIndex,
+    };
+    if (r.url) meta.url = r.url;
+    // Flatten user metadata (strings/numbers/booleans only — Chroma requirement)
+    if (r.metadata) {
+      for (const [k, v] of Object.entries(r.metadata)) {
+        if (v === null || v === undefined) continue;
+        if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
+          meta[k] = v;
+        } else {
+          meta[k] = JSON.stringify(v);
+        }
+      }
+    }
+    return meta;
+  }
+
+  // ── Count ──────────────────────────────────────────────────────────────────
+
+  async count(): Promise<number | null> {
+    try {
+      const collection = await this.getCollection();
+      return await collection.count();
+    } catch {
+      return null;
+    }
+  }
+
+  // ── Query ──────────────────────────────────────────────────────────────────
+
+  async query(
+    vector: number[],
+    topK: number,
+    queryText?: string,
+  ): Promise<QueryHit[]> {
+    const collection = await this.getCollection();
+    const isSemantic = this.config.indexType === "semantic";
+    const isLexical = this.config.indexType === "lexical";
+
+    // Semantic only
+    if (isSemantic || !queryText?.trim()) {
+      return this.denseQuery(collection, vector, topK);
+    }
+
+    // Lexical only (using Chroma whereDocument full-text substring)
+    if (isLexical) {
+      return this.documentQuery(collection, queryText, topK);
+    }
+
+    // Hybrid search
+    const denseHits = await this.denseQuery(collection, vector, topK);
+
+    try {
+      const textHits = await this.documentQuery(collection, queryText, topK);
+      if (textHits.length > 0) {
+        return rrfMerge(denseHits, textHits, topK);
+      }
+    } catch {
+      // Full-text search fallback
+    }
+
+    return denseHits;
+  }
+
+  private async denseQuery(
+    collection: any,
+    vector: number[],
+    topK: number,
+  ): Promise<QueryHit[]> {
+    const result = await collection.query({
+      queryEmbeddings: [vector],
+      nResults: topK,
+      include: ["documents", "metadatas", "distances"],
+    });
+
+    return this.parseQueryResult(result);
+  }
+
+  private async documentQuery(
+    collection: any,
+    queryText: string,
+    topK: number,
+  ): Promise<QueryHit[]> {
+    // Chroma supports substring filtering natively via get() with whereDocument
+    const result = await collection.get({
+      whereDocument: { "$contains": queryText },
+      limit: topK,
+      include: ["documents", "metadatas"],
+    });
+
+    return this.parseGetResult(result);
+  }
+
+  private parseGetResult(result: any): QueryHit[] {
+    if (!result.ids || result.ids.length === 0) return [];
+
+    const ids = result.ids as string[];
+    const documents = result.documents ?? [];
+    const metadatas = result.metadatas ?? [];
+
+    return ids.map((id, i) => {
+      const meta = (metadatas[i] ?? {}) as Record<string, any>;
+      const score = 1.0; // Exact/lexical matches get max base score before RRF
+
+      const {
+        title,
+        url,
+        source,
+        documentId,
+        chunkIndex,
+        ...customMetadata
+      } = meta;
+
+      return {
+        id,
+        score,
+        text: (documents[i] as string) ?? "",
+        title: (title as string) ?? "Untitled",
+        url: (url as string) || undefined,
+        documentId: (documentId as string) ?? "",
+        metadata:
+          Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
+      };
+    });
+  }
+
+  private parseQueryResult(result: any): QueryHit[] {
+    if (!result.ids?.[0]) return [];
+
+    const ids = result.ids[0] as string[];
+    const documents = result.documents?.[0] ?? [];
+    const metadatas = result.metadatas?.[0] ?? [];
+    const distances = result.distances?.[0] ?? [];
+
+    return ids.map((id, i) => {
+      const meta = (metadatas[i] ?? {}) as Record<string, any>;
+      const distance = distances[i] ?? 0;
+      // Chroma cosine distance: 0 = identical, 2 = opposite
+      // Convert to a similarity score
+      const score = 1 / (1 + distance);
+
+      const {
+        title,
+        url,
+        source,
+        documentId,
+        chunkIndex,
+        ...customMetadata
+      } = meta;
+
+      return {
+        id,
+        score,
+        text: (documents[i] as string) ?? "",
+        title: (title as string) ?? "Untitled",
+        url: (url as string) || undefined,
+        documentId: (documentId as string) ?? "",
+        metadata:
+          Object.keys(customMetadata).length > 0 ? customMetadata : undefined,
+      };
+    });
+  }
+
+  // ── Connection test ────────────────────────────────────────────────────────
+
+  async testConnection(_dimensions: number): Promise<void> {
+    let client: any;
+    try {
+      client = await this.getClient();
+    } catch (err: any) {
+      throw new Error(`Failed to connect to Chroma: ${err.message}`);
+    }
+
+    // heartbeat() is the lightest-weight RPC to verify connectivity
+    try {
+      await client.heartbeat();
+    } catch (err: any) {
+      const msg: string = err.message ?? "";
+
+      if (
+        msg.includes("401") ||
+        msg.includes("Unauthorized") ||
+        msg.includes("403") ||
+        msg.includes("Forbidden")
+      ) {
+        throw new Error(
+          "Invalid Chroma credentials. Please check your auth token or API key.",
+        );
+      }
+
+      if (msg.includes("ECONNREFUSED") || msg.includes("fetch failed")) {
+        throw new Error(
+          "Could not reach the Chroma server. Is it running and accessible at the configured URL?",
+        );
+      }
+
+      throw new Error(`Failed to reach Chroma server: ${msg}`);
+    }
+
+    // Verify we can access the specified tenant/database by listing collections
+    try {
+      await client.listCollections();
+    } catch (err: any) {
+      throw new Error(
+        `Connected to Chroma but failed to access the database or tenant. Please verify your tenant and database names. Error: ${err.message}`,
+      );
+    }
+  }
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function rrfMerge(
+  denseHits: QueryHit[],
+  sparseHits: QueryHit[],
+  topK: number,
+): QueryHit[] {
+  const scores = new Map<string, { hit: QueryHit; score: number }>();
+
+  const addList = (hits: QueryHit[]) =>
+    hits.forEach((hit, rank) => {
+      const contrib = 1 / (RRF_K + rank + 1);
+      const prev = scores.get(hit.id);
+      if (prev) {
+        prev.score += contrib;
+      } else {
+        scores.set(hit.id, { hit, score: contrib });
+      }
+    });
+
+  addList(denseHits);
+  addList(sparseHits);
+
+  return [...scores.values()]
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topK)
+    .map(({ hit, score }) => ({ ...hit, score }));
+}

package/src/adapters/lancedb.ts

@@ -0,0 +1,220 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import * as lancedb from "@lancedb/lancedb";
+import type { QueryHit, VectorRecord, VectorStoreAdapter } from "./base";
+
+/**
+ * LanceDB adapter — embedded/on-disk for local dev, or LanceDB Cloud.
+ *
+ * The table schema is inferred from the first batch of rows we add, so `init`
+ * is a no-op connect; the table is (re)created lazily on `reset`/first upsert.
+ */
+
+interface LanceConfig {
+  mode?: string;
+  dbPath?: string;
+  uri?: string;
+  apiKey?: string;
+  tableName?: string;
+}
+
+interface LanceRow extends Record<string, unknown> {
+  id: string;
+  vector: number[];
+  text: string;
+  title: string;
+  url: string;
+  source: string;
+  documentId: string;
+  chunkIndex: number;
+  metadata?: string;
+}
+
+export class LanceDBAdapter implements VectorStoreAdapter {
+  private conn: lancedb.Connection | null = null;
+  private table: lancedb.Table | null = null;
+  private readonly tableName: string;
+
+  constructor(private readonly config: LanceConfig) {
+    this.tableName = config.tableName?.trim() || "documents";
+  }
+
+  private async connect(): Promise<lancedb.Connection> {
+    if (this.conn) return this.conn;
+    if (this.config.mode === "cloud") {
+      if (!this.config.uri || !this.config.apiKey) {
+        throw new Error("LanceDB Cloud requires both a URI and an API key.");
+      }
+      if (!this.config.uri.startsWith("db://")) {
+        throw new Error(
+          "LanceDB Cloud URI must start with 'db://' (e.g. db://my-database).",
+        );
+      }
+      this.conn = await lancedb.connect(this.config.uri, {
+        apiKey: this.config.apiKey,
+      });
+    } else {
+      const dir = this.config.dbPath?.trim() || "./.larkup/lancedb";
+      let abs = path.isAbsolute(dir) ? dir : path.join(process.cwd(), dir);
+
+      try {
+        await fs.mkdir(abs, { recursive: true });
+      } catch (err: any) {
+        if (path.isAbsolute(dir)) {
+          // The user likely forgot the dot (e.g. typed "/lancedb" instead of "./lancedb").
+          // Automatically retry by prepending "." to make it relative to the workspace.
+          const fallbackDir = "." + dir;
+          abs = path.join(process.cwd(), fallbackDir);
+          try {
+            await fs.mkdir(abs, { recursive: true });
+          } catch (fallbackErr: any) {
+            throw new Error(
+              `Could not create directory for LanceDB at "${abs}": ${fallbackErr.message}`,
+            );
+          }
+        } else {
+          throw new Error(
+            `Could not create directory for LanceDB at "${abs}": ${err.message}`,
+          );
+        }
+      }
+
+      this.conn = await lancedb.connect(abs);
+    }
+    return this.conn;
+  }
+
+  async init(): Promise<void> {
+    const conn = await this.connect();
+    const names = await conn.tableNames();
+    if (names.includes(this.tableName)) {
+      this.table = await conn.openTable(this.tableName);
+    }
+  }
+
+  async reset(): Promise<void> {
+    const conn = await this.connect();
+    const names = await conn.tableNames();
+    if (names.includes(this.tableName)) {
+      await conn.dropTable(this.tableName);
+    }
+    this.table = null;
+  }
+
+  private toRow(r: VectorRecord): LanceRow {
+    return {
+      id: r.id,
+      vector: r.vector,
+      text: r.text,
+      title: r.title,
+      url: r.url ?? "",
+      source: r.source,
+      documentId: r.documentId,
+      chunkIndex: r.chunkIndex,
+      metadata: r.metadata ? JSON.stringify(r.metadata) : undefined,
+    };
+  }
+
+  async upsert(records: VectorRecord[]): Promise<void> {
+    if (records.length === 0) return;
+    const conn = await this.connect();
+    const rows = records.map((r) => this.toRow(r));
+
+    if (!this.table) {
+      const names = await conn.tableNames();
+      if (names.includes(this.tableName)) {
+        this.table = await conn.openTable(this.tableName);
+        await this.table.add(rows);
+      } else {
+        // First batch defines the schema (incl. vector dimensions).
+        this.table = await conn.createTable(this.tableName, rows);
+      }
+    } else {
+      await this.table.add(rows);
+    }
+  }
+
+  async count(): Promise<number | null> {
+    if (!this.table) {
+      await this.init();
+      if (!this.table) return 0;
+    }
+    try {
+      return await this.table.countRows();
+    } catch {
+      return null;
+    }
+  }
+
+  async query(vector: number[], topK: number): Promise<QueryHit[]> {
+    if (!this.table) {
+      await this.init();
+      if (!this.table) return [];
+    }
+    const rows = (await this.table
+      .search(vector)
+      .limit(topK)
+      .toArray()) as Array<LanceRow & { _distance?: number }>;
+
+    return rows.map((row) => ({
+      id: row.id as string,
+      // Convert L2 distance → a 0..1 similarity-ish score for display.
+      score: typeof row._distance === "number" ? 1 / (1 + row._distance) : 0,
+      text: row.text as string,
+      title: row.title as string,
+      url: (row.url as string) || undefined,
+      documentId: row.documentId as string,
+      metadata:
+        typeof row.metadata === "string" ? JSON.parse(row.metadata) : undefined,
+    }));
+  }
+
+  async testConnection(dimensions: number): Promise<void> {
+    // ── Local mode: just verify the directory is reachable ──────────────────
+    if (this.config.mode !== "cloud") {
+      try {
+        await this.connect();
+        return;
+      } catch (err: any) {
+        throw new Error(`Failed to connect to local LanceDB: ${err.message}`);
+      }
+    }
+
+    // ── Cloud mode ───────────────────────────────────────────────────────────
+    // connect() is lazy (never contacts the server), so we call tableNames()
+    // to force a real authenticated HTTP round-trip.
+    let conn: lancedb.Connection;
+    try {
+      conn = await this.connect();
+    } catch (err: any) {
+      throw new Error(`Failed to connect to LanceDB Cloud: ${err.message}`);
+    }
+
+    try {
+      await conn.tableNames();
+    } catch (err: any) {
+      const msg: string = err.message ?? "";
+
+      // The SDK puts the HTTP status INSIDE the message string (not as a
+      // property). Pattern: "Http error: ... 401 Unauthorized: Unauthorized"
+      if (
+        msg.includes("401") ||
+        msg.includes("Unauthorized") ||
+        msg.includes("403") ||
+        msg.includes("Forbidden")
+      ) {
+        throw new Error(
+          "Invalid LanceDB Cloud API key. Please check your credentials in the LanceDB Cloud dashboard.",
+        );
+      }
+
+      if (msg.includes("404") || msg.includes("Not Found")) {
+        throw new Error(
+          `LanceDB Cloud database not found. Please verify your URI (e.g. "db://my-database").`,
+        );
+      }
+
+      throw new Error(`Failed to reach LanceDB Cloud: ${msg}`);
+    }
+  }
+}