@larkup/cli 0.1.14

package/dist/lancedb-E2JLQERV.js

@@ -0,0 +1,163 @@
+import "./chunk-3RG5ZIWI.js";
+
+// ../../packages/vector-stores/src/adapters/lancedb.ts
+import path from "path";
+import fs from "fs/promises";
+import * as lancedb from "@lancedb/lancedb";
+var LanceDBAdapter = class {
+  constructor(config) {
+    this.config = config;
+    this.tableName = config.tableName?.trim() || "documents";
+  }
+  config;
+  conn = null;
+  table = null;
+  tableName;
+  async connect() {
+    if (this.conn) return this.conn;
+    if (this.config.mode === "cloud") {
+      if (!this.config.uri || !this.config.apiKey) {
+        throw new Error("LanceDB Cloud requires both a URI and an API key.");
+      }
+      if (!this.config.uri.startsWith("db://")) {
+        throw new Error(
+          "LanceDB Cloud URI must start with 'db://' (e.g. db://my-database)."
+        );
+      }
+      this.conn = await lancedb.connect(this.config.uri, {
+        apiKey: this.config.apiKey
+      });
+    } else {
+      const dir = this.config.dbPath?.trim() || "./.larkup/lancedb";
+      let abs = path.isAbsolute(dir) ? dir : path.join(process.cwd(), dir);
+      try {
+        await fs.mkdir(abs, { recursive: true });
+      } catch (err) {
+        if (path.isAbsolute(dir)) {
+          const fallbackDir = "." + dir;
+          abs = path.join(process.cwd(), fallbackDir);
+          try {
+            await fs.mkdir(abs, { recursive: true });
+          } catch (fallbackErr) {
+            throw new Error(
+              `Could not create directory for LanceDB at "${abs}": ${fallbackErr.message}`
+            );
+          }
+        } else {
+          throw new Error(
+            `Could not create directory for LanceDB at "${abs}": ${err.message}`
+          );
+        }
+      }
+      this.conn = await lancedb.connect(abs);
+    }
+    return this.conn;
+  }
+  async init() {
+    const conn = await this.connect();
+    const names = await conn.tableNames();
+    if (names.includes(this.tableName)) {
+      this.table = await conn.openTable(this.tableName);
+    }
+  }
+  async reset() {
+    const conn = await this.connect();
+    const names = await conn.tableNames();
+    if (names.includes(this.tableName)) {
+      await conn.dropTable(this.tableName);
+    }
+    this.table = null;
+  }
+  toRow(r) {
+    return {
+      id: r.id,
+      vector: r.vector,
+      text: r.text,
+      title: r.title,
+      url: r.url ?? "",
+      source: r.source,
+      documentId: r.documentId,
+      chunkIndex: r.chunkIndex,
+      metadata: r.metadata ? JSON.stringify(r.metadata) : void 0
+    };
+  }
+  async upsert(records) {
+    if (records.length === 0) return;
+    const conn = await this.connect();
+    const rows = records.map((r) => this.toRow(r));
+    if (!this.table) {
+      const names = await conn.tableNames();
+      if (names.includes(this.tableName)) {
+        this.table = await conn.openTable(this.tableName);
+        await this.table.add(rows);
+      } else {
+        this.table = await conn.createTable(this.tableName, rows);
+      }
+    } else {
+      await this.table.add(rows);
+    }
+  }
+  async count() {
+    if (!this.table) {
+      await this.init();
+      if (!this.table) return 0;
+    }
+    try {
+      return await this.table.countRows();
+    } catch {
+      return null;
+    }
+  }
+  async query(vector, topK) {
+    if (!this.table) {
+      await this.init();
+      if (!this.table) return [];
+    }
+    const rows = await this.table.search(vector).limit(topK).toArray();
+    return rows.map((row) => ({
+      id: row.id,
+      // Convert L2 distance → a 0..1 similarity-ish score for display.
+      score: typeof row._distance === "number" ? 1 / (1 + row._distance) : 0,
+      text: row.text,
+      title: row.title,
+      url: row.url || void 0,
+      documentId: row.documentId,
+      metadata: typeof row.metadata === "string" ? JSON.parse(row.metadata) : void 0
+    }));
+  }
+  async testConnection(dimensions) {
+    if (this.config.mode !== "cloud") {
+      try {
+        await this.connect();
+        return;
+      } catch (err) {
+        throw new Error(`Failed to connect to local LanceDB: ${err.message}`);
+      }
+    }
+    let conn;
+    try {
+      conn = await this.connect();
+    } catch (err) {
+      throw new Error(`Failed to connect to LanceDB Cloud: ${err.message}`);
+    }
+    try {
+      await conn.tableNames();
+    } catch (err) {
+      const msg = err.message ?? "";
+      if (msg.includes("401") || msg.includes("Unauthorized") || msg.includes("403") || msg.includes("Forbidden")) {
+        throw new Error(
+          "Invalid LanceDB Cloud API key. Please check your credentials in the LanceDB Cloud dashboard."
+        );
+      }
+      if (msg.includes("404") || msg.includes("Not Found")) {
+        throw new Error(
+          `LanceDB Cloud database not found. Please verify your URI (e.g. "db://my-database").`
+        );
+      }
+      throw new Error(`Failed to reach LanceDB Cloud: ${msg}`);
+    }
+  }
+};
+export {
+  LanceDBAdapter
+};

package/dist/pinecone-YZHFLCKI.js

@@ -0,0 +1,315 @@
+import "./chunk-3RG5ZIWI.js";
+
+// ../../packages/vector-stores/src/adapters/pinecone.ts
+import {
+  Pinecone
+} from "@pinecone-database/pinecone";
+var SPARSE_BATCH = 48;
+var BASE_INTER_BATCH_DELAY_MS = 7e3;
+var RATE_LIMIT_WAIT_MS = 65e3;
+var RATE_LIMIT_MAX_RETRIES = 3;
+var RRF_K = 60;
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+function is429(err) {
+  const e = err;
+  return e?.status === 429 || e?.statusCode === 429 || String(e?.message ?? "").includes("429") || String(e?.message ?? "").includes("RESOURCE_EXHAUSTED");
+}
+var PineconeAdapter = class {
+  constructor(config) {
+    this.config = config;
+    if (!config.apiKey) throw new Error("Pinecone requires an API key.");
+    if (!config.indexName) throw new Error("Pinecone requires an index name.");
+    this.indexName = config.indexName.trim();
+    this.namespace = config.namespace?.trim() || "default";
+  }
+  config;
+  client = null;
+  index = null;
+  indexName;
+  namespace;
+  currentInterBatchDelayMs = 0;
+  getClient() {
+    if (!this.client) {
+      this.client = new Pinecone({ apiKey: this.config.apiKey });
+    }
+    return this.client;
+  }
+  get needsSparse() {
+    return this.config.indexType === "lexical" || this.config.indexType === "hybrid";
+  }
+  // ── Lifecycle ──────────────────────────────────────────────────────────────
+  async init(dimensions) {
+    const pc = this.getClient();
+    const { indexes } = await pc.listIndexes();
+    const exists = (indexes ?? []).some((i) => i.name === this.indexName);
+    if (!exists) {
+      const metric = this.needsSparse ? "dotproduct" : "cosine";
+      await pc.createIndex({
+        name: this.indexName,
+        dimension: dimensions,
+        metric,
+        spec: { serverless: { cloud: "aws", region: "us-east-1" } },
+        waitUntilReady: true
+      });
+    }
+    this.index = pc.index(this.indexName);
+  }
+  ns() {
+    if (!this.index) this.index = this.getClient().index(this.indexName);
+    return this.index.namespace(this.namespace);
+  }
+  async reset() {
+    try {
+      await this.ns().deleteAll();
+    } catch {
+    }
+  }
+  // ── Sparse vector generation with rate-limit handling ─────────────────────
+  /**
+   * Call Pinecone Inference API for one batch of texts, retrying on 429.
+   * Before each retry-sleep the `onRateLimit` hook is called so the indexer
+   * can patch the run store with a warning that the UI polls.
+   */
+  async embedSparseWithRetry(texts) {
+    const pc = this.getClient();
+    for (let attempt = 1; attempt <= RATE_LIMIT_MAX_RETRIES + 1; attempt++) {
+      try {
+        const result = await pc.inference.embed({
+          model: this.config.sparseModel,
+          inputs: texts,
+          parameters: { input_type: "passage", truncate: "END" }
+        });
+        return result.data.map((emb) => {
+          const s = emb;
+          return {
+            indices: s.sparseIndices ?? [],
+            values: s.sparseValues ?? []
+          };
+        });
+      } catch (err) {
+        if (is429(err) && attempt <= RATE_LIMIT_MAX_RETRIES) {
+          const waitSecs = Math.round(RATE_LIMIT_WAIT_MS / 1e3);
+          await this.config.onRateLimit?.(waitSecs, attempt);
+          await sleep(RATE_LIMIT_WAIT_MS);
+          this.currentInterBatchDelayMs = Math.max(
+            this.currentInterBatchDelayMs,
+            BASE_INTER_BATCH_DELAY_MS
+          );
+          continue;
+        }
+        throw err;
+      }
+    }
+    throw new Error("Sparse embedding failed after max retries.");
+  }
+  async buildSparseVectors(texts) {
+    const result = [];
+    for (let i = 0; i < texts.length; i += SPARSE_BATCH) {
+      const batch = texts.slice(i, i + SPARSE_BATCH);
+      const vecs = await this.embedSparseWithRetry(batch);
+      result.push(...vecs);
+      if (i + SPARSE_BATCH < texts.length && this.currentInterBatchDelayMs > 0) {
+        await sleep(this.currentInterBatchDelayMs);
+      }
+    }
+    return result;
+  }
+  async buildQuerySparseVector(text) {
+    const pc = this.getClient();
+    for (let attempt = 1; attempt <= RATE_LIMIT_MAX_RETRIES + 1; attempt++) {
+      try {
+        const result = await pc.inference.embed({
+          model: this.config.sparseModel,
+          inputs: [text],
+          parameters: { input_type: "query", truncate: "END" }
+        });
+        const s = result.data[0];
+        return {
+          indices: s.sparseIndices ?? [],
+          values: s.sparseValues ?? []
+        };
+      } catch (err) {
+        if (is429(err) && attempt <= RATE_LIMIT_MAX_RETRIES) {
+          await sleep(RATE_LIMIT_WAIT_MS);
+          continue;
+        }
+        throw err;
+      }
+    }
+    throw new Error("Sparse query embedding failed after max retries.");
+  }
+  // ── Upsert ─────────────────────────────────────────────────────────────────
+  async upsert(records) {
+    if (records.length === 0) return;
+    const sanitizeMeta = (meta) => {
+      if (!meta) return {};
+      const safe = {};
+      for (const [k, v] of Object.entries(meta)) {
+        if (v === null || v === void 0) continue;
+        if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
+          safe[k] = v;
+        } else if (Array.isArray(v) && v.every((item) => typeof item === "string")) {
+          safe[k] = v;
+        } else {
+          safe[k] = JSON.stringify(v);
+        }
+      }
+      return safe;
+    };
+    const metaOf = (r) => ({
+      ...sanitizeMeta(r.metadata),
+      text: r.text,
+      title: r.title,
+      url: r.url ?? "",
+      source: r.source,
+      documentId: r.documentId,
+      chunkIndex: r.chunkIndex
+    });
+    if (!this.needsSparse || !this.config.sparseModel) {
+      await this.ns().upsert({
+        records: records.map((r) => ({
+          id: r.id,
+          values: r.vector,
+          metadata: metaOf(r)
+        }))
+      });
+      return;
+    }
+    const sparseVecs = await this.buildSparseVectors(
+      records.map((r) => r.text)
+    );
+    await this.ns().upsert({
+      records: records.map((r, i) => ({
+        id: r.id,
+        values: r.vector,
+        sparseValues: sparseVecs[i],
+        metadata: metaOf(r)
+      }))
+    });
+  }
+  // ── Query ──────────────────────────────────────────────────────────────────
+  async query(vector, topK, queryText) {
+    const canHybrid = this.needsSparse && this.config.sparseModel && queryText;
+    if (!canHybrid) {
+      return this.denseQuery(vector, topK);
+    }
+    const fetchN = Math.min(topK * 2, 100);
+    const [denseHits, sparseHits] = await Promise.all([
+      this.denseQuery(vector, fetchN),
+      this.sparseQuery(vector, queryText, fetchN)
+    ]);
+    return rrfMerge(denseHits, sparseHits, topK);
+  }
+  async denseQuery(vector, topK) {
+    const res = await this.ns().query({ vector, topK, includeMetadata: true });
+    return (res.matches ?? []).map(hitFromMatch);
+  }
+  /**
+   * Sparse-weighted query for hybrid search.
+   *
+   * Pinecone's dotproduct index ALWAYS requires a non-empty dense `vector`
+   * array — even when the call is primarily sparse. Passing `[]` triggers the
+   * "You must enter an array of RecordValues" error. We pass the real dense
+   * query vector here; Pinecone's scoring blends both dimensions and the RRF
+   * merge downstream takes care of the final ranking.
+   */
+  async sparseQuery(denseVector, queryText, topK) {
+    const sparseVector = await this.buildQuerySparseVector(queryText);
+    const res = await this.ns().query({
+      vector: denseVector,
+      sparseVector,
+      topK,
+      includeMetadata: true
+    });
+    return (res.matches ?? []).map(hitFromMatch);
+  }
+  // ── Count ──────────────────────────────────────────────────────────────────
+  async count() {
+    try {
+      const stats = await this.ns().describeIndexStats();
+      return stats.namespaces?.[this.namespace]?.recordCount ?? stats.totalRecordCount ?? 0;
+    } catch {
+      return null;
+    }
+  }
+  // ── Connection test ────────────────────────────────────────────────────────
+  async testConnection(dimensions) {
+    const pc = this.getClient();
+    let indexList;
+    try {
+      indexList = await pc.listIndexes();
+    } catch (err) {
+      if (err.message?.toLowerCase().includes("api key") || err.name === "PineconeAuthorizationError" || err.status === 401) {
+        throw new Error("Invalid Pinecone API key.");
+      }
+      throw new Error(`Failed to connect to Pinecone: ${err.message}`);
+    }
+    const indexModel = (indexList.indexes ?? []).find(
+      (i) => i.name === this.indexName
+    );
+    if (!indexModel) {
+      throw new Error(
+        `Index "${this.indexName}" does not exist in your Pinecone project. Please create it first.`
+      );
+    }
+    if (indexModel.dimension !== dimensions) {
+      throw new Error(
+        `Dimension mismatch: Index "${this.indexName}" has dimension ${indexModel.dimension}, but the selected embedding model requires dimension ${dimensions}.`
+      );
+    }
+    if (this.needsSparse) {
+      if (!this.config.sparseModel) {
+        throw new Error(
+          "Hybrid/lexical search requires a sparse model to be selected."
+        );
+      }
+      const metric = indexModel.metric ?? indexModel.spec?.metric;
+      if (metric && metric !== "dotproduct") {
+        throw new Error(
+          `Hybrid/lexical requires your Pinecone index to use the "dotproduct" metric, but "${this.indexName}" uses "${metric}". Please delete and recreate the index with metric = dotproduct.`
+        );
+      }
+    }
+  }
+};
+function hitFromMatch(m) {
+  const meta = m.metadata ?? {};
+  const {
+    text,
+    title,
+    url,
+    source,
+    documentId,
+    chunkIndex,
+    ...customMetadata
+  } = meta;
+  return {
+    id: m.id,
+    score: m.score ?? 0,
+    text: text ?? "",
+    title: title ?? "Untitled",
+    url: url || void 0,
+    documentId: documentId ?? "",
+    metadata: Object.keys(customMetadata).length > 0 ? customMetadata : void 0
+  };
+}
+function rrfMerge(denseHits, sparseHits, topK) {
+  const scores = /* @__PURE__ */ new Map();
+  const addList = (hits) => hits.forEach((hit, rank) => {
+    const contrib = 1 / (RRF_K + rank + 1);
+    const prev = scores.get(hit.id);
+    if (prev) {
+      prev.score += contrib;
+    } else {
+      scores.set(hit.id, { hit, score: contrib });
+    }
+  });
+  addList(denseHits);
+  addList(sparseHits);
+  return [...scores.values()].sort((a, b) => b.score - a.score).slice(0, topK).map(({ hit, score }) => ({ ...hit, score }));
+}
+export {
+  PineconeAdapter
+};

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@larkup/cli",
+  "version": "0.1.14",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "bin": {
+    "larkup": "./dist/index.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "larkup": "tsx src/index.ts"
+  },
+  "dependencies": {
+    "@ai-sdk/cohere": "^3.0.39",
+    "@ai-sdk/deepseek": "^2.0.39",
+    "@ai-sdk/gateway": "^3.0.133",
+    "@ai-sdk/google": "^3.0.83",
+    "@ai-sdk/mistral": "^3.0.40",
+    "@ai-sdk/openai": "^3.0.68",
+    "@ai-sdk/openai-compatible": "^2.0.51",
+    "@clack/prompts": "^0.10.1",
+    "@lancedb/lancedb": "^0.30.0",
+    "@larkup/core": "workspace:*",
+    "@larkup/vector-stores": "workspace:*",
+    "@pinecone-database/pinecone": "^7.2.0",
+    "ai": "^6.0.197",
+    "chalk": "^5.6.2",
+    "chromadb": "^1.10.5",
+    "cli-table3": "^0.6.5",
+    "commander": "^13.1.0",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "tsup": "^8.0.0",
+    "tsx": "^4.22.4",
+    "typescript": "5.7.3"
+  }
+}