akm-cli 0.0.0 → 0.0.17

package/dist/db.js

@@ -0,0 +1,371 @@
+import { Database } from "bun:sqlite";
+import fs from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+import { cosineSimilarity } from "./embedder";
+import { getDbPath } from "./paths";
+import { warn } from "./warn";
+// ── Constants ───────────────────────────────────────────────────────────────
+export const DB_VERSION = 6;
+export const EMBEDDING_DIM = 384;
+// ── Database lifecycle ──────────────────────────────────────────────────────
+export function openDatabase(dbPath, options) {
+    const resolvedPath = dbPath ?? getDbPath();
+    const dir = path.dirname(resolvedPath);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+    const db = new Database(resolvedPath);
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
+    // Try to load sqlite-vec extension
+    loadVecExtension(db);
+    ensureSchema(db, options?.embeddingDim ?? EMBEDDING_DIM);
+    // Warn once at init if using JS fallback with many entries
+    warnIfVecMissing(db, { once: true });
+    return db;
+}
+export function closeDatabase(db) {
+    db.close();
+}
+// ── sqlite-vec extension ────────────────────────────────────────────────────
+const vecStatus = new WeakMap();
+function loadVecExtension(db) {
+    try {
+        const esmRequire = createRequire(import.meta.url);
+        const sqliteVec = esmRequire("sqlite-vec");
+        sqliteVec.load(db);
+        vecStatus.set(db, true);
+    }
+    catch {
+        vecStatus.set(db, false);
+    }
+}
+export function isVecAvailable(db) {
+    return vecStatus.get(db) ?? false;
+}
+const VEC_DOCS_URL = "https://github.com/itlackey/agentikit/blob/main/docs/configuration.md#sqlite-vec-extension";
+const VEC_FALLBACK_THRESHOLD = 10_000;
+let vecInitWarned = false;
+/**
+ * Warn if sqlite-vec is unavailable and embedding count exceeds threshold.
+ * Called from openDatabase (once at init) and from indexer (each run).
+ */
+export function warnIfVecMissing(db, { once } = { once: false }) {
+    if (isVecAvailable(db))
+        return;
+    if (once && vecInitWarned)
+        return;
+    try {
+        const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
+        const count = row?.cnt ?? 0;
+        if (count >= VEC_FALLBACK_THRESHOLD) {
+            warn("Semantic search is using JS fallback for %d entries. Install sqlite-vec for faster performance.\n  See: %s", count, VEC_DOCS_URL);
+            if (once)
+                vecInitWarned = true;
+        }
+    }
+    catch {
+        /* embeddings table may not exist yet during init */
+    }
+}
+// ── Schema ──────────────────────────────────────────────────────────────────
+function ensureSchema(db, embeddingDim) {
+    // Create meta table first so we can check version
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS index_meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `);
+    // Check stored version — if it differs from DB_VERSION, drop and recreate all tables
+    const storedVersion = getMeta(db, "version");
+    if (storedVersion && storedVersion !== String(DB_VERSION)) {
+        db.exec("DROP TABLE IF EXISTS embeddings");
+        db.exec("DROP TABLE IF EXISTS entries_vec");
+        db.exec("DROP TABLE IF EXISTS entries_fts");
+        db.exec("DROP INDEX IF EXISTS idx_entries_dir");
+        db.exec("DROP INDEX IF EXISTS idx_entries_type");
+        db.exec("DROP TABLE IF EXISTS entries");
+        db.exec("DELETE FROM index_meta");
+    }
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS entries (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      entry_key   TEXT NOT NULL UNIQUE,
+      dir_path    TEXT NOT NULL,
+      file_path   TEXT NOT NULL,
+      stash_dir   TEXT NOT NULL,
+      entry_json  TEXT NOT NULL,
+      search_text TEXT NOT NULL,
+      entry_type  TEXT NOT NULL
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
+    CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
+  `);
+    // BLOB-based embedding storage (always available, no sqlite-vec needed)
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      id        INTEGER PRIMARY KEY,
+      embedding BLOB NOT NULL,
+      FOREIGN KEY (id) REFERENCES entries(id)
+    );
+  `);
+    // FTS5 table — standalone with explicit entry_id for joining
+    const ftsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_fts'").get();
+    if (!ftsExists) {
+        db.exec(`
+      CREATE VIRTUAL TABLE entries_fts USING fts5(
+        entry_id UNINDEXED,
+        search_text,
+        tokenize='porter unicode61'
+      );
+    `);
+    }
+    // sqlite-vec table
+    if (isVecAvailable(db)) {
+        // Check if stored embedding dimension differs from configured one
+        const storedDim = getMeta(db, "embeddingDim");
+        if (storedDim && storedDim !== String(embeddingDim)) {
+            try {
+                db.exec("DROP TABLE IF EXISTS entries_vec");
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_vec'").get();
+        if (!vecExists) {
+            db.exec(`
+        CREATE VIRTUAL TABLE entries_vec USING vec0(
+          id       INTEGER PRIMARY KEY,
+          embedding FLOAT[${embeddingDim}]
+        );
+      `);
+        }
+        setMeta(db, "embeddingDim", String(embeddingDim));
+    }
+    // Set version if not present
+    const version = getMeta(db, "version");
+    if (!version) {
+        setMeta(db, "version", String(DB_VERSION));
+    }
+}
+// ── Meta helpers ────────────────────────────────────────────────────────────
+export function getMeta(db, key) {
+    const row = db.prepare("SELECT value FROM index_meta WHERE key = ?").get(key);
+    return row?.value;
+}
+export function setMeta(db, key, value) {
+    db.prepare("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)").run(key, value);
+}
+// ── Entry operations ────────────────────────────────────────────────────────
+export function upsertEntry(db, entryKey, dirPath, filePath, stashDir, entry, searchText) {
+    const stmt = db.prepare(`
+    INSERT INTO entries (entry_key, dir_path, file_path, stash_dir, entry_json, search_text, entry_type)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(entry_key) DO UPDATE SET
+      dir_path = excluded.dir_path,
+      file_path = excluded.file_path,
+      stash_dir = excluded.stash_dir,
+      entry_json = excluded.entry_json,
+      search_text = excluded.search_text,
+      entry_type = excluded.entry_type
+  `);
+    stmt.run(entryKey, dirPath, filePath, stashDir, JSON.stringify(entry), searchText, entry.type);
+    // Fetch the row id explicitly since last_insert_rowid() is unreliable for ON CONFLICT DO UPDATE
+    const row = db.prepare("SELECT id FROM entries WHERE entry_key = ?").get(entryKey);
+    return row.id;
+}
+export function deleteEntriesByDir(db, dirPath) {
+    const ids = db.prepare("SELECT id FROM entries WHERE dir_path = ?").all(dirPath);
+    for (const { id } of ids) {
+        try {
+            db.prepare("DELETE FROM embeddings WHERE id = ?").run(id);
+        }
+        catch {
+            /* ignore */
+        }
+        if (isVecAvailable(db)) {
+            try {
+                db.prepare("DELETE FROM entries_vec WHERE id = ?").run(id);
+            }
+            catch {
+                /* ignore */
+            }
+        }
+    }
+    db.prepare("DELETE FROM entries WHERE dir_path = ?").run(dirPath);
+}
+export function rebuildFts(db) {
+    db.exec("DELETE FROM entries_fts");
+    db.exec("INSERT INTO entries_fts (entry_id, search_text) SELECT CAST(id AS TEXT), search_text FROM entries");
+}
+// ── Vector operations ───────────────────────────────────────────────────────
+export function upsertEmbedding(db, entryId, embedding) {
+    const buf = float32Buffer(embedding);
+    // Always write to BLOB table (works without sqlite-vec)
+    db.prepare("INSERT OR REPLACE INTO embeddings (id, embedding) VALUES (?, ?)").run(entryId, buf);
+    // Also write to sqlite-vec table when available (fast path)
+    if (isVecAvailable(db)) {
+        try {
+            db.prepare("DELETE FROM entries_vec WHERE id = ?").run(entryId);
+        }
+        catch {
+            /* ignore */
+        }
+        db.prepare("INSERT INTO entries_vec (id, embedding) VALUES (?, ?)").run(entryId, buf);
+    }
+}
+export function searchVec(db, queryEmbedding, k) {
+    // Fast path: use sqlite-vec when available
+    if (isVecAvailable(db)) {
+        const buf = float32Buffer(queryEmbedding);
+        try {
+            return db
+                .prepare("SELECT id, distance FROM entries_vec WHERE embedding MATCH ? AND k = ?")
+                .all(buf, k);
+        }
+        catch {
+            return [];
+        }
+    }
+    // Fallback: JS-based cosine similarity over BLOB table
+    return searchBlobVec(db, queryEmbedding, k);
+}
+function float32Buffer(vec) {
+    const f32 = new Float32Array(vec);
+    return Buffer.from(f32.buffer);
+}
+function bufferToFloat32(buf) {
+    const f32 = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
+    return Array.from(f32);
+}
+function searchBlobVec(db, queryEmbedding, k) {
+    try {
+        const rows = db.prepare("SELECT id, embedding FROM embeddings").all();
+        if (rows.length === 0)
+            return [];
+        const scored = [];
+        for (const row of rows) {
+            const embedding = bufferToFloat32(row.embedding);
+            const similarity = cosineSimilarity(queryEmbedding, embedding);
+            scored.push({ id: row.id, similarity });
+        }
+        scored.sort((a, b) => b.similarity - a.similarity);
+        // Convert cosine similarity to L2 distance for compatibility with sqlite-vec interface
+        // For normalized vectors: L2² = 2(1 - cos_sim)
+        return scored.slice(0, k).map(({ id, similarity }) => ({
+            id,
+            distance: Math.sqrt(2 * Math.max(0, 1 - similarity)),
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+// ── FTS5 search ─────────────────────────────────────────────────────────────
+export function searchFts(db, query, limit, entryType) {
+    const ftsQuery = sanitizeFtsQuery(query);
+    if (!ftsQuery)
+        return [];
+    let sql;
+    let params;
+    if (entryType && entryType !== "any") {
+        sql = `
+      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
+             bm25(entries_fts) AS bm25Score
+      FROM entries_fts f
+      JOIN entries e ON e.id = CAST(f.entry_id AS INTEGER)
+      WHERE entries_fts MATCH ?
+        AND e.entry_type = ?
+      ORDER BY bm25Score
+      LIMIT ?
+    `;
+        params = [ftsQuery, entryType, limit];
+    }
+    else {
+        sql = `
+      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
+             bm25(entries_fts) AS bm25Score
+      FROM entries_fts f
+      JOIN entries e ON e.id = CAST(f.entry_id AS INTEGER)
+      WHERE entries_fts MATCH ?
+      ORDER BY bm25Score
+      LIMIT ?
+    `;
+        params = [ftsQuery, limit];
+    }
+    try {
+        const rows = db.prepare(sql).all(...params);
+        return rows.map((row) => ({
+            id: row.id,
+            filePath: row.filePath,
+            entry: JSON.parse(row.entry_json),
+            searchText: row.searchText,
+            bm25Score: row.bm25Score,
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+function sanitizeFtsQuery(query) {
+    const tokens = query
+        .replace(/[^a-zA-Z0-9\s]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length >= 1);
+    if (tokens.length === 0)
+        return "";
+    // Use unquoted tokens so the porter stemmer can normalize word forms
+    return tokens.join(" ");
+}
+// ── All entries ─────────────────────────────────────────────────────────────
+export function getAllEntries(db, entryType) {
+    let sql;
+    let params;
+    if (entryType && entryType !== "any") {
+        sql =
+            "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type = ?";
+        params = [entryType];
+    }
+    else {
+        sql = "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries";
+        params = [];
+    }
+    const rows = db.prepare(sql).all(...params);
+    return rows.map((row) => ({
+        id: row.id,
+        entryKey: row.entry_key,
+        dirPath: row.dir_path,
+        filePath: row.file_path,
+        stashDir: row.stash_dir,
+        entry: JSON.parse(row.entry_json),
+        searchText: row.search_text,
+    }));
+}
+export function getEntryCount(db) {
+    const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
+    return row.cnt;
+}
+export function getEntryById(db, id) {
+    const row = db.prepare("SELECT file_path, entry_json FROM entries WHERE id = ?").get(id);
+    if (!row)
+        return undefined;
+    return { filePath: row.file_path, entry: JSON.parse(row.entry_json) };
+}
+export function getEntriesByDir(db, dirPath) {
+    const rows = db
+        .prepare("SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE dir_path = ?")
+        .all(dirPath);
+    return rows.map((row) => ({
+        id: row.id,
+        entryKey: row.entry_key,
+        dirPath: row.dir_path,
+        filePath: row.file_path,
+        stashDir: row.stash_dir,
+        entry: JSON.parse(row.entry_json),
+        searchText: row.search_text,
+    }));
+}

package/dist/embedder.js

@@ -0,0 +1,150 @@
+import { fetchWithTimeout } from "./common";
+let localEmbedder;
+async function getLocalEmbedder() {
+    if (!localEmbedder) {
+        let pipeline;
+        try {
+            const mod = await import("@xenova/transformers");
+            pipeline = mod.pipeline;
+        }
+        catch {
+            throw new Error("Semantic search requires @xenova/transformers. Install it with: npm install @xenova/transformers");
+        }
+        const pipelineFn = pipeline;
+        localEmbedder = await pipelineFn("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+    }
+    if (!localEmbedder) {
+        throw new Error("Failed to initialize local embedder.");
+    }
+    return localEmbedder;
+}
+async function embedLocal(text) {
+    const model = await getLocalEmbedder();
+    const result = await model(text, { pooling: "mean", normalize: true });
+    return Array.from(result.data);
+}
+// ── OpenAI-compatible remote embedder ───────────────────────────────────────
+async function embedRemote(text, config) {
+    const headers = { "Content-Type": "application/json" };
+    if (config.apiKey) {
+        headers.Authorization = `Bearer ${config.apiKey}`;
+    }
+    const body = {
+        input: text,
+        model: config.model,
+    };
+    if (config.dimension) {
+        body.dimensions = config.dimension;
+    }
+    const response = await fetchWithTimeout(config.endpoint, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(body),
+    });
+    if (!response.ok) {
+        const body = await response.text().catch(() => "");
+        throw new Error(`Embedding request failed (${response.status}): ${body}`);
+    }
+    const json = (await response.json());
+    if (!json.data?.[0]?.embedding) {
+        throw new Error("Unexpected embedding response format: missing data[0].embedding");
+    }
+    return json.data[0].embedding;
+}
+// ── Public API ──────────────────────────────────────────────────────────────
+/**
+ * Generate an embedding for the given text.
+ * If embeddingConfig is provided, uses the configured OpenAI-compatible endpoint.
+ * Otherwise falls back to local @xenova/transformers.
+ */
+export async function embed(text, embeddingConfig) {
+    if (embeddingConfig) {
+        return embedRemote(text, embeddingConfig);
+    }
+    return embedLocal(text);
+}
+// ── Batch embedding ─────────────────────────────────────────────────────────
+/**
+ * Generate embeddings for multiple texts in batch.
+ * Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
+ * Falls back to sequential embedding for local transformer pipeline.
+ */
+export async function embedBatch(texts, embeddingConfig) {
+    if (texts.length === 0)
+        return [];
+    if (embeddingConfig) {
+        return embedRemoteBatch(texts, embeddingConfig);
+    }
+    // Local transformer: process sequentially (pipeline handles one at a time)
+    const results = [];
+    for (const text of texts) {
+        results.push(await embedLocal(text));
+    }
+    return results;
+}
+async function embedRemoteBatch(texts, config) {
+    const BATCH_SIZE = 100;
+    const results = [];
+    const headers = { "Content-Type": "application/json" };
+    if (config.apiKey) {
+        headers.Authorization = `Bearer ${config.apiKey}`;
+    }
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+        const batch = texts.slice(i, i + BATCH_SIZE);
+        const body = {
+            input: batch,
+            model: config.model,
+        };
+        if (config.dimension) {
+            body.dimensions = config.dimension;
+        }
+        const response = await fetchWithTimeout(config.endpoint, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        if (!response.ok) {
+            const respBody = await response.text().catch(() => "");
+            throw new Error(`Embedding batch request failed (${response.status}): ${respBody}`);
+        }
+        const json = (await response.json());
+        if (!json.data || json.data.length !== batch.length) {
+            throw new Error(`Unexpected embedding batch response: expected ${batch.length} embeddings, got ${json.data?.length ?? 0}`);
+        }
+        results.push(...json.data.map((d) => d.embedding));
+    }
+    return results;
+}
+// ── Similarity ──────────────────────────────────────────────────────────────
+export function cosineSimilarity(a, b) {
+    const len = Math.min(a.length, b.length);
+    if (len === 0)
+        return 0;
+    let dot = 0, magA = 0, magB = 0;
+    for (let i = 0; i < len; i++) {
+        dot += a[i] * b[i];
+        magA += a[i] * a[i];
+        magB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(magA) * Math.sqrt(magB);
+    return denom === 0 ? 0 : dot / denom;
+}
+// ── Availability check ──────────────────────────────────────────────────────
+export async function isEmbeddingAvailable(embeddingConfig) {
+    if (embeddingConfig) {
+        try {
+            await embedRemote("test", embeddingConfig);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    try {
+        await getLocalEmbedder();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/errors.js

@@ -0,0 +1,28 @@
+/**
+ * Typed error classes for structured exit code classification.
+ *
+ * - ConfigError  -> exit 78  (configuration / environment problems)
+ * - UsageError   -> exit 2   (bad CLI arguments or invalid input)
+ * - NotFoundError -> exit 1  (requested resource missing)
+ */
+/** Raised when configuration or environment is invalid or missing. */
+export class ConfigError extends Error {
+    constructor(msg) {
+        super(msg);
+        this.name = "ConfigError";
+    }
+}
+/** Raised when the user supplies invalid arguments or input. */
+export class UsageError extends Error {
+    constructor(msg) {
+        super(msg);
+        this.name = "UsageError";
+    }
+}
+/** Raised when a requested resource (asset, entry, file) is not found. */
+export class NotFoundError extends Error {
+    constructor(msg) {
+        super(msg);
+        this.name = "NotFoundError";
+    }
+}