@kyleparrott/where-was-i 0.1.0

package/dist/src/core/semantic.js

@@ -0,0 +1,220 @@
+import fs from "node:fs";
+import { SessionSearchDb } from "./database.js";
+import { EMBEDDING_PROVIDER_UNCONFIGURED_MESSAGE, EmbeddingProviderUnavailableError, OpenAICompatibleEmbeddingProvider, isEmbeddingProviderConfigured } from "./embeddings.js";
+import { clearEmbeddings, countChunksMissingEmbeddings, embeddingCoverage, ensureVectorStore, listChunksMissingEmbeddings, upsertChunkEmbedding } from "./vector.js";
+export async function indexSemanticChunks(options) {
+    const provider = new OpenAICompatibleEmbeddingProvider(options.embedding);
+    const batchSize = Math.min(Math.max(options.batchSize ?? 32, 1), 128);
+    const maxChunks = Math.max(options.maxChunks ?? Number.MAX_SAFE_INTEGER, 1);
+    const db = new SessionSearchDb(options.dbPath);
+    let indexed = 0;
+    const notify = options.onProgress;
+    try {
+        notify?.({
+            phase: "probing",
+            providerId: provider.config.id,
+            model: provider.config.model,
+            dimensions: null,
+            indexed,
+            total: null
+        });
+        let dimensions;
+        try {
+            dimensions = await provider.probeDimensions();
+        }
+        catch (error) {
+            if (error instanceof EmbeddingProviderUnavailableError) {
+                const summary = unavailableSummary(provider, error);
+                notify?.({
+                    phase: "unavailable",
+                    providerId: provider.config.id,
+                    model: provider.config.model,
+                    dimensions: null,
+                    indexed,
+                    total: null,
+                    failed: summary.failed
+                });
+                return summary;
+            }
+            throw error;
+        }
+        ensureVectorStore(db.db, provider.config, dimensions);
+        const totalMissing = countChunksMissingEmbeddings(db.db, provider.config, dimensions, options.sessionIds ?? []);
+        const total = Math.min(totalMissing, maxChunks);
+        notify?.({
+            phase: "start",
+            providerId: provider.config.id,
+            model: provider.config.model,
+            dimensions,
+            indexed,
+            total
+        });
+        while (indexed < total) {
+            const remaining = total - indexed;
+            const chunks = listChunksMissingEmbeddings(db.db, provider.config, dimensions, Math.min(batchSize, remaining), options.sessionIds ?? []);
+            if (chunks.length === 0) {
+                const summary = {
+                    available: true,
+                    providerId: provider.config.id,
+                    model: provider.config.model,
+                    dimensions,
+                    indexed,
+                    skipped: 0,
+                    failed: null
+                };
+                notify?.({
+                    phase: "complete",
+                    providerId: provider.config.id,
+                    model: provider.config.model,
+                    dimensions,
+                    indexed,
+                    total
+                });
+                return summary;
+            }
+            notify?.({
+                phase: "batch",
+                providerId: provider.config.id,
+                model: provider.config.model,
+                dimensions,
+                indexed,
+                total,
+                batchSize: chunks.length
+            });
+            const batch = await provider.embedDocuments(chunks.map((chunk) => chunk.text));
+            for (let index = 0; index < chunks.length; index += 1) {
+                const chunk = chunks[index];
+                const embedding = batch.embeddings[index];
+                if (chunk && embedding) {
+                    upsertChunkEmbedding(db.db, provider.config, dimensions, chunk, embedding);
+                    indexed += 1;
+                }
+            }
+            notify?.({
+                phase: "progress",
+                providerId: provider.config.id,
+                model: provider.config.model,
+                dimensions,
+                indexed,
+                total,
+                batchSize: chunks.length
+            });
+        }
+        const summary = {
+            available: true,
+            providerId: provider.config.id,
+            model: provider.config.model,
+            dimensions,
+            indexed,
+            skipped: 0,
+            failed: null
+        };
+        notify?.({
+            phase: "complete",
+            providerId: provider.config.id,
+            model: provider.config.model,
+            dimensions,
+            indexed,
+            total
+        });
+        return summary;
+    }
+    catch (error) {
+        if (error instanceof EmbeddingProviderUnavailableError) {
+            const summary = unavailableSummary(provider, error, indexed);
+            notify?.({
+                phase: "unavailable",
+                providerId: provider.config.id,
+                model: provider.config.model,
+                dimensions: null,
+                indexed,
+                total: null,
+                failed: summary.failed
+            });
+            return summary;
+        }
+        throw error;
+    }
+    finally {
+        db.close();
+    }
+}
+export function clearSemanticEmbeddings(dbPath) {
+    const db = new SessionSearchDb(dbPath);
+    try {
+        return clearEmbeddings(db.db);
+    }
+    finally {
+        db.close();
+    }
+}
+export async function semanticFreshness(options) {
+    const provider = new OpenAICompatibleEmbeddingProvider(options.embedding);
+    let dimensions = options.dimensions ?? null;
+    let providerAvailable = null;
+    let providerError = null;
+    const configured = isEmbeddingProviderConfigured(provider.config);
+    if (!configured) {
+        providerAvailable = false;
+        providerError = EMBEDDING_PROVIDER_UNCONFIGURED_MESSAGE;
+    }
+    else if (options.probeProvider) {
+        try {
+            dimensions = await provider.probeDimensions();
+            providerAvailable = true;
+        }
+        catch (error) {
+            providerAvailable = false;
+            providerError = error instanceof Error ? error.message : String(error);
+        }
+    }
+    if (!fs.existsSync(options.dbPath)) {
+        const coverage = emptyEmbeddingCoverage(provider.config, configured ? dimensions : null);
+        return {
+            ...coverage,
+            staleForConfiguredProvider: false,
+            recommendation: configured ? coverage.recommendation : "Configure semantic embeddings before running semantic search.",
+            providerAvailable,
+            providerError
+        };
+    }
+    const db = new SessionSearchDb(options.dbPath);
+    try {
+        const coverage = embeddingCoverage(db.db, provider.config, configured ? dimensions : null);
+        return {
+            ...coverage,
+            staleForConfiguredProvider: configured ? coverage.staleForConfiguredProvider : false,
+            recommendation: configured ? coverage.recommendation : "Configure semantic embeddings before running semantic search.",
+            providerAvailable,
+            providerError
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+function emptyEmbeddingCoverage(provider, dimensions) {
+    return {
+        providerId: provider.id,
+        model: provider.model,
+        dimensions,
+        totalEligibleChunks: 0,
+        indexedChunks: 0,
+        missingChunks: dimensions ? 0 : null,
+        lastIndexedAt: null,
+        incompatibleStoredVectors: 0,
+        staleForConfiguredProvider: false,
+        recommendation: null
+    };
+}
+function unavailableSummary(provider, error, indexed = 0) {
+    return {
+        available: false,
+        providerId: provider.config.id,
+        model: provider.config.model,
+        dimensions: null,
+        indexed,
+        skipped: 0,
+        failed: error.message
+    };
+}

package/dist/src/core/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/core/vector.js

@@ -0,0 +1,311 @@
+import crypto from "node:crypto";
+export function ensureVectorStore(db, provider, dimensions) {
+    if (!Number.isInteger(dimensions) || dimensions <= 0) {
+        throw new Error(`Vector dimensions must be a positive integer. Received ${dimensions}.`);
+    }
+    ensureVectorTable(db, dimensions);
+    assertNoIncompatibleStoredVectors(db, provider, dimensions);
+    db.prepare(`INSERT INTO embedding_providers (id, base_url, model, dimensions, updated_at)
+     VALUES (@id, @baseUrl, @model, @dimensions, @updatedAt)
+     ON CONFLICT(id) DO UPDATE SET
+      base_url = excluded.base_url,
+      model = excluded.model,
+      dimensions = excluded.dimensions,
+      updated_at = excluded.updated_at`).run({
+        id: provider.id,
+        baseUrl: provider.baseUrl,
+        model: provider.model,
+        dimensions,
+        updatedAt: new Date().toISOString()
+    });
+}
+export function upsertChunkEmbedding(db, provider, dimensions, chunk, embedding) {
+    if (embedding.length !== dimensions) {
+        throw new Error(`Embedding dimension mismatch for chunk ${chunk.id}: expected ${dimensions}, got ${embedding.length}.`);
+    }
+    ensureVectorStore(db, provider, dimensions);
+    deleteChunkEmbedding(db, chunk.id);
+    const indexedAt = new Date().toISOString();
+    const result = db
+        .prepare(`INSERT INTO vector_chunks_metadata
+        (chunk_id, message_id, session_id, provider_id, model, dimensions, indexed_at, source_fingerprint)
+       VALUES
+        (@chunkId, @messageId, @sessionId, @providerId, @model, @dimensions, @indexedAt, @sourceFingerprint)`)
+        .run({
+        chunkId: chunk.id,
+        messageId: chunk.messageId,
+        sessionId: chunk.sessionId,
+        providerId: provider.id,
+        model: provider.model,
+        dimensions,
+        indexedAt,
+        sourceFingerprint: sourceFingerprint(chunk)
+    });
+    const vectorRowid = BigInt(result.lastInsertRowid);
+    db.prepare("INSERT INTO vector_chunks(rowid, embedding) VALUES (?, ?)").run(vectorRowid, vectorToBuffer(embedding));
+}
+export function deleteChunkEmbedding(db, chunkId) {
+    if (!tableExists(db, "vector_chunks")) {
+        return;
+    }
+    const row = db
+        .prepare("SELECT vector_rowid AS vectorRowid FROM vector_chunks_metadata WHERE chunk_id = ?")
+        .get(chunkId);
+    if (!row) {
+        return;
+    }
+    db.prepare("DELETE FROM vector_chunks WHERE rowid = ?").run(BigInt(row.vectorRowid));
+    db.prepare("DELETE FROM vector_chunks_metadata WHERE vector_rowid = ?").run(row.vectorRowid);
+}
+export function searchVectorChunks(db, provider, embedding, limit) {
+    if (!tableExists(db, "vector_chunks")) {
+        return [];
+    }
+    assertVectorTableDimensions(db, embedding.length);
+    assertNoIncompatibleStoredVectors(db, provider, embedding.length);
+    const rows = db
+        .prepare(`SELECT
+        vector_chunks.rowid AS vectorRowid,
+        vector_chunks.distance AS distance,
+        m.chunk_id AS chunkId,
+        m.message_id AS messageId,
+        m.session_id AS sessionId,
+        m.provider_id AS providerId,
+        m.model AS model,
+        m.dimensions AS dimensions,
+        m.indexed_at AS indexedAt,
+        m.source_fingerprint AS sourceFingerprint
+       FROM vector_chunks
+       JOIN vector_chunks_metadata m ON m.vector_rowid = vector_chunks.rowid
+       WHERE embedding MATCH ? AND k = ?
+        AND m.provider_id = ?
+        AND m.model = ?
+        AND m.dimensions = ?
+       ORDER BY distance
+       LIMIT ?`)
+        .all(vectorToBuffer(embedding), Math.max(limit, 1), provider.id, provider.model, embedding.length, Math.max(limit, 1));
+    return rows;
+}
+export function listChunksMissingEmbeddings(db, provider, dimensions, limit = 500, sessionIds = []) {
+    const sessionFilter = sessionIds.length > 0 ? `AND c.session_id IN (${sessionIds.map(() => "?").join(", ")})` : "";
+    const rows = db
+        .prepare(`SELECT
+        c.id,
+        c.message_id AS messageId,
+        c.session_id AS sessionId,
+        c.conversation_id AS conversationId,
+        c.turn_id AS turnId,
+        c.source,
+        c.source_path AS sourcePath,
+        c.ordinal,
+        c.chunk_index AS chunkIndex,
+        c.role,
+        c.kind,
+        c.timestamp,
+        c.text,
+        c.line_start AS lineStart,
+        c.line_end AS lineEnd,
+        c.metadata_json AS metadataJson
+       FROM chunks c
+       LEFT JOIN vector_chunks_metadata v
+        ON v.chunk_id = c.id
+        AND v.provider_id = ?
+        AND v.model = ?
+        AND v.dimensions = ?
+       WHERE c.role IN ('user', 'assistant')
+        AND c.message_id IS NOT NULL
+        AND c.conversation_id IS NOT NULL
+        AND c.turn_id IS NOT NULL
+        ${sessionFilter}
+        AND v.chunk_id IS NULL
+       ORDER BY c.ordinal ASC
+       LIMIT ?`)
+        .all(provider.id, provider.model, dimensions, ...sessionIds, Math.max(limit, 1));
+    return rows.map((row) => {
+        const { metadataJson, ...rest } = row;
+        return {
+            ...rest,
+            metadata: JSON.parse(metadataJson)
+        };
+    });
+}
+export function countChunksMissingEmbeddings(db, provider, dimensions, sessionIds = []) {
+    const sessionFilter = sessionIds.length > 0 ? `AND c.session_id IN (${sessionIds.map(() => "?").join(", ")})` : "";
+    const row = db
+        .prepare(`SELECT COUNT(*) AS count
+       FROM chunks c
+       LEFT JOIN vector_chunks_metadata v
+        ON v.chunk_id = c.id
+        AND v.provider_id = ?
+        AND v.model = ?
+        AND v.dimensions = ?
+       WHERE c.role IN ('user', 'assistant')
+        AND c.message_id IS NOT NULL
+        AND c.conversation_id IS NOT NULL
+        AND c.turn_id IS NOT NULL
+        ${sessionFilter}
+        AND v.chunk_id IS NULL`)
+        .get(provider.id, provider.model, dimensions, ...sessionIds);
+    return row.count;
+}
+export function vectorToBuffer(vector) {
+    return Buffer.from(Float32Array.from(vector).buffer);
+}
+export function clearEmbeddings(db) {
+    const vectorsDeleted = tableExists(db, "vector_chunks_metadata")
+        ? (db.prepare("SELECT COUNT(*) AS count FROM vector_chunks_metadata").get().count ?? 0)
+        : 0;
+    const providersDeleted = tableExists(db, "embedding_providers")
+        ? (db.prepare("SELECT COUNT(*) AS count FROM embedding_providers").get().count ?? 0)
+        : 0;
+    const clear = db.transaction(() => {
+        if (tableExists(db, "vector_chunks")) {
+            db.exec("DROP TABLE vector_chunks");
+        }
+        if (tableExists(db, "vector_chunks_metadata")) {
+            db.prepare("DELETE FROM vector_chunks_metadata").run();
+        }
+        if (tableExists(db, "embedding_providers")) {
+            db.prepare("DELETE FROM embedding_providers").run();
+        }
+    });
+    clear();
+    return { vectorsDeleted, providersDeleted };
+}
+export function embeddingCoverage(db, provider, dimensions) {
+    const storedDimensions = dimensions ?? storedProviderDimensions(db, provider);
+    const totalEligibleChunks = countEligibleChunks(db);
+    const incompatibleStoredVectors = countIncompatibleStoredVectors(db, provider, storedDimensions);
+    const lastIndexedAt = latestEmbeddingIndexedAt(db, provider, storedDimensions);
+    if (!storedDimensions) {
+        return {
+            providerId: provider.id,
+            model: provider.model,
+            dimensions: null,
+            totalEligibleChunks,
+            indexedChunks: 0,
+            missingChunks: null,
+            lastIndexedAt,
+            incompatibleStoredVectors,
+            staleForConfiguredProvider: totalEligibleChunks > 0 || incompatibleStoredVectors > 0,
+            recommendation: incompatibleStoredVectors > 0
+                ? "Run `wwi embeddings clear` before switching embedding models or dimensions."
+                : totalEligibleChunks > 0
+                    ? "Run `wwi index --semantic` to build embeddings for the configured provider."
+                    : null
+        };
+    }
+    const indexedChunks = tableExists(db, "vector_chunks_metadata")
+        ? db
+            .prepare(`SELECT COUNT(*) AS count
+           FROM vector_chunks_metadata
+           WHERE provider_id = ? AND model = ? AND dimensions = ?`)
+            .get(provider.id, provider.model, storedDimensions).count
+        : 0;
+    const missingChunks = Math.max(totalEligibleChunks - indexedChunks, 0);
+    return {
+        providerId: provider.id,
+        model: provider.model,
+        dimensions: storedDimensions,
+        totalEligibleChunks,
+        indexedChunks,
+        missingChunks,
+        lastIndexedAt,
+        incompatibleStoredVectors,
+        staleForConfiguredProvider: missingChunks > 0 || incompatibleStoredVectors > 0,
+        recommendation: incompatibleStoredVectors > 0
+            ? "Run `wwi embeddings clear` before switching embedding models or dimensions."
+            : missingChunks > 0
+                ? "Run `wwi index --semantic` to refresh embeddings for the configured provider."
+                : null
+    };
+}
+function latestEmbeddingIndexedAt(db, provider, dimensions) {
+    if (!tableExists(db, "vector_chunks_metadata")) {
+        return null;
+    }
+    if (!dimensions) {
+        return db
+            .prepare(`SELECT MAX(indexed_at) AS lastIndexedAt
+           FROM vector_chunks_metadata
+           WHERE provider_id = ? AND model = ?`)
+            .get(provider.id, provider.model).lastIndexedAt;
+    }
+    return db
+        .prepare(`SELECT MAX(indexed_at) AS lastIndexedAt
+         FROM vector_chunks_metadata
+         WHERE provider_id = ? AND model = ? AND dimensions = ?`)
+        .get(provider.id, provider.model, dimensions).lastIndexedAt;
+}
+function countIncompatibleStoredVectors(db, provider, dimensions) {
+    if (!tableExists(db, "vector_chunks_metadata")) {
+        return 0;
+    }
+    if (!dimensions) {
+        return db.prepare("SELECT COUNT(*) AS count FROM vector_chunks_metadata").get().count;
+    }
+    return db
+        .prepare(`SELECT COUNT(*) AS count
+         FROM vector_chunks_metadata
+         WHERE provider_id != ? OR model != ? OR dimensions != ?`)
+        .get(provider.id, provider.model, dimensions).count;
+}
+function assertNoIncompatibleStoredVectors(db, provider, dimensions) {
+    const incompatibleStoredVectors = countIncompatibleStoredVectors(db, provider, dimensions);
+    if (incompatibleStoredVectors > 0) {
+        throw new Error(`Found ${incompatibleStoredVectors} stored vector(s) for a different embedding model or dimension count. Run \`wwi embeddings clear\` and rebuild semantic embeddings before switching embedding settings.`);
+    }
+}
+function ensureVectorTable(db, dimensions) {
+    const existing = db
+        .prepare("SELECT sql FROM sqlite_master WHERE name = 'vector_chunks'")
+        .get();
+    if (existing) {
+        const expected = `float[${dimensions}]`;
+        if (!existing.sql.includes(expected)) {
+            throw new Error(`Existing vector_chunks table does not match ${expected}. Run \`wwi embeddings clear\` and rebuild semantic embeddings. Table SQL: ${existing.sql}`);
+        }
+        return;
+    }
+    db.exec(`CREATE VIRTUAL TABLE vector_chunks USING vec0(embedding float[${dimensions}])`);
+}
+function assertVectorTableDimensions(db, dimensions) {
+    const existingDimensions = vectorTableDimensions(db);
+    if (existingDimensions !== null && existingDimensions !== dimensions) {
+        throw new Error(`Existing vector store dimensions (${existingDimensions}) do not match configured embedding dimensions (${dimensions}). Run \`wwi embeddings clear\` and rebuild semantic embeddings.`);
+    }
+}
+function vectorTableDimensions(db) {
+    const existing = db
+        .prepare("SELECT sql FROM sqlite_master WHERE name = 'vector_chunks'")
+        .get();
+    const match = existing?.sql.match(/float\[(\d+)\]/);
+    return match ? Number(match[1]) : null;
+}
+function storedProviderDimensions(db, provider) {
+    if (!tableExists(db, "embedding_providers")) {
+        return null;
+    }
+    const row = db
+        .prepare("SELECT dimensions FROM embedding_providers WHERE id = ? AND model = ?")
+        .get(provider.id, provider.model);
+    return row?.dimensions ?? null;
+}
+function countEligibleChunks(db) {
+    const row = db
+        .prepare(`SELECT COUNT(*) AS count
+       FROM chunks
+       WHERE role IN ('user', 'assistant')
+        AND message_id IS NOT NULL
+        AND conversation_id IS NOT NULL
+        AND turn_id IS NOT NULL`)
+        .get();
+    return row.count;
+}
+function sourceFingerprint(chunk) {
+    return `${chunk.id}:${crypto.createHash("sha1").update(chunk.text).digest("hex")}`;
+}
+function tableExists(db, table) {
+    const row = db.prepare("SELECT 1 FROM sqlite_master WHERE name = ?").get(table);
+    return Boolean(row);
+}