opencode-diane 0.0.5

package/dist/store/sqlite-store.d.ts

@@ -0,0 +1,89 @@
+/**
+ * SQLite-backed durable storage for the memory store.
+ *
+ * Replaces the old single-JSON-file persistence. The store lives at
+ * `.opencode/diane.db` — a real SQLite database in WAL mode.
+ *
+ * Why this exists: the JSON store rewrote the *entire* file on every
+ * debounced flush. On a large repo that is the dominant cost and the
+ * reason the plugin didn't scale. SQLite writes only the *changed*
+ * rows, in one transaction. That is the whole point of the migration.
+ *
+ * Scope, stated honestly: this is a durable *log*, not a query engine.
+ * The repository still keeps the working set in memory — the `byId`
+ * cache and the inverted index, which it needs anyway for the custom
+ * co-change-boosted BM25 scoring that FTS5 can't express. SQLite is
+ * touched in exactly two places: once at load (a single table scan)
+ * and at each flush (one delta transaction). Reads never hit it.
+ * Moving retrieval into SQLite (FTS5) would be a separate project.
+ *
+ * Uses `bun:sqlite` — built into the Bun runtime that OpenCode loads
+ * plugins under, so this adds no dependency.
+ */
+import type { Memory, MemoryStoreFile } from "../types.js";
+export declare function dbFilePath(root: string): string;
+/** What the repository gets back from a load. */
+export interface LoadedStore {
+    memories: Memory[];
+    meta: MemoryStoreFile["meta"];
+}
+export declare class SqliteStore {
+    private db;
+    private readonly upsertStmt;
+    private readonly deleteStmt;
+    private readonly metaUpsertStmt;
+    private constructor();
+    /**
+     * Open (or create) the store for a project root. On first open, if
+     * there is a legacy JSON store and no DB yet, the JSON is migrated
+     * into the fresh DB and renamed aside. Returns the store handle and
+     * everything in it (the repository reads this once at construction).
+     *
+     * A legacy-migration failure is reported via `onMigrationError`
+     * rather than thrown: the plugin's startup keeps going with an
+     * empty fresh database — losing memories is recoverable (the next
+     * open retries the migration); failing to start is not. See
+     * `migrateFromJson` for the rationale.
+     */
+    static open(root: string, log?: (msg: string) => void, onMigrationError?: (e: unknown) => void): {
+        store: SqliteStore;
+        loaded: LoadedStore;
+    };
+    /** Read the whole store back — called once, at repository construction. */
+    loadAll(): LoadedStore;
+    /**
+     * Persist a delta in ONE transaction: upsert every memory in
+     * `dirty`, delete every id in `deleted`, write `meta`. This is the
+     * incremental write that replaces the JSON whole-file rewrite —
+     * three changed memories cost three row writes, not a re-serialise
+     * of the entire store. A single transaction also means the flush is
+     * atomic: a crash mid-flush leaves the DB at the previous state, no
+     * temp-file-rename dance required.
+     */
+    flush(dirty: Iterable<Memory>, deleted: Iterable<string>, meta: MemoryStoreFile["meta"]): void;
+    /** Close the underlying database handle. */
+    close(): void;
+    /**
+     * One-time legacy migration: if a `diane.json` exists, load it,
+     * bulk-insert into the fresh DB in one transaction, and rename the
+     * JSON to `.json.migrated` so it is not re-migrated and the user
+     * keeps a backup.
+     *
+     * **Failure is not propagated.** A legacy-store migration is best-
+     * effort housekeeping; if it cannot complete for any reason (the
+     * JSON is corrupt, the database is held by another process, disk is
+     * full, a concurrent plugin's startup is blocking us) we MUST NOT
+     * crash the plugin's startup — that was the failure mode this code
+     * was rewritten to fix. Instead we log the cause, leave the JSON
+     * file untouched (the user keeps their data), and return 0 so the
+     * caller continues with an empty fresh database. The next open will
+     * find the JSON still in place and try the migration again.
+     *
+     * Returns the number of memories actually migrated, or 0 on any
+     * failure (including no JSON to migrate, which is also "0 migrated").
+     * `onError`, if provided, is called once on a real failure (not the
+     * "no JSON file" or "wrong schema version" cases) with the cause —
+     * the caller surfaces it as a structured event.
+     */
+    private migrateFromJson;
+}

package/dist/store/sqlite-store.js

@@ -0,0 +1,252 @@
+/**
+ * SQLite-backed durable storage for the memory store.
+ *
+ * Replaces the old single-JSON-file persistence. The store lives at
+ * `.opencode/diane.db` — a real SQLite database in WAL mode.
+ *
+ * Why this exists: the JSON store rewrote the *entire* file on every
+ * debounced flush. On a large repo that is the dominant cost and the
+ * reason the plugin didn't scale. SQLite writes only the *changed*
+ * rows, in one transaction. That is the whole point of the migration.
+ *
+ * Scope, stated honestly: this is a durable *log*, not a query engine.
+ * The repository still keeps the working set in memory — the `byId`
+ * cache and the inverted index, which it needs anyway for the custom
+ * co-change-boosted BM25 scoring that FTS5 can't express. SQLite is
+ * touched in exactly two places: once at load (a single table scan)
+ * and at each flush (one delta transaction). Reads never hit it.
+ * Moving retrieval into SQLite (FTS5) would be a separate project.
+ *
+ * Uses `bun:sqlite` — built into the Bun runtime that OpenCode loads
+ * plugins under, so this adds no dependency.
+ */
+import { Database } from "bun:sqlite";
+import { mkdirSync, existsSync, readFileSync, renameSync } from "node:fs";
+import { dirname } from "node:path";
+const DB_REL = ".opencode/diane.db";
+/** Legacy JSON store — migrated into SQLite on first open, then renamed aside. */
+const JSON_REL = ".opencode/diane.json";
+export function dbFilePath(root) {
+    return `${root}/${DB_REL}`;
+}
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS memories (
+  id          TEXT PRIMARY KEY,
+  category    TEXT NOT NULL,
+  subject     TEXT NOT NULL,
+  content     TEXT NOT NULL,
+  tags        TEXT NOT NULL,
+  source      TEXT NOT NULL,
+  created_at  INTEGER NOT NULL,
+  used_at     INTEGER NOT NULL,
+  use_count   INTEGER NOT NULL,
+  size_bytes  INTEGER NOT NULL,
+  pinned      INTEGER NOT NULL DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+`;
+function rowToMemory(r) {
+    return {
+        id: r.id,
+        category: r.category,
+        subject: r.subject,
+        content: r.content,
+        tags: JSON.parse(r.tags),
+        source: r.source,
+        createdAt: r.created_at,
+        usedAt: r.used_at,
+        useCount: r.use_count,
+        sizeBytes: r.size_bytes,
+        // pinned is optional on Memory — only set it when actually pinned,
+        // matching how the rest of the codebase treats the field.
+        pinned: r.pinned === 1 ? true : undefined,
+    };
+}
+function emptyMeta() {
+    return { ingestedAt: {}, lastEvictionAt: null, schema: 1 };
+}
+export class SqliteStore {
+    db;
+    // Prepared statements — created once, reused for every flush. This
+    // is the other half of "use SQLite well": the query planner runs
+    // once per statement, not once per row.
+    upsertStmt;
+    deleteStmt;
+    metaUpsertStmt;
+    constructor(db) {
+        this.db = db;
+        // WAL: concurrent-read friendly and the right default for a store
+        // two OpenCode sessions might touch at once. NORMAL synchronous is
+        // safe under WAL and much faster than FULL.
+        db.exec("PRAGMA journal_mode = WAL");
+        db.exec("PRAGMA synchronous = NORMAL");
+        db.exec(SCHEMA);
+        this.upsertStmt = db.query(`INSERT INTO memories
+         (id, category, subject, content, tags, source,
+          created_at, used_at, use_count, size_bytes, pinned)
+       VALUES
+         ($id, $category, $subject, $content, $tags, $source,
+          $createdAt, $usedAt, $useCount, $sizeBytes, $pinned)
+       ON CONFLICT(id) DO UPDATE SET
+         category   = excluded.category,
+         subject    = excluded.subject,
+         content    = excluded.content,
+         tags       = excluded.tags,
+         source     = excluded.source,
+         created_at = excluded.created_at,
+         used_at    = excluded.used_at,
+         use_count  = excluded.use_count,
+         size_bytes = excluded.size_bytes,
+         pinned     = excluded.pinned`);
+        this.deleteStmt = db.query(`DELETE FROM memories WHERE id = $id`);
+        this.metaUpsertStmt = db.query(`INSERT INTO meta (key, value) VALUES ($key, $value)
+       ON CONFLICT(key) DO UPDATE SET value = excluded.value`);
+    }
+    /**
+     * Open (or create) the store for a project root. On first open, if
+     * there is a legacy JSON store and no DB yet, the JSON is migrated
+     * into the fresh DB and renamed aside. Returns the store handle and
+     * everything in it (the repository reads this once at construction).
+     *
+     * A legacy-migration failure is reported via `onMigrationError`
+     * rather than thrown: the plugin's startup keeps going with an
+     * empty fresh database — losing memories is recoverable (the next
+     * open retries the migration); failing to start is not. See
+     * `migrateFromJson` for the rationale.
+     */
+    static open(root, log, onMigrationError) {
+        const dbPath = dbFilePath(root);
+        mkdirSync(dirname(dbPath), { recursive: true });
+        const dbExisted = existsSync(dbPath);
+        const db = new Database(dbPath, { create: true });
+        const store = new SqliteStore(db);
+        if (!dbExisted) {
+            const migrated = store.migrateFromJson(root, onMigrationError);
+            if (migrated > 0 && log) {
+                log(`migrated ${migrated} memories from legacy diane.json`);
+            }
+        }
+        return { store, loaded: store.loadAll() };
+    }
+    /** Read the whole store back — called once, at repository construction. */
+    loadAll() {
+        const rows = this.db.query(`SELECT * FROM memories`).all();
+        const memories = rows.map(rowToMemory);
+        const meta = emptyMeta();
+        const metaRows = this.db.query(`SELECT key, value FROM meta`).all();
+        for (const { key, value } of metaRows) {
+            try {
+                if (key === "ingestedAt")
+                    meta.ingestedAt = JSON.parse(value);
+                else if (key === "lastEvictionAt")
+                    meta.lastEvictionAt = JSON.parse(value);
+                else if (key === "schema")
+                    meta.schema = JSON.parse(value);
+            }
+            catch {
+                /* a corrupt meta row falls back to the default — non-fatal */
+            }
+        }
+        return { memories, meta };
+    }
+    /**
+     * Persist a delta in ONE transaction: upsert every memory in
+     * `dirty`, delete every id in `deleted`, write `meta`. This is the
+     * incremental write that replaces the JSON whole-file rewrite —
+     * three changed memories cost three row writes, not a re-serialise
+     * of the entire store. A single transaction also means the flush is
+     * atomic: a crash mid-flush leaves the DB at the previous state, no
+     * temp-file-rename dance required.
+     */
+    flush(dirty, deleted, meta) {
+        const run = this.db.transaction(() => {
+            for (const m of dirty) {
+                this.upsertStmt.run({
+                    $id: m.id,
+                    $category: m.category,
+                    $subject: m.subject,
+                    $content: m.content,
+                    $tags: JSON.stringify(m.tags),
+                    $source: m.source,
+                    $createdAt: m.createdAt,
+                    $usedAt: m.usedAt,
+                    $useCount: m.useCount,
+                    $sizeBytes: m.sizeBytes,
+                    $pinned: m.pinned ? 1 : 0,
+                });
+            }
+            for (const id of deleted) {
+                this.deleteStmt.run({ $id: id });
+            }
+            this.metaUpsertStmt.run({ $key: "ingestedAt", $value: JSON.stringify(meta.ingestedAt) });
+            this.metaUpsertStmt.run({ $key: "lastEvictionAt", $value: JSON.stringify(meta.lastEvictionAt) });
+            this.metaUpsertStmt.run({ $key: "schema", $value: JSON.stringify(meta.schema) });
+        });
+        run();
+    }
+    /** Close the underlying database handle. */
+    close() {
+        this.db.close();
+    }
+    /**
+     * One-time legacy migration: if a `diane.json` exists, load it,
+     * bulk-insert into the fresh DB in one transaction, and rename the
+     * JSON to `.json.migrated` so it is not re-migrated and the user
+     * keeps a backup.
+     *
+     * **Failure is not propagated.** A legacy-store migration is best-
+     * effort housekeeping; if it cannot complete for any reason (the
+     * JSON is corrupt, the database is held by another process, disk is
+     * full, a concurrent plugin's startup is blocking us) we MUST NOT
+     * crash the plugin's startup — that was the failure mode this code
+     * was rewritten to fix. Instead we log the cause, leave the JSON
+     * file untouched (the user keeps their data), and return 0 so the
+     * caller continues with an empty fresh database. The next open will
+     * find the JSON still in place and try the migration again.
+     *
+     * Returns the number of memories actually migrated, or 0 on any
+     * failure (including no JSON to migrate, which is also "0 migrated").
+     * `onError`, if provided, is called once on a real failure (not the
+     * "no JSON file" or "wrong schema version" cases) with the cause —
+     * the caller surfaces it as a structured event.
+     */
+    migrateFromJson(root, onError) {
+        const jsonPath = `${root}/${JSON_REL}`;
+        if (!existsSync(jsonPath))
+            return 0;
+        let parsed;
+        try {
+            parsed = JSON.parse(readFileSync(jsonPath, "utf-8"));
+        }
+        catch {
+            return 0; // corrupt JSON — start fresh, leave the file untouched
+        }
+        if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.memories)) {
+            return 0;
+        }
+        const meta = parsed.meta ?? emptyMeta();
+        try {
+            this.flush(parsed.memories, [], meta);
+        }
+        catch (e) {
+            // The bulk insert failed mid-way. The SQLite transaction is
+            // rolled back automatically; the JSON file is still in place;
+            // we report the cause and leave the database in its empty
+            // freshly-created state. The plugin's startup continues.
+            if (onError)
+                onError(e);
+            return 0;
+        }
+        try {
+            renameSync(jsonPath, `${jsonPath}.migrated`);
+        }
+        catch {
+            /* best effort — if the rename fails the DB now exists, so the
+               next open won't re-migrate anyway */
+        }
+        return parsed.memories.length;
+    }
+}

package/dist/store/vector-store.d.ts

@@ -0,0 +1,66 @@
+/**
+ * vector-store.ts — persistence and search for memory embeddings.
+ *
+ * This is a SELF-CONTAINED, OPT-IN component. It is constructed only
+ * when `enableSemanticSearch` is on, and it uses its OWN database file
+ * (`.opencode/diane-vectors.db`) — the primary store, its schema, and
+ * its migration path are never touched. When the feature is off this
+ * file is never imported and the file never created, so the default
+ * configuration carries zero cost from semantic search.
+ *
+ * The cache is keyed to a model id. If the configured embedding model
+ * changes, the stored vectors are from a different space and are
+ * dropped wholesale on open — a stale vector is worse than none.
+ *
+ * Vectors are L2-normalised on the way in, so similarity search is a
+ * plain dot product.
+ */
+import { type FusedItem } from "../search/embedder.js";
+/** Absolute path of the vector database for a project root. */
+export declare function vectorDbPath(root: string): string;
+export declare class VectorStore {
+    private db;
+    /** In-memory mirror — every stored vector, for brute-force search. */
+    private mem;
+    /** Vector dimension, learned from the first vector stored. 0 until then. */
+    private dim;
+    readonly modelId: string;
+    private constructor();
+    /**
+     * Open (or create) the vector store for a project root, bound to a
+     * model id. If a different model produced the existing vectors, they
+     * are discarded — vectors from two models are not comparable.
+     */
+    static open(root: string, modelId: string): VectorStore;
+    /** Load every persisted vector into the in-memory mirror. */
+    private loadAll;
+    /** Number of vectors held. */
+    size(): number;
+    /** Whether a memory id already has a stored vector. */
+    has(id: string): boolean;
+    /** The memory ids in `ids` that do NOT yet have a vector — the embedding to-do list. */
+    missing(ids: Iterable<string>): string[];
+    /**
+     * Store a batch of (id, vector) pairs — normalised, mirrored in
+     * memory, and persisted in one transaction. A vector whose length
+     * disagrees with the established dimension is skipped (it cannot
+     * have come from the same model) rather than corrupting search.
+     */
+    putMany(entries: Array<{
+        id: string;
+        vec: Float32Array;
+    }>): void;
+    /**
+     * Drop vectors whose id is not in `validIds` — used to clear out
+     * memories that were evicted or replaced. Returns the count removed.
+     */
+    prune(validIds: Set<string>): number;
+    /**
+     * Top-`k` memory ids by cosine similarity to `queryVec` (vectors are
+     * normalised, so this is a dot product), highest first. A
+     * dimension mismatch yields an empty result rather than a throw.
+     */
+    search(queryVec: Float32Array, k: number): FusedItem[];
+    /** Close the underlying database handle. */
+    close(): void;
+}

package/dist/store/vector-store.js

@@ -0,0 +1,160 @@
+/**
+ * vector-store.ts — persistence and search for memory embeddings.
+ *
+ * This is a SELF-CONTAINED, OPT-IN component. It is constructed only
+ * when `enableSemanticSearch` is on, and it uses its OWN database file
+ * (`.opencode/diane-vectors.db`) — the primary store, its schema, and
+ * its migration path are never touched. When the feature is off this
+ * file is never imported and the file never created, so the default
+ * configuration carries zero cost from semantic search.
+ *
+ * The cache is keyed to a model id. If the configured embedding model
+ * changes, the stored vectors are from a different space and are
+ * dropped wholesale on open — a stale vector is worse than none.
+ *
+ * Vectors are L2-normalised on the way in, so similarity search is a
+ * plain dot product.
+ */
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { dot, normalize } from "../search/embedder.js";
+const DB_REL = ".opencode/diane-vectors.db";
+/** Absolute path of the vector database for a project root. */
+export function vectorDbPath(root) {
+    return join(root, DB_REL);
+}
+export class VectorStore {
+    db;
+    /** In-memory mirror — every stored vector, for brute-force search. */
+    mem = new Map();
+    /** Vector dimension, learned from the first vector stored. 0 until then. */
+    dim = 0;
+    modelId;
+    constructor(db, modelId) {
+        this.db = db;
+        this.modelId = modelId;
+    }
+    /**
+     * Open (or create) the vector store for a project root, bound to a
+     * model id. If a different model produced the existing vectors, they
+     * are discarded — vectors from two models are not comparable.
+     */
+    static open(root, modelId) {
+        const path = vectorDbPath(root);
+        mkdirSync(dirname(path), { recursive: true });
+        const db = new Database(path, { create: true });
+        db.exec("PRAGMA journal_mode = WAL");
+        db.exec("PRAGMA synchronous = NORMAL");
+        db.exec("CREATE TABLE IF NOT EXISTS vectors (memory_id TEXT PRIMARY KEY, vec BLOB NOT NULL)");
+        db.exec("CREATE TABLE IF NOT EXISTS vmeta (key TEXT PRIMARY KEY, value TEXT NOT NULL)");
+        const storedModel = db.query("SELECT value FROM vmeta WHERE key = 'model'").get()?.value;
+        if (storedModel && storedModel !== modelId) {
+            // Model changed — the cached vectors live in a different space.
+            db.exec("DELETE FROM vectors");
+        }
+        db.query("INSERT OR REPLACE INTO vmeta (key, value) VALUES ('model', ?)").run(modelId);
+        const store = new VectorStore(db, modelId);
+        store.loadAll();
+        return store;
+    }
+    /** Load every persisted vector into the in-memory mirror. */
+    loadAll() {
+        const rows = this.db.query("SELECT memory_id, vec FROM vectors").all();
+        for (const row of rows) {
+            const f32 = bytesToFloat32(row.vec);
+            if (this.dim === 0)
+                this.dim = f32.length;
+            if (f32.length === this.dim)
+                this.mem.set(row.memory_id, f32);
+        }
+    }
+    /** Number of vectors held. */
+    size() {
+        return this.mem.size;
+    }
+    /** Whether a memory id already has a stored vector. */
+    has(id) {
+        return this.mem.has(id);
+    }
+    /** The memory ids in `ids` that do NOT yet have a vector — the embedding to-do list. */
+    missing(ids) {
+        const out = [];
+        for (const id of ids)
+            if (!this.mem.has(id))
+                out.push(id);
+        return out;
+    }
+    /**
+     * Store a batch of (id, vector) pairs — normalised, mirrored in
+     * memory, and persisted in one transaction. A vector whose length
+     * disagrees with the established dimension is skipped (it cannot
+     * have come from the same model) rather than corrupting search.
+     */
+    putMany(entries) {
+        const insert = this.db.query("INSERT OR REPLACE INTO vectors (memory_id, vec) VALUES (?, ?)");
+        const tx = this.db.transaction((batch) => {
+            for (const { id, vec } of batch) {
+                if (this.dim === 0)
+                    this.dim = vec.length;
+                if (vec.length !== this.dim)
+                    continue;
+                const n = normalize(vec);
+                this.mem.set(id, n);
+                insert.run(id, float32ToBytes(n));
+            }
+        });
+        tx(entries);
+    }
+    /**
+     * Drop vectors whose id is not in `validIds` — used to clear out
+     * memories that were evicted or replaced. Returns the count removed.
+     */
+    prune(validIds) {
+        const stale = [];
+        for (const id of this.mem.keys())
+            if (!validIds.has(id))
+                stale.push(id);
+        if (stale.length === 0)
+            return 0;
+        const del = this.db.query("DELETE FROM vectors WHERE memory_id = ?");
+        const tx = this.db.transaction((ids) => {
+            for (const id of ids) {
+                del.run(id);
+                this.mem.delete(id);
+            }
+        });
+        tx(stale);
+        return stale.length;
+    }
+    /**
+     * Top-`k` memory ids by cosine similarity to `queryVec` (vectors are
+     * normalised, so this is a dot product), highest first. A
+     * dimension mismatch yields an empty result rather than a throw.
+     */
+    search(queryVec, k) {
+        if (this.dim === 0 || queryVec.length !== this.dim || k <= 0)
+            return [];
+        const q = normalize(Float32Array.from(queryVec));
+        const scored = [];
+        for (const [id, vec] of this.mem) {
+            scored.push({ id, score: dot(q, vec) });
+        }
+        scored.sort((a, b) => b.score - a.score);
+        return scored.slice(0, k);
+    }
+    /** Close the underlying database handle. */
+    close() {
+        this.db.close();
+    }
+}
+/** Float32Array → a Buffer of its raw little-endian bytes for BLOB storage. */
+function float32ToBytes(v) {
+    return new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
+}
+/** Raw BLOB bytes → Float32Array. Copies, so alignment is never an issue. */
+function bytesToFloat32(bytes) {
+    const copy = new Uint8Array(bytes.length);
+    copy.set(bytes);
+    return new Float32Array(copy.buffer);
+}