opencode-diane 0.0.5

package/dist/search/inverted-index.d.ts

@@ -0,0 +1,51 @@
+/**
+ * In-memory inverted index plus auxiliary indexes used for hierarchical
+ * filtering. Rebuilt from the store on plugin startup; kept consistent
+ * by the repository on every CRUD.
+ */
+import type { Category, Memory } from "../types.js";
+/** Per-memory bookkeeping used by BM25 scoring. */
+interface DocInfo {
+    /** Token → term frequency. */
+    tf: Map<string, number>;
+    /** Total token count (used as |d| in BM25). */
+    length: number;
+}
+export declare class InvertedIndex {
+    /** token → set of memory ids that contain it */
+    readonly postings: Map<string, Set<string>>;
+    /** memory id → per-doc bookkeeping */
+    readonly docs: Map<string, DocInfo>;
+    /** category → set of memory ids */
+    readonly byCategory: Map<Category, Set<string>>;
+    /** subject → set of memory ids */
+    readonly bySubject: Map<string, Set<string>>;
+    /**
+     * Undirected file co-change graph: file path → set of file paths it
+     * was modified together with. Built from the `co-change` memories
+     * the git ingester produces (tags = ["co-change", fileA, fileB]).
+     * Used by BM25 search to propagate score to structurally-related
+     * files — the Aider PageRank idea, one hop, over edges we already
+     * compute. Survives restarts because co-change memories persist.
+     */
+    readonly coChange: Map<string, Set<string>>;
+    /** Running sum of doc lengths — for avgdl. */
+    totalLength: number;
+    /** Number of indexed docs. */
+    docCount: number;
+    rebuildFromAll(memories: Memory[]): void;
+    add(memory: Memory): void;
+    remove(memory: Memory): void;
+    avgDocLength(): number;
+    /** Files structurally coupled to `file` (one hop). Empty if none. */
+    coChangeNeighbors(file: string): ReadonlySet<string> | undefined;
+    /**
+     * A co-change memory carries tags `["co-change", fileA, fileB]`.
+     * If this memory is one, record the undirected edge. Anything else
+     * is ignored. Edges are de-duplicated by Set semantics, so the same
+     * pair appearing in multiple memories is harmless.
+     */
+    private indexCoChangeEdge;
+    private removeCoChangeEdge;
+}
+export {};

package/dist/search/inverted-index.js

@@ -0,0 +1,139 @@
+/**
+ * In-memory inverted index plus auxiliary indexes used for hierarchical
+ * filtering. Rebuilt from the store on plugin startup; kept consistent
+ * by the repository on every CRUD.
+ */
+import { termFreq, tokenize } from "./tokenize.js";
+export class InvertedIndex {
+    /** token → set of memory ids that contain it */
+    postings = new Map();
+    /** memory id → per-doc bookkeeping */
+    docs = new Map();
+    /** category → set of memory ids */
+    byCategory = new Map();
+    /** subject → set of memory ids */
+    bySubject = new Map();
+    /**
+     * Undirected file co-change graph: file path → set of file paths it
+     * was modified together with. Built from the `co-change` memories
+     * the git ingester produces (tags = ["co-change", fileA, fileB]).
+     * Used by BM25 search to propagate score to structurally-related
+     * files — the Aider PageRank idea, one hop, over edges we already
+     * compute. Survives restarts because co-change memories persist.
+     */
+    coChange = new Map();
+    /** Running sum of doc lengths — for avgdl. */
+    totalLength = 0;
+    /** Number of indexed docs. */
+    docCount = 0;
+    rebuildFromAll(memories) {
+        this.postings.clear();
+        this.docs.clear();
+        this.byCategory.clear();
+        this.bySubject.clear();
+        this.coChange.clear();
+        this.totalLength = 0;
+        this.docCount = 0;
+        for (const m of memories)
+            this.add(m);
+    }
+    add(memory) {
+        // Build searchable text from subject + content + tags
+        const text = `${memory.subject} ${memory.content} ${memory.tags.join(" ")}`;
+        const tokens = tokenize(text);
+        const tf = termFreq(tokens);
+        this.docs.set(memory.id, { tf, length: tokens.length });
+        this.totalLength += tokens.length;
+        this.docCount += 1;
+        for (const token of tf.keys()) {
+            let set = this.postings.get(token);
+            if (!set) {
+                set = new Set();
+                this.postings.set(token, set);
+            }
+            set.add(memory.id);
+        }
+        let catSet = this.byCategory.get(memory.category);
+        if (!catSet) {
+            catSet = new Set();
+            this.byCategory.set(memory.category, catSet);
+        }
+        catSet.add(memory.id);
+        let subSet = this.bySubject.get(memory.subject);
+        if (!subSet) {
+            subSet = new Set();
+            this.bySubject.set(memory.subject, subSet);
+        }
+        subSet.add(memory.id);
+        this.indexCoChangeEdge(memory);
+    }
+    remove(memory) {
+        const info = this.docs.get(memory.id);
+        if (info) {
+            this.totalLength -= info.length;
+            this.docCount -= 1;
+            for (const token of info.tf.keys()) {
+                const set = this.postings.get(token);
+                if (set) {
+                    set.delete(memory.id);
+                    if (set.size === 0)
+                        this.postings.delete(token);
+                }
+            }
+            this.docs.delete(memory.id);
+        }
+        this.byCategory.get(memory.category)?.delete(memory.id);
+        this.bySubject.get(memory.subject)?.delete(memory.id);
+        this.removeCoChangeEdge(memory);
+    }
+    avgDocLength() {
+        return this.docCount > 0 ? this.totalLength / this.docCount : 0;
+    }
+    /** Files structurally coupled to `file` (one hop). Empty if none. */
+    coChangeNeighbors(file) {
+        return this.coChange.get(file);
+    }
+    /**
+     * A co-change memory carries tags `["co-change", fileA, fileB]`.
+     * If this memory is one, record the undirected edge. Anything else
+     * is ignored. Edges are de-duplicated by Set semantics, so the same
+     * pair appearing in multiple memories is harmless.
+     */
+    indexCoChangeEdge(memory) {
+        const pair = coChangePair(memory);
+        if (!pair)
+            return;
+        const [a, b] = pair;
+        addEdge(this.coChange, a, b);
+        addEdge(this.coChange, b, a);
+    }
+    removeCoChangeEdge(memory) {
+        const pair = coChangePair(memory);
+        if (!pair)
+            return;
+        const [a, b] = pair;
+        this.coChange.get(a)?.delete(b);
+        this.coChange.get(b)?.delete(a);
+        if (this.coChange.get(a)?.size === 0)
+            this.coChange.delete(a);
+        if (this.coChange.get(b)?.size === 0)
+            this.coChange.delete(b);
+    }
+}
+function coChangePair(memory) {
+    if (!memory.tags.includes("co-change"))
+        return null;
+    // tags = ["co-change", fileA, fileB]
+    const files = memory.tags.filter((t) => t !== "co-change");
+    if (files.length < 2)
+        return null;
+    return [files[0], files[1]];
+}
+function addEdge(graph, from, to) {
+    let set = graph.get(from);
+    if (!set) {
+        set = new Set();
+        graph.set(from, set);
+    }
+    set.add(to);
+}

package/dist/search/ppr.d.ts

@@ -0,0 +1,44 @@
+/**
+ * ppr.ts — Personalized PageRank over the co-change graph.
+ *
+ * The default co-change boost (bm25.ts step 3b) is a single hop: a
+ * textual hit about file X lifts memories about X's *direct* co-change
+ * neighbours, and stops there. Personalized PageRank generalises that
+ * to the whole graph — a random walk with restart, where the restart
+ * (teleport) distribution is concentrated on the query's textual hits.
+ * Relevance then spreads across multiple hops and is graded by how
+ * reachable each file is from that seed set: a direct neighbour scores
+ * higher than a two-hop file, which still scores above an unrelated
+ * one.
+ *
+ * This is opt-in (`personalizedPageRank`, default off). When off,
+ * retrieval uses the cheaper one-hop boost and this file is never
+ * reached — so the default path keeps its O(seeds × neighbours) cost
+ * and its full inspectability.
+ *
+ * Deterministic: a fixed teleport probability, a fixed node-iteration
+ * order, and a fixed convergence tolerance with an iteration cap mean
+ * the same graph and personalization always yield the same scores.
+ */
+export interface PprOptions {
+    /** Teleport / restart probability. Default 0.15 (i.e. 0.85 damping). */
+    alpha?: number;
+    /** Hard cap on power-iteration steps. Default 60. */
+    maxIterations?: number;
+    /** L1 convergence threshold — stop once the score vector barely moves. Default 1e-7. */
+    tolerance?: number;
+    /** Safety valve: return empty (skip PPR) if the graph exceeds this node count. Default 100000. */
+    maxNodes?: number;
+}
+/**
+ * Personalized PageRank over an undirected, unweighted graph.
+ *
+ * @param graph           adjacency: node → set of neighbour nodes
+ * @param personalization restart distribution: node → weight. Weights
+ *                        need not be normalised; non-positive weights
+ *                        are ignored.
+ * @returns node → stationary score, summing to ~1. Empty when the
+ *          personalization carries no mass, the graph is empty, or the
+ *          graph exceeds `maxNodes`.
+ */
+export declare function personalizedPageRank(graph: ReadonlyMap<string, ReadonlySet<string>>, personalization: ReadonlyMap<string, number>, options?: PprOptions): Map<string, number>;

package/dist/search/ppr.js

@@ -0,0 +1,118 @@
+/**
+ * ppr.ts — Personalized PageRank over the co-change graph.
+ *
+ * The default co-change boost (bm25.ts step 3b) is a single hop: a
+ * textual hit about file X lifts memories about X's *direct* co-change
+ * neighbours, and stops there. Personalized PageRank generalises that
+ * to the whole graph — a random walk with restart, where the restart
+ * (teleport) distribution is concentrated on the query's textual hits.
+ * Relevance then spreads across multiple hops and is graded by how
+ * reachable each file is from that seed set: a direct neighbour scores
+ * higher than a two-hop file, which still scores above an unrelated
+ * one.
+ *
+ * This is opt-in (`personalizedPageRank`, default off). When off,
+ * retrieval uses the cheaper one-hop boost and this file is never
+ * reached — so the default path keeps its O(seeds × neighbours) cost
+ * and its full inspectability.
+ *
+ * Deterministic: a fixed teleport probability, a fixed node-iteration
+ * order, and a fixed convergence tolerance with an iteration cap mean
+ * the same graph and personalization always yield the same scores.
+ */
+/**
+ * Personalized PageRank over an undirected, unweighted graph.
+ *
+ * @param graph           adjacency: node → set of neighbour nodes
+ * @param personalization restart distribution: node → weight. Weights
+ *                        need not be normalised; non-positive weights
+ *                        are ignored.
+ * @returns node → stationary score, summing to ~1. Empty when the
+ *          personalization carries no mass, the graph is empty, or the
+ *          graph exceeds `maxNodes`.
+ */
+export function personalizedPageRank(graph, personalization, options = {}) {
+    const alpha = options.alpha ?? 0.15;
+    const maxIterations = options.maxIterations ?? 60;
+    const tolerance = options.tolerance ?? 1e-7;
+    const maxNodes = options.maxNodes ?? 100_000;
+    // ── node set: every graph node, plus every personalized node ─────
+    // A personalized file with no co-change history still belongs in the
+    // walk — it just becomes a dangling node holding its restart mass.
+    const idOf = new Map();
+    const nodes = [];
+    const intern = (name) => {
+        let i = idOf.get(name);
+        if (i === undefined) {
+            i = nodes.length;
+            idOf.set(name, i);
+            nodes.push(name);
+        }
+        return i;
+    };
+    for (const [node, neighbours] of graph) {
+        intern(node);
+        for (const nb of neighbours)
+            intern(nb);
+    }
+    for (const node of personalization.keys())
+        intern(node);
+    const n = nodes.length;
+    if (n === 0 || n > maxNodes)
+        return new Map();
+    // ── personalization vector p, normalised to sum 1 ────────────────
+    const p = new Float64Array(n);
+    let pTotal = 0;
+    for (const [node, weight] of personalization) {
+        if (weight > 0) {
+            p[idOf.get(node)] += weight;
+            pTotal += weight;
+        }
+    }
+    if (pTotal === 0)
+        return new Map(); // nothing to personalize toward
+    for (let i = 0; i < n; i++)
+        p[i] /= pTotal;
+    // ── adjacency as integer arrays (fixed, deterministic order) ─────
+    const adj = nodes.map(() => []);
+    for (const [node, neighbours] of graph) {
+        const j = idOf.get(node);
+        for (const nb of neighbours)
+            adj[j].push(idOf.get(nb));
+    }
+    // ── power iteration with dangling-mass redistribution ────────────
+    //   r_new[i] = (α + (1-α)·danglingMass)·p[i]  +  (1-α)·Σ_{j→i} r[j]/deg(j)
+    // A dangling node (no out-edges) would otherwise leak probability;
+    // sending its mass back through p keeps Σr = 1 every iteration.
+    let r = Float64Array.from(p);
+    let next = new Float64Array(n);
+    for (let iter = 0; iter < maxIterations; iter++) {
+        let dangling = 0;
+        for (let i = 0; i < n; i++)
+            if (adj[i].length === 0)
+                dangling += r[i];
+        const base = alpha + (1 - alpha) * dangling;
+        for (let i = 0; i < n; i++)
+            next[i] = base * p[i];
+        for (let j = 0; j < n; j++) {
+            const out = adj[j];
+            if (out.length === 0)
+                continue;
+            const share = ((1 - alpha) * r[j]) / out.length;
+            for (const i of out)
+                next[i] += share;
+        }
+        let delta = 0;
+        for (let i = 0; i < n; i++)
+            delta += Math.abs(next[i] - r[i]);
+        const swap = r;
+        r = next;
+        next = swap;
+        if (delta < tolerance)
+            break;
+    }
+    const result = new Map();
+    for (let i = 0; i < n; i++)
+        result.set(nodes[i], r[i]);
+    return result;
+}

package/dist/search/tokenize.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Deterministic tokenizer for memory search.
+ *
+ * Two scripts, two strategies, one pass:
+ *
+ *  - Latin / digit runs are split identifier-aware, so an agent's
+ *    camelCase / snake_case queries match well:
+ *      "AuthService.login_user" → ["authservice","auth","service","login","user"]
+ *
+ *  - CJK runs (Chinese, Japanese kana, Korean) are emitted as
+ *    overlapping bigrams:
+ *      "数据库连接" → ["数据","据库","库连","连接"]
+ *    CJK text has no word delimiters, so a whitespace/punctuation
+ *    splitter would drop it entirely. Bigrams give BM25 overlapping
+ *    units to match on — a query "数据库" → ["数据","据库"] overlaps a
+ *    stored "数据库连接池" — with no dictionary, model, or word
+ *    segmenter. This is the same approach Lucene's CJK analyzer and
+ *    SQLite FTS5 use; it is deterministic and dependency-free, which
+ *    is why it's preferred here over a statistical segmenter (whose
+ *    dictionary alone would blow the package size budget).
+ *
+ * Both indexing and querying call this function, so the two sides
+ * always agree regardless of script.
+ */
+export declare function tokenize(text: string): string[];
+export declare function termFreq(tokens: string[]): Map<string, number>;

package/dist/search/tokenize.js

@@ -0,0 +1,98 @@
+/**
+ * Deterministic tokenizer for memory search.
+ *
+ * Two scripts, two strategies, one pass:
+ *
+ *  - Latin / digit runs are split identifier-aware, so an agent's
+ *    camelCase / snake_case queries match well:
+ *      "AuthService.login_user" → ["authservice","auth","service","login","user"]
+ *
+ *  - CJK runs (Chinese, Japanese kana, Korean) are emitted as
+ *    overlapping bigrams:
+ *      "数据库连接" → ["数据","据库","库连","连接"]
+ *    CJK text has no word delimiters, so a whitespace/punctuation
+ *    splitter would drop it entirely. Bigrams give BM25 overlapping
+ *    units to match on — a query "数据库" → ["数据","据库"] overlaps a
+ *    stored "数据库连接池" — with no dictionary, model, or word
+ *    segmenter. This is the same approach Lucene's CJK analyzer and
+ *    SQLite FTS5 use; it is deterministic and dependency-free, which
+ *    is why it's preferred here over a statistical segmenter (whose
+ *    dictionary alone would blow the package size budget).
+ *
+ * Both indexing and querying call this function, so the two sides
+ * always agree regardless of script.
+ */
+// English stopwords. Not applied to CJK bigrams — there's no reliable
+// script-agnostic stopword notion for bigrams, and BM25's IDF already
+// down-weights ubiquitous bigrams without a hand-maintained list.
+const STOPWORDS = new Set([
+    "a", "an", "and", "are", "as", "at", "be", "but", "by", "for",
+    "from", "has", "have", "if", "in", "into", "is", "it", "its", "of",
+    "on", "or", "so", "such", "that", "the", "their", "then", "there",
+    "these", "they", "this", "to", "was", "were", "will", "with",
+    "we", "us", "you", "your", "i", "me", "my", "do", "does", "did",
+]);
+const MIN_LEN = 2;
+const MAX_LEN = 32;
+// CJK scripts handled as bigrams: Han (Chinese, Japanese kanji),
+// Hiragana + Katakana (Japanese), Hangul (Korean).
+const CJK_SCRIPTS = "\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}";
+// One pass: match either a CJK run OR an ASCII word run. Everything
+// else (spaces, punctuation, CJK punctuation) is a separator.
+const RUN_RE = new RegExp(`[${CJK_SCRIPTS}]+|[A-Za-z0-9_]+`, "gu");
+const CJK_RE = new RegExp(`[${CJK_SCRIPTS}]`, "u");
+export function tokenize(text) {
+    if (!text)
+        return [];
+    const out = [];
+    for (const match of text.matchAll(RUN_RE)) {
+        const run = match[0];
+        // ── CJK run → overlapping bigrams ───────────────────────────────
+        if (CJK_RE.test(run)) {
+            // Iterate code points (not UTF-16 units) so astral-plane
+            // ideographs (CJK Extension B+) bigram correctly.
+            const chars = [...run];
+            if (chars.length === 1) {
+                // A lone ideograph — rare, but keep it so a single-character
+                // query still matches. CJK tokens bypass the Latin MIN_LEN.
+                out.push(chars[0]);
+            }
+            else {
+                for (let i = 0; i < chars.length - 1; i++) {
+                    out.push(chars[i] + chars[i + 1]);
+                }
+            }
+            continue;
+        }
+        // ── Latin / digit run → identifier-aware splitting ──────────────
+        // camelCase split works because the run is not yet lowercased.
+        const camelParts = /[a-z][A-Z]/.test(run) ? run.split(/(?=[A-Z])/u) : [run];
+        for (const part of camelParts) {
+            const lower = part.toLowerCase();
+            const snakeParts = lower.includes("_") ? lower.split("_") : [lower];
+            for (const sp of snakeParts) {
+                if (sp.length < MIN_LEN || sp.length > MAX_LEN)
+                    continue;
+                if (STOPWORDS.has(sp))
+                    continue;
+                out.push(sp);
+            }
+        }
+        // Also keep the original (lowercased) full token, so exact-string
+        // queries like "authservice" still match documents with "AuthService".
+        const full = run.toLowerCase();
+        if (full.length >= MIN_LEN &&
+            full.length <= MAX_LEN &&
+            !STOPWORDS.has(full) &&
+            !out.includes(full)) {
+            out.push(full);
+        }
+    }
+    return out;
+}
+export function termFreq(tokens) {
+    const tf = new Map();
+    for (const t of tokens)
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    return tf;
+}

package/dist/store/eviction.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Eviction policy: least-frequently-used first (the user's
+ * explicit requirement), with least-recently-used as tiebreaker.
+ * Pinned entries are never evicted.
+ *
+ * Called after each ingest batch and after explicit agent writes.
+ *
+ * `currentTotalBytes` is supplied by the caller (the repository
+ * already tracks it incrementally) so this function does not
+ * recompute the sum, and — importantly — it evicts against the
+ * *same* number the repository reports from `totalBytes()`,
+ * including the fixed store overhead. Otherwise the effective
+ * budget would silently drift by that constant.
+ */
+import type { Memory } from "../types.js";
+export declare function evictIfOverBudget(memories: readonly Memory[], maxBytes: number, currentTotalBytes: number): Memory[];

package/dist/store/eviction.js

@@ -0,0 +1,37 @@
+/**
+ * Eviction policy: least-frequently-used first (the user's
+ * explicit requirement), with least-recently-used as tiebreaker.
+ * Pinned entries are never evicted.
+ *
+ * Called after each ingest batch and after explicit agent writes.
+ *
+ * `currentTotalBytes` is supplied by the caller (the repository
+ * already tracks it incrementally) so this function does not
+ * recompute the sum, and — importantly — it evicts against the
+ * *same* number the repository reports from `totalBytes()`,
+ * including the fixed store overhead. Otherwise the effective
+ * budget would silently drift by that constant.
+ */
+export function evictIfOverBudget(memories, maxBytes, currentTotalBytes) {
+    let total = currentTotalBytes;
+    if (total <= maxBytes)
+        return [];
+    // Eligible = not pinned. Sort ascending by (useCount, usedAt) so
+    // the first elements are the cheapest to lose.
+    const eligible = memories
+        .filter((m) => !m.pinned)
+        .slice()
+        .sort((a, b) => {
+        if (a.useCount !== b.useCount)
+            return a.useCount - b.useCount;
+        return a.usedAt - b.usedAt;
+    });
+    const removed = [];
+    for (const m of eligible) {
+        if (total <= maxBytes)
+            break;
+        removed.push(m);
+        total -= m.sizeBytes;
+    }
+    return removed;
+}