@claude-flow/cli 3.32.3 → 3.32.5

package/dist/src/memory/graph-edge-writer.js

@@ -0,0 +1,217 @@
+/**
+ * Graph Edge Writer — ADR-130 Phase 1
+ *
+ * Provides a minimal interface for inserting rows into the graph_edges
+ * SQLite table defined by MEMORY_SCHEMA_V3.
+ *
+ * #2431 fix (2026-06-22) — replaces the prior sql.js implementation. The
+ * prior version opened sql.js, performed in-memory updates, then called
+ * `fs.writeFileSync(dbPath, db.export())` after every edge insert. This
+ * whole-file flush overwrote the main `memory.db` while the better-sqlite3
+ * bridge was actively writing through its WAL — exactly the dual-write
+ * race that ADR-068 (#1257) removed. Symptom: PRAGMA integrity_check
+ * reports `database disk image is malformed (11)` after a single
+ * memory_store + causal-edge sequence.
+ *
+ * Fix posture: use better-sqlite3 directly (the same native engine the
+ * memory bridge uses). WAL-native, no whole-file fsync, no race. Keeps
+ * the public API surface identical so callers don't have to change.
+ *
+ * Note: this is the minimum-safe fix. The architecturally cleaner fix
+ * (route writes through the bridge's controller layer) is scoped to a
+ * future ADR — see #2431 for the discussion. Until that ADR lands, this
+ * module owns its own better-sqlite3 handle but on the same file, with
+ * WAL mode enabled (which makes concurrent writers safe by SQLite's own
+ * design — no overlapping fsync).
+ *
+ * The module is designed for fire-and-forget callers — every public
+ * function suppresses errors internally so callers never need try/catch.
+ *
+ * @module v3/cli/memory/graph-edge-writer
+ */
+import * as path from 'path';
+import * as fs from 'fs';
+import * as crypto from 'crypto';
+import { getMemoryRoot } from './memory-initializer.js';
+import { encodeEmbedding } from './embedding-quantization.js';
+// ============================================================================
+// Lazy-cached better-sqlite3 db handle
+// ============================================================================
+let _db = null;
+let _dbPath = '';
+let _dbInitializing = false;
+/**
+ * Return the better-sqlite3 Database instance for graph_edges writes.
+ * Creates the graph_edges table if it is absent (idempotent).
+ * Returns null if better-sqlite3 is not available or db cannot be opened.
+ *
+ * #2246 fix: `createIfMissing` (default false for back-compat) — when true,
+ * lazily creates an empty memory.db with the graph_edges schema so
+ * graph-pathfinder works on fresh environments before any memory writes.
+ *
+ * #2431 fix: better-sqlite3 + WAL mode replaces sql.js + whole-file
+ * writeFileSync. Eliminates the dual-write race that corrupted memory.db
+ * when called alongside the memory bridge's better-sqlite3 writer.
+ */
+export async function getBridgeDb(customDbPath, opts) {
+    const dbPath = customDbPath ?? path.join(getMemoryRoot(), 'memory.db');
+    const createIfMissing = opts?.createIfMissing === true;
+    if (_db && _dbPath === dbPath)
+        return _db;
+    if (_dbInitializing)
+        return null;
+    _dbInitializing = true;
+    try {
+        const dbExists = fs.existsSync(dbPath);
+        if (!dbExists && !createIfMissing)
+            return null;
+        // Ensure parent dir exists for createIfMissing case.
+        if (!dbExists && createIfMissing) {
+            const dir = path.dirname(dbPath);
+            if (!fs.existsSync(dir))
+                fs.mkdirSync(dir, { recursive: true });
+        }
+        // better-sqlite3 may not be available on all platforms (it's a native
+        // module, so platform-specific binaries are required). If unavailable,
+        // return null and let callers degrade gracefully — same posture as the
+        // prior sql.js version when sql.js failed to load.
+        //
+        // The module name is hidden behind a variable so the TypeScript compiler
+        // does not statically resolve and require the `better-sqlite3` types at
+        // build time — they would only be installed via optionalDependencies,
+        // which CI doesn't always install. This is the standard pattern for
+        // runtime-only optional native deps. The actual presence is gated by
+        // the try/catch below.
+        let BetterSqlite3;
+        try {
+            const mod = 'better-sqlite3';
+            BetterSqlite3 = (await import(mod)).default;
+        }
+        catch {
+            return null;
+        }
+        const db = new BetterSqlite3(dbPath);
+        // WAL mode is the load-bearing piece of the #2431 fix: it lets multiple
+        // connections (this module + the memory bridge) write to the same file
+        // without overlapping fsyncs corrupting each other. SQLite's WAL is
+        // designed for exactly this case.
+        db.pragma('journal_mode = WAL');
+        db.pragma('synchronous = NORMAL'); // WAL-safe + faster than FULL
+        db.pragma('busy_timeout = 5000'); // wait up to 5s on lock contention
+        // Ensure graph_edges table exists (in case this is an older DB that
+        // predates ADR-130 Phase 1 schema migration).
+        db.exec(`
+      CREATE TABLE IF NOT EXISTS graph_edges (
+        id              TEXT PRIMARY KEY,
+        source_id       TEXT NOT NULL,
+        target_id       TEXT NOT NULL,
+        relation        TEXT NOT NULL,
+        weight          REAL DEFAULT 1.0,
+        confidence      REAL DEFAULT 1.0,
+        decay_rate      REAL DEFAULT 0.0,
+        last_reinforced TEXT,
+        witness_id      TEXT,
+        embedding_ref   TEXT,
+        metadata        TEXT,
+        created_at      TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_source ON graph_edges (source_id);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_target ON graph_edges (target_id);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_relation ON graph_edges (relation);
+      CREATE INDEX IF NOT EXISTS idx_graph_edges_reinforced ON graph_edges (last_reinforced);
+    `);
+        _db = db;
+        _dbPath = dbPath;
+        return db;
+    }
+    catch {
+        return null;
+    }
+    finally {
+        _dbInitializing = false;
+    }
+}
+/**
+ * Insert a single edge into graph_edges.
+ * Fire-and-forget — errors are suppressed.
+ * Returns true if the write succeeded, false otherwise.
+ *
+ * #2431 fix: uses better-sqlite3 prepared statements + implicit WAL
+ * journal. No `fs.writeFileSync` whole-file flush — the WAL handles
+ * durability without overwriting the main file out from under other
+ * writers.
+ */
+export async function insertGraphEdge(input) {
+    try {
+        const db = await getBridgeDb(input.dbPath);
+        if (!db)
+            return false;
+        const id = `edge-${crypto.randomUUID()}`;
+        const createdAt = new Date().toISOString();
+        let embeddingRef = null;
+        if (input.embedding && input.embedding.length > 0) {
+            embeddingRef = encodeEmbedding(input.embedding);
+        }
+        const metaStr = input.metadata ? JSON.stringify(input.metadata) : null;
+        db.prepare(`INSERT OR IGNORE INTO graph_edges
+        (id, source_id, target_id, relation, weight, confidence, decay_rate,
+         last_reinforced, witness_id, embedding_ref, metadata, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, input.sourceId, input.targetId, input.relation, input.weight ?? 1.0, input.confidence ?? 1.0, input.decayRate ?? 0.0, input.lastReinforced ?? null, input.witnessId ?? null, embeddingRef, metaStr, createdAt);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Query graph_edges by source_id.
+ * Returns rows or empty array on error.
+ */
+export async function queryEdgesBySource(sourceId, relation, dbPath) {
+    try {
+        const db = await getBridgeDb(dbPath);
+        if (!db)
+            return [];
+        const sql = relation
+            ? `SELECT id, source_id, target_id, relation, weight FROM graph_edges WHERE source_id = ? AND relation = ? LIMIT 1000`
+            : `SELECT id, source_id, target_id, relation, weight FROM graph_edges WHERE source_id = ? LIMIT 1000`;
+        const args = relation ? [sourceId, relation] : [sourceId];
+        return db.prepare(sql).all(...args);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Count rows in graph_edges (for test assertions).
+ */
+export async function countGraphEdges(dbPath) {
+    try {
+        const db = await getBridgeDb(dbPath);
+        if (!db)
+            return 0;
+        const row = db.prepare(`SELECT COUNT(*) AS n FROM graph_edges`).get();
+        return row?.n ?? 0;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Reset the cached db handle (for tests that need a fresh DB).
+ *
+ * #2431 fix: also explicitly closes the prior handle so file locks
+ * release immediately — better-sqlite3 holds an OS-level file handle
+ * which the prior sql.js implementation did not.
+ */
+export function _resetBridgeDb() {
+    if (_db) {
+        try {
+            _db.close();
+        }
+        catch { /* best-effort */ }
+    }
+    _db = null;
+    _dbPath = '';
+}
+//# sourceMappingURL=graph-edge-writer.js.map

package/dist/src/memory/hybrid-retrieval.d.ts

@@ -0,0 +1,77 @@
+/** Tokenise text into lowercase content tokens (drops stopwords + length<3). */
+export declare function tokenize(text: string): string[];
+export interface CorpusStats {
+    /** docFreq: how many documents contain each token */
+    df: Map<string, number>;
+    /** idf: log((N - df + 0.5) / (df + 0.5) + 1) — BM25's smoothed IDF */
+    idf: Map<string, number>;
+    /** averageDocumentLength in tokens */
+    avgDocLen: number;
+    /** total docs in corpus */
+    N: number;
+}
+/** Build BM25 corpus statistics from a set of tokenised documents. */
+export declare function buildCorpusStats(tokenisedDocs: string[][]): CorpusStats;
+/**
+ * Type penalty for meta-commits — multiplies the hybrid score by `factor`
+ * when the document's name matches the meta-commit regex (release bumps,
+ * merge commits, badge updates, etc.). These commits bundle every issue
+ * number from a release window and steal top-1 from real work commits.
+ *
+ * Defaults: factor 0.5, regex covers `chore(release)`, `Merge `, `bump `,
+ * `[Dream Cycle]`, `publish 3.x.y`. Tune via env or call site.
+ */
+export declare const META_COMMIT_REGEX: RegExp;
+export declare function typePenalty(name: string | undefined, factor?: number, regex?: RegExp): number;
+/**
+ * BM25 score of one document against a query.
+ * Standard Okapi formula with k1=1.5 and b=0.75.
+ */
+export declare function bm25Score(queryTokens: string[], docTokens: string[], stats: CorpusStats, k1?: number, b?: number): number;
+/**
+ * Min-max normalise a score vector to [0, 1].
+ * Returns the original vector if it is constant (avoid divide-by-zero).
+ */
+export declare function normalise(scores: number[]): number[];
+/**
+ * Combine cosine and BM25 scores. Both vectors must be aligned by docIndex.
+ * Returns hybridScores[] aligned to the same index.
+ * alpha controls the cosine weight; (1-alpha) is the BM25 weight.
+ */
+export declare function hybridScores(cosine: number[], bm25: number[], alpha?: number): number[];
+/**
+ * Multi-field BM25 score — treats subject and body as separate fields with
+ * independent token frequencies, then combines `subjectWeight * subjectBM25
+ * + bodyWeight * bodyBM25`. Subject (commit title / pattern name) carries
+ * the high-signal tokens (file names, action verbs, ADR refs); body is
+ * often boilerplate. Default 3:1 weight reflects that asymmetry.
+ *
+ * Caller must build separate CorpusStats for subjects and bodies (their
+ * IDF distributions differ — subjects are short, bodies long).
+ */
+export declare function multiFieldBM25(queryTokens: string[], subjectTokens: string[], bodyTokens: string[], subjectStats: CorpusStats, bodyStats: CorpusStats, subjectWeight?: number, bodyWeight?: number): number;
+/**
+ * Cosine similarity between two equal-length numeric vectors.
+ * Returns 0 if either has zero norm.
+ */
+export declare function cosineSim(a: number[], b: number[]): number;
+/**
+ * Maximal Marginal Relevance rerank.
+ *
+ * Greedy selection: at each step pick the candidate that maximises
+ *   lambda * relevance(c) - (1-lambda) * max(similarity(c, picked))
+ *
+ * lambda=1.0 → pure relevance (no diversity adjustment)
+ * lambda=0.0 → pure diversity
+ *
+ * Default lambda=0.5 balances both.
+ */
+export declare function mmrRerank<T extends {
+    embedding: number[];
+}>(candidates: Array<T & {
+    relevance: number;
+}>, k: number, lambda?: number): Array<T & {
+    relevance: number;
+    mmrScore: number;
+}>;
+//# sourceMappingURL=hybrid-retrieval.d.ts.map

package/dist/src/memory/hybrid-retrieval.js

@@ -0,0 +1,192 @@
+// Hybrid retrieval — sparse (BM25) + dense (cosine) + MMR diversity.
+//
+// Why: on small corpora (N<1k) a bi-encoder like all-MiniLM-L6-v2 produces
+// noisy cosine scores — random short commits with low-IDF token overlap can
+// outrank exact-keyword matches. BM25 over the same text recovers the
+// "did the query's content tokens actually appear?" signal. We linearly
+// combine the two on normalised scores, then MMR-rerank to suppress
+// near-duplicate top-K.
+//
+// Pure functions, no external deps. Tested via __tests__/hybrid-retrieval.test.ts
+// and the live A/B in scripts/benchmark-pretrained-retrieval.mjs (HYBRID=0|1).
+const STOPWORDS = new Set([
+    'the', 'a', 'an', 'and', 'or', 'but', 'if', 'then', 'else', 'of', 'in', 'to', 'for', 'on', 'at',
+    'by', 'with', 'from', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
+    'do', 'does', 'did', 'will', 'would', 'should', 'could', 'can', 'may', 'might', 'must', 'this',
+    'that', 'these', 'those', 'it', 'its', 'as', 'also', 'not', 'no', 'so', 'too', 'very',
+]);
+/** Tokenise text into lowercase content tokens (drops stopwords + length<3). */
+export function tokenize(text) {
+    if (!text)
+        return [];
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9_\-/.]+/g)
+        .filter((t) => t.length >= 3 && !STOPWORDS.has(t));
+}
+/** Build BM25 corpus statistics from a set of tokenised documents. */
+export function buildCorpusStats(tokenisedDocs) {
+    const N = tokenisedDocs.length;
+    const df = new Map();
+    let totalLen = 0;
+    for (const doc of tokenisedDocs) {
+        totalLen += doc.length;
+        const seen = new Set();
+        for (const tok of doc) {
+            if (seen.has(tok))
+                continue;
+            seen.add(tok);
+            df.set(tok, (df.get(tok) ?? 0) + 1);
+        }
+    }
+    const idf = new Map();
+    for (const [tok, dfVal] of df) {
+        // BM25 smoothed IDF — never negative, always > 0
+        idf.set(tok, Math.log(1 + (N - dfVal + 0.5) / (dfVal + 0.5)));
+    }
+    return { df, idf, avgDocLen: N > 0 ? totalLen / N : 0, N };
+}
+/**
+ * Type penalty for meta-commits — multiplies the hybrid score by `factor`
+ * when the document's name matches the meta-commit regex (release bumps,
+ * merge commits, badge updates, etc.). These commits bundle every issue
+ * number from a release window and steal top-1 from real work commits.
+ *
+ * Defaults: factor 0.5, regex covers `chore(release)`, `Merge `, `bump `,
+ * `[Dream Cycle]`, `publish 3.x.y`. Tune via env or call site.
+ */
+export const META_COMMIT_REGEX = /^(chore\(release\)|Merge\s|bump\s|publish\s+\d|\[Dream Cycle\b)/i;
+export function typePenalty(name, factor = 0.5, regex = META_COMMIT_REGEX) {
+    if (!name)
+        return 1.0;
+    return regex.test(name) ? factor : 1.0;
+}
+/**
+ * BM25 score of one document against a query.
+ * Standard Okapi formula with k1=1.5 and b=0.75.
+ */
+export function bm25Score(queryTokens, docTokens, stats, k1 = 1.5, b = 0.75) {
+    if (queryTokens.length === 0 || docTokens.length === 0)
+        return 0;
+    // Term frequency in this document
+    const tf = new Map();
+    for (const tok of docTokens)
+        tf.set(tok, (tf.get(tok) ?? 0) + 1);
+    const docLen = docTokens.length;
+    const norm = docLen / (stats.avgDocLen || 1);
+    let score = 0;
+    for (const qt of queryTokens) {
+        const f = tf.get(qt);
+        if (!f)
+            continue;
+        const idf = stats.idf.get(qt) ?? 0;
+        if (idf === 0)
+            continue;
+        const numerator = f * (k1 + 1);
+        const denominator = f + k1 * (1 - b + b * norm);
+        score += idf * (numerator / denominator);
+    }
+    return score;
+}
+/**
+ * Min-max normalise a score vector to [0, 1].
+ * Returns the original vector if it is constant (avoid divide-by-zero).
+ */
+export function normalise(scores) {
+    if (scores.length === 0)
+        return scores;
+    let lo = Infinity, hi = -Infinity;
+    for (const s of scores) {
+        if (s < lo)
+            lo = s;
+        if (s > hi)
+            hi = s;
+    }
+    const range = hi - lo;
+    if (range < 1e-9)
+        return scores.map(() => 0.5);
+    return scores.map((s) => (s - lo) / range);
+}
+/**
+ * Combine cosine and BM25 scores. Both vectors must be aligned by docIndex.
+ * Returns hybridScores[] aligned to the same index.
+ * alpha controls the cosine weight; (1-alpha) is the BM25 weight.
+ */
+export function hybridScores(cosine, bm25, alpha = 0.6) {
+    if (cosine.length !== bm25.length) {
+        throw new Error('hybridScores: cosine and bm25 length mismatch');
+    }
+    const cN = normalise(cosine);
+    const bN = normalise(bm25);
+    return cN.map((c, i) => alpha * c + (1 - alpha) * bN[i]);
+}
+/**
+ * Multi-field BM25 score — treats subject and body as separate fields with
+ * independent token frequencies, then combines `subjectWeight * subjectBM25
+ * + bodyWeight * bodyBM25`. Subject (commit title / pattern name) carries
+ * the high-signal tokens (file names, action verbs, ADR refs); body is
+ * often boilerplate. Default 3:1 weight reflects that asymmetry.
+ *
+ * Caller must build separate CorpusStats for subjects and bodies (their
+ * IDF distributions differ — subjects are short, bodies long).
+ */
+export function multiFieldBM25(queryTokens, subjectTokens, bodyTokens, subjectStats, bodyStats, subjectWeight = 3.0, bodyWeight = 1.0) {
+    const sScore = bm25Score(queryTokens, subjectTokens, subjectStats);
+    const bScore = bm25Score(queryTokens, bodyTokens, bodyStats);
+    return subjectWeight * sScore + bodyWeight * bScore;
+}
+/**
+ * Cosine similarity between two equal-length numeric vectors.
+ * Returns 0 if either has zero norm.
+ */
+export function cosineSim(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0, na = 0, nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    const denom = Math.sqrt(na) * Math.sqrt(nb);
+    return denom > 1e-9 ? dot / denom : 0;
+}
+/**
+ * Maximal Marginal Relevance rerank.
+ *
+ * Greedy selection: at each step pick the candidate that maximises
+ *   lambda * relevance(c) - (1-lambda) * max(similarity(c, picked))
+ *
+ * lambda=1.0 → pure relevance (no diversity adjustment)
+ * lambda=0.0 → pure diversity
+ *
+ * Default lambda=0.5 balances both.
+ */
+export function mmrRerank(candidates, k, lambda = 0.5) {
+    if (candidates.length === 0 || k === 0)
+        return [];
+    const picked = [];
+    const remaining = candidates.slice();
+    while (picked.length < k && remaining.length > 0) {
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < remaining.length; i++) {
+            const cand = remaining[i];
+            let maxSimToPicked = 0;
+            for (const p of picked) {
+                const s = cosineSim(cand.embedding, p.embedding);
+                if (s > maxSimToPicked)
+                    maxSimToPicked = s;
+            }
+            const mmrScore = lambda * cand.relevance - (1 - lambda) * maxSimToPicked;
+            if (mmrScore > bestScore) {
+                bestScore = mmrScore;
+                bestIdx = i;
+            }
+        }
+        const chosen = remaining.splice(bestIdx, 1)[0];
+        picked.push({ ...chosen, mmrScore: bestScore });
+    }
+    return picked;
+}
+//# sourceMappingURL=hybrid-retrieval.js.map