opencode-fractal-memory 0.2.0

package/dist/storage/sqlite.js

@@ -0,0 +1,329 @@
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
+import { Database } from "bun:sqlite";
+import { runMigrations, getConfig, setConfig } from "./migrations";
+import { getHNSWIndex } from "../hnsw-index";
+import { tokenize, extractLinks, embeddingToBlob, blobToEmbedding, withRetry, withRetryableTransaction } from "./utils";
+export { extractLinks, embeddingToBlob, blobToEmbedding, tokenize, withRetry, withRetryableTransaction };
+import { queryListNodes, queryGetNodeByLabel, queryGetNodeByLabelFull, queryGetNodeByPrefix, queryCreateNode, queryUpdateNode, queryDeleteNode } from "./queries/nodes";
+import { queryStoreLinks, queryUpdateLinksForNewNode, queryGetLinks, queryDeleteLinks } from "./queries/links";
+import { updateBM25Index, removeBM25Index } from "./queries/search-helpers";
+import { CompressionHelper, COMPRESSION_LEVELS } from "./compression";
+export { CompressionHelper, COMPRESSION_LEVELS };
+import { memLog } from "../logging";
+import { insertToolUsageLog, queryToolPatterns, queryFrequentSequences, deleteUsageLog, getToolCategory } from "./tool-usage";
+import { insertAgentToolCall, createSessionMetrics as createSessionMetricsRow, updateSessionMetrics, incrementSessionToolCall, getSessionStats as getSessionStatsForSession } from "./session-tracking";
+import { insertInjectionMetrics, getPendingInjections as getPendingInjectionRows, markInjectionProcessed, updateMemoryToolCall, finalizeInjection, insertInjectionFeedback, queryInjectionMetrics, querySessionMetrics } from "./injection-events";
+import { runScoreDecay as runScoreDecayFn, calculateNodeConfidence as calculateNodeConfidenceFn } from "./scoring";
+import { ensureSeed as ensureSeedFn, resolveNode as resolveNodeFn, getNode as getNodeFn, verifyNode as verifyNodeFn } from "./lifecycle";
+import { searchByEmbedding as searchByEmbeddingFn, detectTopicBoundaries as detectTopicBoundariesFn, drilldownQuery as drilldownQueryFn, getDrilldownPath as getDrilldownPathFn } from "./search";
+import { retrieveFractal as retrieveFractalFn, getFractalStats as getFractalStatsFn } from "./navigation";
+import { getCompressionCandidates as getCompressionCandidatesFn, runCompression as runCompressionFn, runPatternExtraction as runPatternExtractionFn } from "./compress-ops";
+import { getExpiredNodes as getExpiredNodesFn, deleteExpiredNodes as deleteExpiredNodesFn, pruneNodes as pruneNodesFn } from "./expiration";
+import { backfillLinks as backfillLinksFn, backfillBinaryEmbeddingsAndBM25 as backfillBinaryEmbeddingsAndBM25Fn, rebuildHNSWIndex as rebuildHNSWIndexFn } from "./maintenance";
+const SEED_BLOCKS = [
+    { scope: "global", label: "persona" },
+    { scope: "global", label: "human" },
+    { scope: "project", label: "project" },
+];
+function scopeDbPath(projectDirectory, scope, globalDbPath) {
+    return scope === "global"
+        ? (globalDbPath ?? path.join(os.homedir(), ".config", "opencode", "memory.db"))
+        : path.join(projectDirectory, ".opencode", "memory.db");
+}
+function validateLabel(label) {
+    const trimmed = label.trim();
+    if (!/^[a-z0-9][a-z0-9-_:]{1,60}$/i.test(trimmed)) {
+        throw new Error(`Invalid label "${label}". Use letters/numbers/dash/underscore/colon (2-61 chars).`);
+    }
+    return trimmed;
+}
+class SqliteMemoryStore {
+    dbs = new Map();
+    dbInitPromises = new Map();
+    idScopeCache = new Map();
+    projectDirectory;
+    globalDbPath;
+    constructor(projectDirectory, globalDbPath) {
+        this.projectDirectory = projectDirectory;
+        this.globalDbPath = globalDbPath;
+    }
+    async getDb(scope) {
+        const key = `${scope}:${this.projectDirectory}:${this.globalDbPath ?? ""}`;
+        if (this.dbs.has(key)) {
+            return this.dbs.get(key);
+        }
+        const existing = this.dbInitPromises.get(key);
+        if (existing)
+            return existing;
+        const promise = this.initDb(key, scope);
+        this.dbInitPromises.set(key, promise);
+        try {
+            const db = await promise;
+            return db;
+        }
+        catch (err) {
+            this.dbInitPromises.delete(key);
+            throw err;
+        }
+    }
+    async initDb(key, scope) {
+        const dbPath = scopeDbPath(this.projectDirectory, scope, this.globalDbPath);
+        const dbDir = path.dirname(dbPath);
+        await fs.mkdir(dbDir, { recursive: true });
+        const db = new Database(dbPath);
+        db.run("PRAGMA journal_mode = WAL");
+        db.run("PRAGMA synchronous = NORMAL");
+        db.run("PRAGMA busy_timeout = 5000");
+        runMigrations(db);
+        this.dbs.set(key, db);
+        this.dbInitPromises.delete(key);
+        return db;
+    }
+    async getGlobalDb() {
+        return this.getDb("global");
+    }
+    async runScoreDecay(decayDays) {
+        return runScoreDecayFn((s) => this.getDb(s), decayDays);
+    }
+    async close() {
+        for (const [key, db] of this.dbs) {
+            try {
+                db.close();
+            }
+            catch (error) {
+                memLog("error", "storage", `Error closing database ${key}:`, { error });
+            }
+        }
+        this.dbs.clear();
+        this.idScopeCache.clear();
+    }
+    async ensureSeed() {
+        return ensureSeedFn((s) => this.getDb(s), SEED_BLOCKS);
+    }
+    async listNodes(scope, level, limit = 50, offset = 0, includeExpired) {
+        const scopes = scope === "all" ? ["global", "project"] : [scope];
+        const nodes = [];
+        for (const s of scopes) {
+            const db = await this.getDb(s);
+            const rows = await queryListNodes(db, s, level, limit, offset, includeExpired);
+            nodes.push(...rows);
+        }
+        return nodes;
+    }
+    async getNode(id) {
+        return getNodeFn((s) => this.getDb(s), id);
+    }
+    async getNodeByLabel(scope, label) {
+        const db = await this.getDb(scope);
+        return queryGetNodeByLabelFull(db, scope, label, false);
+    }
+    async getNodeByPrefix(prefix) {
+        const node = await queryGetNodeByPrefix(await this.getDb("global"), prefix)
+            ?? await queryGetNodeByPrefix(await this.getDb("project"), prefix);
+        return node;
+    }
+    async createNode(node) {
+        const db = await this.getDb(node.scope);
+        return await withRetryableTransaction(db, async () => {
+            return await queryCreateNode(db, node, (scope, id, content) => this.storeLinks(scope, id, content), (scope, label, id) => this.updateLinksForNewNode(scope, label, id), (db, id, content, label, scope) => updateBM25Index(db, id, content, label, scope));
+        });
+    }
+    async updateNode(id, updates) {
+        const { db, scope } = await resolveNodeFn((s) => this.getDb(s), this.idScopeCache, id);
+        await withRetryableTransaction(db, async () => {
+            await queryUpdateNode(db, id, updates);
+            if (updates.content !== undefined) {
+                await this.storeLinks(scope, id, updates.content);
+                const labelRow = db.query("SELECT label FROM memory_nodes WHERE id = ?").get(id);
+                updateBM25Index(db, id, updates.content, labelRow?.label, scope);
+            }
+        });
+    }
+    async deleteNode(id) {
+        const { db, scope } = await resolveNodeFn((s) => this.getDb(s), this.idScopeCache, id);
+        await withRetryableTransaction(db, async () => {
+            queryDeleteNode(db, id);
+            queryDeleteLinks(db, id);
+            removeBM25Index(db, id);
+        });
+        this.idScopeCache.delete(id);
+        const hnsw = getHNSWIndex();
+        await hnsw.removeNode(scope, id);
+    }
+    async getConfig(scope, key, defaultValue) {
+        const db = await this.getDb(scope);
+        return getConfig(db, key, defaultValue);
+    }
+    async setConfig(scope, key, value) {
+        const db = await this.getDb(scope);
+        await withRetry(() => setConfig(db, key, value));
+    }
+    async storeLinks(scope, sourceId, content) {
+        const db = await this.getDb(scope);
+        await queryStoreLinks(db, sourceId, content, queryGetNodeByLabel);
+    }
+    async updateLinksForNewNode(scope, label, nodeId) {
+        const db = await this.getDb(scope);
+        queryUpdateLinksForNewNode(db, label, nodeId);
+    }
+    async backfillLinks(scope) {
+        const db = await this.getDb(scope);
+        return backfillLinksFn(db);
+    }
+    async getLinkedNodes(scope, sourceId) {
+        const db = await this.getDb(scope);
+        const rows = queryGetLinks(db, sourceId);
+        const linkedNodes = [];
+        for (const row of rows) {
+            if (row.target_id) {
+                try {
+                    const node = await this.getNode(row.target_id);
+                    linkedNodes.push(node);
+                }
+                catch { /* Node was deleted */ }
+            }
+        }
+        return linkedNodes;
+    }
+    async backfillBinaryEmbeddingsAndBM25(scope) {
+        const db = await this.getDb(scope);
+        return backfillBinaryEmbeddingsAndBM25Fn(db, scope);
+    }
+    async rebuildHNSWIndex(scope) {
+        return rebuildHNSWIndexFn((s) => this.getDb(s), scope);
+    }
+    async searchByEmbedding(query, limit = 5, options) {
+        return searchByEmbeddingFn((s) => this.getDb(s), query, limit, options);
+    }
+    async getCompressionCandidates(scope, level, maxAgeMs, force) {
+        return getCompressionCandidatesFn((s) => this.getDb(s), scope, level, maxAgeMs, force);
+    }
+    async runCompression(scope, force, client) {
+        return runCompressionFn({
+            getCompressionCandidates: (s, l, maxAge, f) => this.getCompressionCandidates(s, l, maxAge, f),
+            createNode: (node) => this.createNode(node),
+            updateNode: (id, updates) => this.updateNode(id, updates),
+        }, scope, force, client);
+    }
+    async runPatternExtraction(scope, minSourceCount = 2) {
+        return runPatternExtractionFn({
+            listNodes: (s, level, limit, offset, includeExpired) => this.listNodes(s, level, limit, offset, includeExpired),
+            createNode: (node) => this.createNode(node),
+            updateNode: (id, updates) => this.updateNode(id, updates),
+        }, scope, minSourceCount);
+    }
+    async getFractalStats(scope) {
+        return getFractalStatsFn((s) => this.listNodes(s), (id) => this.getNode(id), scope);
+    }
+    async retrieveFractal(id, maxDepth = 10) {
+        return retrieveFractalFn((id) => this.getNode(id), id, maxDepth);
+    }
+    async detectTopicBoundaries(scope, minSimilarity = 0.7) {
+        return detectTopicBoundariesFn((s) => this.getDb(s), scope, minSimilarity);
+    }
+    async logToolCall(toolName, resultTokens, contextWarning, success, durationMs = 0) {
+        const db = await this.getDb("global");
+        insertToolUsageLog(db, toolName, resultTokens, contextWarning, success, durationMs);
+    }
+    async getToolPatterns(_scope) {
+        const db = await this.getDb("global");
+        return queryToolPatterns(db);
+    }
+    async getFrequentSequences(_scope, minCount = 3) {
+        const db = await this.getDb("global");
+        return queryFrequentSequences(db, minCount);
+    }
+    async pruneUsageLog(maxAgeMs) {
+        const db = await this.getDb("global");
+        return deleteUsageLog(db, maxAgeMs);
+    }
+    getToolCategory(toolName) {
+        return getToolCategory(toolName);
+    }
+    async recordAgentToolCall(sessionId, toolName, args, output, success, durationMs) {
+        const db = await this.getGlobalDb();
+        const category = getToolCategory(toolName);
+        insertAgentToolCall(db, sessionId, toolName, args, output, success, durationMs, category);
+        if (sessionId) {
+            await this.incrementSessionToolCall(sessionId, toolName, success ?? true, null);
+        }
+    }
+    async createSessionMetrics(sessionId, startedAt) {
+        const db = await this.getGlobalDb();
+        createSessionMetricsRow(db, sessionId, startedAt);
+    }
+    async updateSessionMetrics(sessionId, updates) {
+        const db = await this.getGlobalDb();
+        updateSessionMetrics(db, sessionId, updates);
+    }
+    async incrementSessionToolCall(sessionId, toolName, success, filePath) {
+        const db = await this.getGlobalDb();
+        incrementSessionToolCall(db, sessionId, toolName, success, filePath);
+    }
+    async getSessionStats(sessionId) {
+        const db = await this.getGlobalDb();
+        return getSessionStatsForSession(db, sessionId);
+    }
+    async verifyNode(id) {
+        return verifyNodeFn((s) => this.getDb(s), id);
+    }
+    calculateNodeConfidence(node) {
+        return calculateNodeConfidenceFn(node);
+    }
+    async drilldownQuery(query, maxResults = 20) {
+        return drilldownQueryFn({
+            getDb: (s) => this.getDb(s),
+            searchByEmbedding: (q, limit, opts) => this.searchByEmbedding(q, limit, opts),
+            getDrilldownPath: (nodeId, maxDepth) => getDrilldownPathFn((id) => this.getNode(id), nodeId, maxDepth),
+        }, query, maxResults);
+    }
+    async pruneNodes(scope, options = {}) {
+        return pruneNodesFn({
+            getDb: (s) => this.getDb(s),
+            listNodes: (s, level, limit, offset, includeExpired) => this.listNodes(s, level, limit, offset, includeExpired),
+        }, scope, options);
+    }
+    async getExpiredNodes(scope = "all") {
+        return getExpiredNodesFn((s) => this.getDb(s), scope);
+    }
+    async deleteExpiredNodes(scope = "all") {
+        return deleteExpiredNodesFn((s) => this.getDb(s), scope);
+    }
+    async logInjectionMetrics(sessionId, data) {
+        const db = await this.getGlobalDb();
+        insertInjectionMetrics(db, sessionId, data);
+    }
+    async getPendingInjections() {
+        const db = await this.getGlobalDb();
+        return getPendingInjectionRows(db);
+    }
+    async markInjectionProcessed(id) {
+        const db = await this.getGlobalDb();
+        markInjectionProcessed(db, id);
+    }
+    async recordMemoryToolCall(sessionId, toolName, _args) {
+        const db = await this.getGlobalDb();
+        updateMemoryToolCall(db, sessionId, toolName);
+    }
+    async finalizeInjection(sessionId, effectivenessScore, taskDescription) {
+        const db = await this.getGlobalDb();
+        finalizeInjection(db, sessionId, effectivenessScore, taskDescription);
+    }
+    async recordInjectionFeedback(sessionId, upvotes, downvotes, taskOutcome, neededNodes) {
+        const db = await this.getGlobalDb();
+        insertInjectionFeedback(db, sessionId, upvotes, downvotes, taskOutcome, neededNodes);
+    }
+    async getInjectionMetrics(limit = 100) {
+        const db = await this.getGlobalDb();
+        return queryInjectionMetrics(db, limit);
+    }
+    async getSessionMetrics(sessionId) {
+        const db = await this.getGlobalDb();
+        return querySessionMetrics(db, sessionId);
+    }
+}
+export function createSqliteMemoryStore(projectDirectory, globalDbPath) {
+    return new SqliteMemoryStore(projectDirectory, globalDbPath);
+}

package/dist/storage/tool-usage.js

@@ -0,0 +1,56 @@
+import { randomUUID } from "node:crypto";
+import { withRetry } from "./utils";
+export async function insertToolUsageLog(db, toolName, resultTokens, contextWarning, success, durationMs = 0) {
+    await withRetry(() => {
+        db.run("INSERT INTO memory_usage_log (id, tool_name, timestamp, result_tokens, context_warning, success, duration_ms) VALUES (?, ?, ?, ?, ?, ?, ?)", [randomUUID(), toolName, Date.now(), resultTokens, contextWarning ? 1 : 0, success ? 1 : 0, Math.round(durationMs)]);
+    });
+}
+export function queryToolPatterns(db) {
+    const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    const rows = db.query("SELECT tool_name, COUNT(*) as count, AVG(result_tokens) as avg_tokens, AVG(duration_ms) as avg_duration, AVG(context_warning) as warning_rate, AVG(success) as success_rate FROM memory_usage_log WHERE timestamp > ? GROUP BY tool_name ORDER BY count DESC").all(cutoff);
+    return rows.map(r => ({
+        toolName: r.tool_name,
+        count: r.count,
+        avgTokens: Math.round(r.avg_tokens),
+        avgDurationMs: Math.round(r.avg_duration ?? 0),
+        warningRate: Math.round(r.warning_rate * 100),
+        successRate: Math.round(r.success_rate * 100),
+    }));
+}
+export function queryFrequentSequences(db, minCount = 3) {
+    const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    const rows = db.query("SELECT tool_name, timestamp FROM memory_usage_log WHERE timestamp > ? ORDER BY timestamp ASC").all(cutoff);
+    const pairs = new Map();
+    for (let i = 0; i < rows.length - 1; i++) {
+        const prev = rows[i].tool_name;
+        const next = rows[i + 1].tool_name;
+        if (prev === next)
+            continue;
+        const key = `${prev}\u2192${next}`;
+        pairs.set(key, (pairs.get(key) ?? 0) + 1);
+    }
+    const result = [];
+    for (const [key, count] of pairs) {
+        if (count >= minCount) {
+            const [prev, next] = key.split("\u2192");
+            result.push({ prev: prev, next: next, count });
+        }
+    }
+    result.sort((a, b) => b.count - a.count);
+    return result.slice(0, 10);
+}
+export async function deleteUsageLog(db, maxAgeMs) {
+    const threshold = maxAgeMs ?? 30 * 24 * 60 * 60 * 1000;
+    const cutoff = Date.now() - threshold;
+    const result = await withRetry(() => db.run("DELETE FROM memory_usage_log WHERE timestamp < ?", [cutoff]));
+    return result.changes;
+}
+export function getToolCategory(toolName) {
+    if (toolName.startsWith("memory_") || toolName.startsWith("journal_"))
+        return "memory";
+    if (["read", "edit", "write", "glob", "grep", "search"].includes(toolName))
+        return "file";
+    if (["bash", "shell"].includes(toolName))
+        return "shell";
+    return "other";
+}

package/dist/storage/types.js

@@ -0,0 +1 @@
+export {};

package/dist/storage/utils.js

@@ -0,0 +1,94 @@
+export function extractLinks(content) {
+    const links = [];
+    let i = 0;
+    while (i < content.length) {
+        if (content[i] === "[" && content[i + 1] === "[") {
+            const end = content.indexOf("]]", i + 2);
+            if (end !== -1) {
+                const label = content.slice(i + 2, end).trim();
+                if (label && label.length <= 60 && /^[a-z0-9][a-z0-9-_]{1,60}$/i.test(label)) {
+                    if (!links.includes(label)) {
+                        links.push(label);
+                    }
+                }
+                i = end + 2;
+                continue;
+            }
+        }
+        i++;
+    }
+    return links;
+}
+export function embeddingToBlob(embedding) {
+    const floats = new Float32Array(embedding);
+    return Buffer.from(floats.buffer);
+}
+export function blobToEmbedding(blob) {
+    const floats = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+    return Array.from(floats);
+}
+export function tokenize(text) {
+    return text.toLowerCase()
+        .replace(/[^\w\s]/g, " ")
+        .split(/\s+/)
+        .filter(t => t.length >= 2);
+}
+// Retry wrapper for database operations with exponential backoff
+export async function withRetry(operation, maxRetries = 3, baseDelay = 100) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            lastError = error;
+            const isBusyError = error instanceof Error &&
+                (error.message.includes("database is locked") ||
+                    error.message.includes("database is busy") ||
+                    error.message.includes("SQLITE_BUSY") ||
+                    error.message.includes("SQLITE_LOCKED"));
+            if (!isBusyError || attempt === maxRetries) {
+                throw error;
+            }
+            const delay = baseDelay * Math.pow(2, attempt);
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+    }
+    throw lastError;
+}
+// Transaction wrapper for atomic operations
+export async function withTransaction(db, operation) {
+    db.run("BEGIN");
+    try {
+        const result = await operation();
+        db.run("COMMIT");
+        return result;
+    }
+    catch (error) {
+        db.run("ROLLBACK");
+        throw error;
+    }
+}
+// Retryable transaction: retries entire transaction on busy errors
+export async function withRetryableTransaction(db, operation, maxRetries = 3) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await withTransaction(db, operation);
+        }
+        catch (error) {
+            lastError = error;
+            const isBusyError = error instanceof Error &&
+                (error.message.includes("database is locked") ||
+                    error.message.includes("database is busy") ||
+                    error.message.includes("SQLITE_BUSY") ||
+                    error.message.includes("SQLITE_LOCKED"));
+            if (!isBusyError || attempt === maxRetries) {
+                throw error;
+            }
+            const delay = 100 * Math.pow(2, attempt);
+            await new Promise(resolve => setTimeout(resolve, delay));
+        }
+    }
+    throw lastError;
+}

package/dist/tools/auto-test.js

@@ -0,0 +1,24 @@
+import { tool } from "@opencode-ai/plugin";
+import { generateEmbedding } from "../embeddings";
+import { rerankDocuments } from "../ollama";
+export function createMemoryAutoTest(store) {
+    const testQuery = "JSON parsing and validation best practices";
+    return tool({
+        description: "Test auto-retrieval pipeline with a hardcoded query - used for debugging",
+        args: {},
+        async execute() {
+            try {
+                const queryEmbedding = await generateEmbedding(testQuery);
+                const candidates = await store.searchByEmbedding(queryEmbedding, 10, { bm25Weight: 0.4 });
+                if (candidates.length === 0)
+                    return "No candidates found";
+                const results = await rerankDocuments(testQuery, candidates.map((c) => ({ id: c.id, label: c.label ?? c.id, content: c.content.slice(0, 200) })), { topK: 3 });
+                const selected = candidates.filter((c) => results.some((r) => r.id === c.id));
+                return `Pipeline test: query="${testQuery}", candidates=${candidates.length}, selected=${selected.length}\n\nSelected:\n${selected.map((n, i) => `${i + 1}. ${n.label}: ${n.content.slice(0, 100)}`).join("\n")}`;
+            }
+            catch (err) {
+                return `Error: ${err}`;
+            }
+        },
+    });
+}

package/dist/tools/cache-status.js

@@ -0,0 +1,36 @@
+import { tool } from "@opencode-ai/plugin";
+import { getWorkingCache } from "../cache";
+import { wrapWithTracking } from "./shared";
+export function MemoryCacheStatus(store) {
+    const t = tool({
+        description: "Show current working-memory cache usage (size, max size, recent files).",
+        args: {},
+        async execute(_args) {
+            const cache = getWorkingCache("dashboard");
+            const maxSize = 8;
+            const ttlHours = 2;
+            const lines = [
+                "## Working Memory Cache",
+                "",
+                `Current size: ${cache.length} / ${maxSize}`,
+                `TTL: ${ttlHours} hours`,
+                "",
+            ];
+            if (cache.length === 0) {
+                lines.push("Cache is empty. Nodes are added to the working cache as they are accessed during a session.");
+                return lines.join("\n");
+            }
+            lines.push("### Cached Nodes");
+            lines.push("| Label | Importance | Cached At |");
+            lines.push("|-------|------------|-----------|");
+            for (const n of cache) {
+                const date = new Date(n.cachedAt).toLocaleString();
+                lines.push(`| ${n.label} | ${n.importance.toFixed(2)} | ${date} |`);
+            }
+            lines.push("");
+            lines.push("_Cache entries expire after ${ttlHours} hours or when the cache exceeds ${maxSize} entries._");
+            return lines.join("\n");
+        },
+    });
+    return wrapWithTracking(t, store, "memory_cache_status");
+}