opencode-fractal-memory 0.2.0

package/dist/storage/search.js

@@ -0,0 +1,233 @@
+import { rowToNode } from "./queries/base";
+import { getHNSWIndex } from "../hnsw-index";
+import { generateEmbedding } from "../embeddings";
+import { cosineSimilarity, computeBM25ScoresSQL, computeFinalScores, rerankResults } from "./queries/search-helpers";
+import { tokenize, blobToEmbedding } from "./utils";
+export async function searchByEmbedding(getDb, query, limit = 5, options) {
+    const weights = options?.levelWeights ?? {};
+    const bm25Weight = options?.bm25Weight ?? 0;
+    const doRerank = options?.rerank ?? false;
+    const queryText = options?.queryText ?? "";
+    const hnsw = getHNSWIndex();
+    const hnswResults = await hnsw.search(query, limit * 5);
+    let scoredNodes = [];
+    if (hnswResults.length > 0) {
+        const candidateIds = new Set(hnswResults.map(r => r.id));
+        const hnswScoreMap = new Map(hnswResults.map(r => [r.id, r.score]));
+        for (const scope of ["global", "project"]) {
+            const db = await getDb(scope);
+            const placeholders = Array.from(candidateIds).map(() => "?").join(",");
+            const rows = db.query(`SELECT * FROM memory_nodes WHERE id IN (${placeholders}) AND (embedding IS NOT NULL OR embedding_blob IS NOT NULL) AND (expires_at IS NULL OR expires_at > ?)`).all(...Array.from(candidateIds), Date.now());
+            for (const row of rows) {
+                const level = row.level;
+                const node = rowToNode(row);
+                if (!node.embedding)
+                    continue;
+                if (options?.minLevel !== undefined && level < options.minLevel)
+                    continue;
+                if (options?.maxLevel !== undefined && level > options.maxLevel)
+                    continue;
+                if (options?.minUsefulness !== undefined && (node.usefulnessScore ?? 0) < options.minUsefulness)
+                    continue;
+                let embedding = node.embedding;
+                if (row.embedding_blob) {
+                    embedding = blobToEmbedding(row.embedding_blob);
+                }
+                const hnswScore = hnswScoreMap.get(node.id) ?? 0;
+                const levelWeight = weights[level] ?? 1;
+                const confidence = node.confidence ?? 0.5;
+                const confidenceWeight = 0.5 + 0.5 * confidence;
+                scoredNodes.push({
+                    ...node,
+                    importance: hnswScore * levelWeight * confidenceWeight * (1 + (node.usefulnessScore ?? 0) * 0.1),
+                });
+            }
+        }
+    }
+    else {
+        const minLevel = options?.minLevel ?? 0;
+        const maxLevel = options?.maxLevel ?? 5;
+        const minUsefulness = options?.minUsefulness ?? 0;
+        for (const scope of ["global", "project"]) {
+            const db = await getDb(scope);
+            const rows = db.query(`SELECT * FROM memory_nodes WHERE (embedding IS NOT NULL OR embedding_blob IS NOT NULL) AND level >= ? AND level <= ? AND usefulness_score >= ? AND (expires_at IS NULL OR expires_at > ?)`).all(minLevel, maxLevel, minUsefulness, Date.now());
+            for (const row of rows) {
+                const level = row.level;
+                const node = rowToNode(row);
+                if (!node.embedding)
+                    continue;
+                let embedding = node.embedding;
+                if (row.embedding_blob) {
+                    embedding = blobToEmbedding(row.embedding_blob);
+                }
+                const semanticScore = cosineSimilarity(query, embedding);
+                const levelWeight = weights[level] ?? 1;
+                const confidence = node.confidence ?? 0.5;
+                const confidenceWeight = 0.5 + 0.5 * confidence;
+                scoredNodes.push({
+                    ...node,
+                    importance: semanticScore * levelWeight * confidenceWeight * (1 + (node.usefulnessScore ?? 0) * 0.1),
+                });
+            }
+        }
+    }
+    if (bm25Weight > 0 && queryText) {
+        const queryTerms = tokenize(queryText);
+        const bm25Scores = new Map();
+        for (const scope of ["global", "project"]) {
+            const db = await getDb(scope);
+            const scopeNodeIds = scoredNodes.filter(n => n.scope === scope).map(n => n.id);
+            if (scopeNodeIds.length === 0)
+                continue;
+            const scopeScores = computeBM25ScoresSQL(db, scope, queryTerms, scopeNodeIds);
+            for (const [id, score] of scopeScores) {
+                bm25Scores.set(id, score);
+            }
+        }
+        options = { ...options, bm25Scores };
+    }
+    const finalNodes = computeFinalScores(scoredNodes, options);
+    let finalResults = finalNodes.slice(0, limit);
+    if (doRerank && queryText.length > 0 && finalResults.length > 3) {
+        const reranked = rerankResults(queryText, finalResults, limit);
+        finalResults = reranked.map(r => ({
+            ...r.node,
+            importance: r.finalScore
+        }));
+    }
+    const now = Date.now();
+    for (const node of finalResults) {
+        const db = await getDb(node.scope);
+        db.run(`UPDATE memory_nodes SET times_used = times_used + 1, last_accessed = ? WHERE id = ?`, [now, node.id]);
+    }
+    return finalResults;
+}
+export async function getDrilldownPath(getNode, nodeId, maxDepth) {
+    const path = [];
+    let currentId = nodeId;
+    let depth = 0;
+    const visited = new Set();
+    while (currentId && depth < maxDepth) {
+        if (visited.has(currentId))
+            break;
+        visited.add(currentId);
+        try {
+            const node = await getNode(currentId);
+            path.push(node);
+            if (!node.parentIds || node.parentIds.length === 0)
+                break;
+            currentId = node.parentIds[0] ?? null;
+            depth++;
+        }
+        catch {
+            break;
+        }
+    }
+    return path;
+}
+export async function drilldownQuery(deps, query, maxResults = 20) {
+    const queryEmbedding = await generateEmbedding(query);
+    const results = [];
+    const seenIds = new Set();
+    const summaries = await deps.searchByEmbedding(queryEmbedding, 5, { minLevel: 2, maxLevel: 4 });
+    for (const node of summaries) {
+        if (seenIds.has(node.id))
+            continue;
+        seenIds.add(node.id);
+        const path = await deps.getDrilldownPath(node.id, 3);
+        results.push({
+            node,
+            relevance: node.importance,
+            path,
+            level: "summary",
+        });
+    }
+    const intermediates = await deps.searchByEmbedding(queryEmbedding, 10, { minLevel: 1, maxLevel: 1 });
+    for (const node of intermediates) {
+        if (seenIds.has(node.id))
+            continue;
+        if (results.length >= maxResults)
+            break;
+        seenIds.add(node.id);
+        const path = await deps.getDrilldownPath(node.id, 2);
+        results.push({
+            node,
+            relevance: node.importance * 0.8,
+            path,
+            level: "intermediate",
+        });
+    }
+    const details = await deps.searchByEmbedding(queryEmbedding, maxResults, { minLevel: 0, maxLevel: 0 });
+    for (const node of details) {
+        if (seenIds.has(node.id))
+            continue;
+        if (results.length >= maxResults)
+            break;
+        seenIds.add(node.id);
+        results.push({
+            node,
+            relevance: node.importance * 0.6,
+            path: [node],
+            level: "detail",
+        });
+    }
+    if (results.length < maxResults) {
+        for (const scope of ["global", "project"]) {
+            const db = await deps.getDb(scope);
+            const rows = db.query("SELECT * FROM memory_nodes WHERE (label LIKE ? OR content LIKE ?) AND (embedding IS NULL AND embedding_blob IS NULL)").all(`%${query}%`, `%${query}%`);
+            for (const row of rows) {
+                if (results.length >= maxResults)
+                    break;
+                const node = rowToNode(row);
+                if (seenIds.has(node.id))
+                    continue;
+                seenIds.add(node.id);
+                results.push({
+                    node,
+                    relevance: 0.5,
+                    path: [node],
+                    level: "detail",
+                });
+            }
+            if (results.length >= maxResults)
+                break;
+        }
+    }
+    return results.sort((a, b) => b.relevance - a.relevance).slice(0, maxResults);
+}
+export async function detectTopicBoundaries(getDb, scope, minSimilarity = 0.7) {
+    const scopes = scope === "all" ? ["global", "project"] : [scope];
+    const nodesWithEmbeddings = [];
+    for (const s of scopes) {
+        const db = await getDb(s);
+        const rows = db.query("SELECT * FROM memory_nodes WHERE (embedding IS NOT NULL OR embedding_blob IS NOT NULL) AND length(content) > 50 ORDER BY created_at DESC").all();
+        for (const row of rows) {
+            nodesWithEmbeddings.push(rowToNode(row));
+        }
+    }
+    if (nodesWithEmbeddings.length < 2)
+        return [];
+    const sorted = nodesWithEmbeddings;
+    const boundaries = [];
+    let currentCluster = [sorted[0]];
+    for (let i = 1; i < sorted.length; i++) {
+        const prev = sorted[i - 1];
+        const curr = sorted[i];
+        if (!prev.embedding || !curr.embedding)
+            continue;
+        const similarity = cosineSimilarity(prev.embedding, curr.embedding);
+        if (similarity >= minSimilarity) {
+            currentCluster.push(curr);
+        }
+        else {
+            if (currentCluster.length >= 2) {
+                boundaries.push(currentCluster);
+            }
+            currentCluster = [curr];
+        }
+    }
+    if (currentCluster.length >= 2) {
+        boundaries.push(currentCluster);
+    }
+    return boundaries;
+}

package/dist/storage/session-tracking.js

@@ -0,0 +1,180 @@
+import { randomUUID } from "node:crypto";
+import { withRetry } from "./utils";
+export async function insertAgentToolCall(db, sessionId, toolName, args, output, success, durationMs, toolCategory) {
+    const timestamp = Date.now();
+    const id = randomUUID();
+    let filePath = null;
+    let command = null;
+    if (args) {
+        if (args.filePath && typeof args.filePath === "string") {
+            filePath = args.filePath;
+        }
+        else if (args.path && typeof args.path === "string") {
+            filePath = args.path;
+        }
+        if (args.command && typeof args.command === "string") {
+            command = args.command.substring(0, 500);
+        }
+    }
+    const outputPreview = output ? output.substring(0, 500) : null;
+    await withRetry(() => {
+        db.run(`INSERT INTO agent_tool_calls 
+       (id, session_id, timestamp, tool_name, args_json, output_preview, success, duration_ms, tool_category, file_path, command)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+            id,
+            sessionId,
+            timestamp,
+            toolName,
+            args ? JSON.stringify(args) : null,
+            outputPreview,
+            success !== null ? (success ? 1 : 0) : null,
+            durationMs ?? null,
+            toolCategory,
+            filePath,
+            command,
+        ]);
+    });
+}
+export async function createSessionMetrics(db, sessionId, startedAt) {
+    const timestamp = startedAt ?? Date.now();
+    await withRetry(() => {
+        db.run(`INSERT OR REPLACE INTO session_metrics (session_id, started_at, status) VALUES (?, ?, 'active')`, [sessionId, timestamp]);
+    });
+}
+export async function updateSessionMetrics(db, sessionId, updates) {
+    const setClauses = [];
+    const values = [];
+    if (updates.endedAt !== undefined) {
+        setClauses.push("ended_at = ?");
+        values.push(updates.endedAt);
+    }
+    if (updates.totalToolCalls !== undefined) {
+        setClauses.push("total_tool_calls = ?");
+        values.push(updates.totalToolCalls);
+    }
+    if (updates.fileReads !== undefined) {
+        setClauses.push("file_reads = ?");
+        values.push(updates.fileReads);
+    }
+    if (updates.fileEdits !== undefined) {
+        setClauses.push("file_edits = ?");
+        values.push(updates.fileEdits);
+    }
+    if (updates.bashCommands !== undefined) {
+        setClauses.push("bash_commands = ?");
+        values.push(updates.bashCommands);
+    }
+    if (updates.memoryTools !== undefined) {
+        setClauses.push("memory_tools = ?");
+        values.push(updates.memoryTools);
+    }
+    if (updates.failedTools !== undefined) {
+        setClauses.push("failed_tools = ?");
+        values.push(updates.failedTools);
+    }
+    if (updates.uniqueFilesTouched !== undefined) {
+        setClauses.push("unique_files_touched = ?");
+        values.push(JSON.stringify(updates.uniqueFilesTouched));
+    }
+    if (updates.injectionCount !== undefined) {
+        setClauses.push("injection_count = ?");
+        values.push(updates.injectionCount);
+    }
+    if (updates.injectedTokens !== undefined) {
+        setClauses.push("injected_tokens = ?");
+        values.push(updates.injectedTokens);
+    }
+    if (updates.taskDescription !== undefined) {
+        setClauses.push("task_description = ?");
+        values.push(updates.taskDescription);
+    }
+    if (updates.status !== undefined) {
+        setClauses.push("status = ?");
+        values.push(updates.status);
+    }
+    if (setClauses.length === 0)
+        return;
+    values.push(sessionId);
+    await withRetry(() => {
+        db.run(`UPDATE session_metrics SET ${setClauses.join(", ")} WHERE session_id = ?`, values);
+    });
+}
+export async function incrementSessionToolCall(db, sessionId, toolName, success, filePath, toolCategory) {
+    const session = db.query("SELECT * FROM session_metrics WHERE session_id = ?").get(sessionId);
+    if (!session) {
+        await createSessionMetrics(db, sessionId);
+        await incrementSessionToolCall(db, sessionId, toolName, success, filePath, toolCategory);
+        return;
+    }
+    const category = toolCategory ?? getToolCategory(toolName);
+    const updates = {
+        totalToolCalls: session.total_tool_calls + 1,
+    };
+    if (category === "file") {
+        if (toolName === "read")
+            updates.fileReads = session.file_reads + 1;
+        else if (toolName === "edit")
+            updates.fileEdits = session.file_edits + 1;
+    }
+    else if (category === "shell") {
+        updates.bashCommands = session.bash_commands + 1;
+    }
+    else if (category === "memory") {
+        updates.memoryTools = session.memory_tools + 1;
+    }
+    if (!success) {
+        updates.failedTools = session.failed_tools + 1;
+    }
+    if (filePath) {
+        const currentFiles = session.unique_files_touched
+            ? JSON.parse(session.unique_files_touched)
+            : [];
+        if (!currentFiles.includes(filePath)) {
+            updates.uniqueFilesTouched = [...currentFiles, filePath];
+        }
+    }
+    await updateSessionMetrics(db, sessionId, updates);
+}
+function getToolCategory(toolName) {
+    if (toolName.startsWith("memory_") || toolName.startsWith("journal_"))
+        return "memory";
+    if (["read", "edit", "write", "glob", "grep", "search"].includes(toolName))
+        return "file";
+    if (["bash", "shell"].includes(toolName))
+        return "shell";
+    return "other";
+}
+export function getSessionStats(db, sessionId) {
+    const session = db.query("SELECT * FROM session_metrics WHERE session_id = ?").get(sessionId);
+    if (!session)
+        return null;
+    const toolCalls = db.query(`SELECT tool_name, timestamp, tool_category, file_path, command, success 
+     FROM agent_tool_calls 
+     WHERE session_id = ? 
+     ORDER BY timestamp ASC`).all(sessionId);
+    return {
+        sessionId: session.session_id,
+        startedAt: session.started_at,
+        endedAt: session.ended_at,
+        status: session.status,
+        totalToolCalls: session.total_tool_calls,
+        fileReads: session.file_reads,
+        fileEdits: session.file_edits,
+        bashCommands: session.bash_commands,
+        memoryTools: session.memory_tools,
+        failedTools: session.failed_tools,
+        uniqueFilesTouched: session.unique_files_touched
+            ? JSON.parse(session.unique_files_touched)
+            : [],
+        injectionCount: session.injection_count,
+        injectedTokens: session.injected_tokens,
+        toolCalls: toolCalls.map((tc) => ({
+            toolName: tc.tool_name,
+            timestamp: tc.timestamp,
+            toolCategory: tc.tool_category,
+            filePath: tc.file_path,
+            command: tc.command,
+            success: tc.success !== null ? tc.success === 1 : null,
+        })),
+    };
+}