dsh-tdai-memory 0.2.9 → 0.2.11

package/vendor/tdai/core/tools/conversation-search.js

@@ -1,199 +1,199 @@
-/**
- * conversation_search tool: Agent-callable tool for searching L0 conversation records.
- *
- * Supports three search strategies with automatic degradation:
- *   1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
- *      merged via Reciprocal Rank Fusion (RRF).
- *   2. **embedding** — pure vector similarity (when FTS5 is unavailable).
- *   3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
- *
- * The tool is registered via `api.registerTool()` in index.ts.
- */
-import { buildFtsQuery } from "../store/sqlite.js";
-const TAG = "[memory-tdai][tdai_conversation_search]";
-// ============================
-// RRF (Reciprocal Rank Fusion)
-// ============================
-/** Standard RRF constant from the original RRF paper. */
-const RRF_K = 60;
-/**
- * Merge multiple ranked lists of `ConversationSearchResultItem` via Reciprocal
- * Rank Fusion. Items appearing in multiple lists get their RRF scores summed.
- *
- * Returns items sorted by descending RRF score. The `score` field of each
- * returned item is replaced by the RRF score for consistent ranking semantics.
- */
-function rrfMergeL0(...lists) {
-    const map = new Map();
-    for (const list of lists) {
-        for (let rank = 0; rank < list.length; rank++) {
-            const item = list[rank];
-            const score = 1 / (RRF_K + rank + 1);
-            const existing = map.get(item.id);
-            if (existing) {
-                existing.rrfScore += score;
-            }
-            else {
-                map.set(item.id, { item, rrfScore: score });
-            }
-        }
-    }
-    return [...map.values()]
-        .sort((a, b) => b.rrfScore - a.rrfScore)
-        .map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
-}
-// ============================
-// Search implementation
-// ============================
-export async function executeConversationSearch(params) {
-    const { query, limit, sessionKey: sessionFilter, vectorStore, embeddingService, logger, } = params;
-    logger?.debug?.(`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
-        `sessionFilter=${sessionFilter ?? "(none)"}, ` +
-        `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
-        `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`);
-    if (!query || query.trim().length === 0) {
-        logger?.debug?.(`${TAG} Empty query, returning empty`);
-        return { results: [], total: 0, strategy: "none" };
-    }
-    if (!vectorStore) {
-        logger?.warn?.(`${TAG} VectorStore not available`);
-        return { results: [], total: 0, strategy: "none" };
-    }
-    // ── Determine available capabilities ──
-    const hasEmbedding = !!embeddingService;
-    const hasFts = vectorStore.isFtsAvailable();
-    if (!hasEmbedding && !hasFts) {
-        logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
-        return {
-            results: [],
-            total: 0,
-            strategy: "none",
-            message: "Embedding service is not configured and FTS is not available. " +
-                "Conversation search requires an embedding provider or FTS5 support. " +
-                "Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
-        };
-    }
-    // ── Over-retrieve for later filtering and RRF merging ──
-    const candidateK = sessionFilter ? limit * 4 : limit * 3;
-    // ── Run available search strategies in parallel ──
-    const [ftsItems, vecItems] = await Promise.all([
-        // FTS5 keyword search on L0
-        (async () => {
-            if (!hasFts)
-                return [];
-            try {
-                const ftsQuery = buildFtsQuery(query);
-                if (!ftsQuery) {
-                    logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
-                    return [];
-                }
-                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
-                const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK);
-                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
-                return ftsResults.map((r) => ({
-                    id: r.record_id,
-                    session_key: r.session_key,
-                    role: r.role,
-                    content: r.message_text,
-                    score: r.score,
-                    recorded_at: r.recorded_at,
-                }));
-            }
-            catch (err) {
-                logger?.warn?.(`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
-                return [];
-            }
-        })(),
-        // Vector embedding search on L0
-        (async () => {
-            if (!hasEmbedding)
-                return [];
-            try {
-                logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
-                const queryEmbedding = await embeddingService.embed(query);
-                logger?.debug?.(`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`);
-                const vecResults = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query);
-                logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
-                return vecResults.map((r) => ({
-                    id: r.record_id,
-                    session_key: r.session_key,
-                    role: r.role,
-                    content: r.message_text,
-                    score: r.score,
-                    recorded_at: r.recorded_at,
-                }));
-            }
-            catch (err) {
-                logger?.warn?.(`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
-                return [];
-            }
-        })(),
-    ]);
-    // ── Determine effective strategy ──
-    const ftsOk = ftsItems.length > 0;
-    const vecOk = vecItems.length > 0;
-    let strategy;
-    if (ftsOk && vecOk) {
-        strategy = "hybrid";
-    }
-    else if (vecOk) {
-        strategy = "embedding";
-    }
-    else if (ftsOk) {
-        strategy = "fts";
-    }
-    else {
-        logger?.debug?.(`${TAG} Both search paths returned 0 results`);
-        return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
-    }
-    // ── Merge results ──
-    let results;
-    if (strategy === "hybrid") {
-        results = rrfMergeL0(ftsItems, vecItems);
-        logger?.debug?.(`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`);
-    }
-    else {
-        // Single-source: use whichever list has results (already sorted by score)
-        results = ftsOk ? ftsItems : vecItems;
-    }
-    // ── Apply session key filter ──
-    if (sessionFilter) {
-        const preFilterCount = results.length;
-        results = results.filter((r) => r.session_key === sessionFilter);
-        logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`);
-    }
-    // ── Trim to requested limit ──
-    const trimmed = results.slice(0, limit);
-    logger?.debug?.(`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` +
-        `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`);
-    return {
-        results: trimmed,
-        total: trimmed.length,
-        strategy,
-    };
-}
-// ============================
-// Tool response formatter
-// ============================
-export function formatConversationSearchResponse(result) {
-    if (result.message) {
-        return result.message;
-    }
-    if (result.results.length === 0) {
-        return "No matching conversation messages found.";
-    }
-    const lines = [
-        `Found ${result.total} matching message(s):`,
-        "",
-    ];
-    for (const item of result.results) {
-        const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
-        const dateStr = item.recorded_at ? ` [${item.recorded_at}]` : "";
-        lines.push(`---`);
-        lines.push(`**[${item.role}]** Session: ${item.session_key}${dateStr}${scoreStr}`);
-        lines.push("");
-        lines.push(item.content);
-        lines.push("");
-    }
-    return lines.join("\n");
-}
+/**
+ * conversation_search tool: Agent-callable tool for searching L0 conversation records.
+ *
+ * Supports three search strategies with automatic degradation:
+ *   1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
+ *      merged via Reciprocal Rank Fusion (RRF).
+ *   2. **embedding** — pure vector similarity (when FTS5 is unavailable).
+ *   3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
+ *
+ * The tool is registered via `api.registerTool()` in index.ts.
+ */
+import { buildFtsQuery } from "../store/sqlite.js";
+const TAG = "[memory-tdai][tdai_conversation_search]";
+// ============================
+// RRF (Reciprocal Rank Fusion)
+// ============================
+/** Standard RRF constant from the original RRF paper. */
+const RRF_K = 60;
+/**
+ * Merge multiple ranked lists of `ConversationSearchResultItem` via Reciprocal
+ * Rank Fusion. Items appearing in multiple lists get their RRF scores summed.
+ *
+ * Returns items sorted by descending RRF score. The `score` field of each
+ * returned item is replaced by the RRF score for consistent ranking semantics.
+ */
+function rrfMergeL0(...lists) {
+    const map = new Map();
+    for (const list of lists) {
+        for (let rank = 0; rank < list.length; rank++) {
+            const item = list[rank];
+            const score = 1 / (RRF_K + rank + 1);
+            const existing = map.get(item.id);
+            if (existing) {
+                existing.rrfScore += score;
+            }
+            else {
+                map.set(item.id, { item, rrfScore: score });
+            }
+        }
+    }
+    return [...map.values()]
+        .sort((a, b) => b.rrfScore - a.rrfScore)
+        .map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
+}
+// ============================
+// Search implementation
+// ============================
+export async function executeConversationSearch(params) {
+    const { query, limit, sessionKey: sessionFilter, vectorStore, embeddingService, logger, } = params;
+    logger?.debug?.(`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
+        `sessionFilter=${sessionFilter ?? "(none)"}, ` +
+        `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
+        `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`);
+    if (!query || query.trim().length === 0) {
+        logger?.debug?.(`${TAG} Empty query, returning empty`);
+        return { results: [], total: 0, strategy: "none" };
+    }
+    if (!vectorStore) {
+        logger?.warn?.(`${TAG} VectorStore not available`);
+        return { results: [], total: 0, strategy: "none" };
+    }
+    // ── Determine available capabilities ──
+    const hasEmbedding = !!embeddingService;
+    const hasFts = vectorStore.isFtsAvailable();
+    if (!hasEmbedding && !hasFts) {
+        logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
+        return {
+            results: [],
+            total: 0,
+            strategy: "none",
+            message: "Embedding service is not configured and FTS is not available. " +
+                "Conversation search requires an embedding provider or FTS5 support. " +
+                "Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
+        };
+    }
+    // ── Over-retrieve for later filtering and RRF merging ──
+    const candidateK = sessionFilter ? limit * 4 : limit * 3;
+    // ── Run available search strategies in parallel ──
+    const [ftsItems, vecItems] = await Promise.all([
+        // FTS5 keyword search on L0
+        (async () => {
+            if (!hasFts)
+                return [];
+            try {
+                const ftsQuery = buildFtsQuery(query);
+                if (!ftsQuery) {
+                    logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
+                    return [];
+                }
+                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
+                const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK);
+                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
+                return ftsResults.map((r) => ({
+                    id: r.record_id,
+                    session_key: r.session_key,
+                    role: r.role,
+                    content: r.message_text,
+                    score: r.score,
+                    recorded_at: r.recorded_at,
+                }));
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+                return [];
+            }
+        })(),
+        // Vector embedding search on L0
+        (async () => {
+            if (!hasEmbedding)
+                return [];
+            try {
+                logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
+                const queryEmbedding = await embeddingService.embed(query);
+                logger?.debug?.(`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`);
+                const vecResults = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query);
+                logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
+                return vecResults.map((r) => ({
+                    id: r.record_id,
+                    session_key: r.session_key,
+                    role: r.role,
+                    content: r.message_text,
+                    score: r.score,
+                    recorded_at: r.recorded_at,
+                }));
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+                return [];
+            }
+        })(),
+    ]);
+    // ── Determine effective strategy ──
+    const ftsOk = ftsItems.length > 0;
+    const vecOk = vecItems.length > 0;
+    let strategy;
+    if (ftsOk && vecOk) {
+        strategy = "hybrid";
+    }
+    else if (vecOk) {
+        strategy = "embedding";
+    }
+    else if (ftsOk) {
+        strategy = "fts";
+    }
+    else {
+        logger?.debug?.(`${TAG} Both search paths returned 0 results`);
+        return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
+    }
+    // ── Merge results ──
+    let results;
+    if (strategy === "hybrid") {
+        results = rrfMergeL0(ftsItems, vecItems);
+        logger?.debug?.(`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`);
+    }
+    else {
+        // Single-source: use whichever list has results (already sorted by score)
+        results = ftsOk ? ftsItems : vecItems;
+    }
+    // ── Apply session key filter ──
+    if (sessionFilter) {
+        const preFilterCount = results.length;
+        results = results.filter((r) => r.session_key === sessionFilter);
+        logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`);
+    }
+    // ── Trim to requested limit ──
+    const trimmed = results.slice(0, limit);
+    logger?.debug?.(`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` +
+        `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`);
+    return {
+        results: trimmed,
+        total: trimmed.length,
+        strategy,
+    };
+}
+// ============================
+// Tool response formatter
+// ============================
+export function formatConversationSearchResponse(result) {
+    if (result.message) {
+        return result.message;
+    }
+    if (result.results.length === 0) {
+        return "No matching conversation messages found.";
+    }
+    const lines = [
+        `Found ${result.total} matching message(s):`,
+        "",
+    ];
+    for (const item of result.results) {
+        const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
+        const dateStr = item.recorded_at ? ` [${item.recorded_at}]` : "";
+        lines.push(`---`);
+        lines.push(`**[${item.role}]** Session: ${item.session_key}${dateStr}${scoreStr}`);
+        lines.push("");
+        lines.push(item.content);
+        lines.push("");
+    }
+    return lines.join("\n");
+}