dsh-tdai-memory 0.1.0

package/vendor/tdai/core/tools/memory-search.js

@@ -0,0 +1,207 @@
+/**
+ * memory_search tool: Agent-callable tool for searching L1 memory records.
+ *
+ * Supports three search strategies with automatic degradation:
+ *   1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
+ *      merged via Reciprocal Rank Fusion (RRF).
+ *   2. **embedding** — pure vector similarity (when FTS5 is unavailable).
+ *   3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
+ *
+ * The tool is registered via `api.registerTool()` in index.ts.
+ */
+import { buildFtsQuery } from "../store/sqlite.js";
+const TAG = "[memory-tdai][tdai_memory_search]";
+// ============================
+// RRF (Reciprocal Rank Fusion)
+// ============================
+/** Standard RRF constant from the original RRF paper. */
+const RRF_K = 60;
+/**
+ * Merge multiple ranked lists of `MemorySearchResultItem` via Reciprocal Rank
+ * Fusion. Items appearing in multiple lists get their RRF scores summed.
+ *
+ * Returns items sorted by descending RRF score. The `score` field of each
+ * returned item is replaced by the RRF score for consistent ranking semantics.
+ */
+function rrfMergeL1(...lists) {
+    const map = new Map();
+    for (const list of lists) {
+        for (let rank = 0; rank < list.length; rank++) {
+            const item = list[rank];
+            const score = 1 / (RRF_K + rank + 1);
+            const existing = map.get(item.id);
+            if (existing) {
+                existing.rrfScore += score;
+            }
+            else {
+                map.set(item.id, { item, rrfScore: score });
+            }
+        }
+    }
+    return [...map.values()]
+        .sort((a, b) => b.rrfScore - a.rrfScore)
+        .map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
+}
+// ============================
+// Search implementation
+// ============================
+export async function executeMemorySearch(params) {
+    const { query, limit, type: typeFilter, scene: sceneFilter, vectorStore, embeddingService, logger, } = params;
+    logger?.debug?.(`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
+        `typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` +
+        `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
+        `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`);
+    if (!query || query.trim().length === 0) {
+        logger?.debug?.(`${TAG} Empty query, returning empty`);
+        return { results: [], total: 0, strategy: "none" };
+    }
+    if (!vectorStore) {
+        logger?.warn?.(`${TAG} VectorStore not available`);
+        return { results: [], total: 0, strategy: "none" };
+    }
+    // ── Determine available capabilities ──
+    const hasEmbedding = !!embeddingService;
+    const hasFts = vectorStore.isFtsAvailable();
+    if (!hasEmbedding && !hasFts) {
+        logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
+        return {
+            results: [],
+            total: 0,
+            strategy: "none",
+            message: "Embedding service is not configured and FTS is not available. " +
+                "Memory search requires an embedding provider or FTS5 support. " +
+                "Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
+        };
+    }
+    // ── Over-retrieve for later filtering and RRF merging ──
+    const candidateK = limit * 3;
+    // ── Run available search strategies in parallel ──
+    const [ftsItems, vecItems] = await Promise.all([
+        // FTS5 keyword search
+        (async () => {
+            if (!hasFts)
+                return [];
+            try {
+                const ftsQuery = buildFtsQuery(query);
+                if (!ftsQuery) {
+                    logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
+                    return [];
+                }
+                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
+                const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK);
+                logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
+                return ftsResults.map((r) => ({
+                    id: r.record_id,
+                    content: r.content,
+                    type: r.type,
+                    priority: r.priority,
+                    scene_name: r.scene_name,
+                    score: r.score,
+                    created_at: r.timestamp_start,
+                    updated_at: r.timestamp_end,
+                }));
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+                return [];
+            }
+        })(),
+        // Vector embedding search
+        (async () => {
+            if (!hasEmbedding)
+                return [];
+            try {
+                logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
+                const queryEmbedding = await embeddingService.embed(query);
+                logger?.debug?.(`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`);
+                const vecResults = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query);
+                logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
+                return vecResults.map((r) => ({
+                    id: r.record_id,
+                    content: r.content,
+                    type: r.type,
+                    priority: r.priority,
+                    scene_name: r.scene_name,
+                    score: r.score,
+                    created_at: r.timestamp_start,
+                    updated_at: r.timestamp_end,
+                }));
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+                return [];
+            }
+        })(),
+    ]);
+    // ── Determine effective strategy ──
+    const ftsOk = ftsItems.length > 0;
+    const vecOk = vecItems.length > 0;
+    let strategy;
+    if (ftsOk && vecOk) {
+        strategy = "hybrid";
+    }
+    else if (vecOk) {
+        strategy = "embedding";
+    }
+    else if (ftsOk) {
+        strategy = "fts";
+    }
+    else {
+        logger?.debug?.(`${TAG} Both search paths returned 0 results`);
+        return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
+    }
+    // ── Merge results ──
+    let results;
+    if (strategy === "hybrid") {
+        results = rrfMergeL1(ftsItems, vecItems);
+        logger?.debug?.(`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`);
+    }
+    else {
+        // Single-source: use whichever list has results (already sorted by score)
+        results = ftsOk ? ftsItems : vecItems;
+    }
+    // ── Apply secondary filters (type, scene) ──
+    const preFilterCount = results.length;
+    if (typeFilter) {
+        results = results.filter((r) => r.type === typeFilter);
+        logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`);
+    }
+    if (sceneFilter) {
+        const normalizedScene = sceneFilter.toLowerCase();
+        results = results.filter((r) => r.scene_name.toLowerCase().includes(normalizedScene));
+        logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`);
+    }
+    // ── Trim to requested limit ──
+    const trimmed = results.slice(0, limit);
+    logger?.debug?.(`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` +
+        `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`);
+    return {
+        results: trimmed,
+        total: trimmed.length,
+        strategy,
+    };
+}
+// ============================
+// Tool response formatter
+// ============================
+export function formatSearchResponse(result) {
+    if (result.message) {
+        return result.message;
+    }
+    if (result.results.length === 0) {
+        return "No matching memories found.";
+    }
+    const lines = [
+        `Found ${result.total} matching memories:`,
+        "",
+    ];
+    for (const item of result.results) {
+        const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
+        const sceneStr = item.scene_name ? ` [scene: ${item.scene_name}]` : "";
+        const priorityStr = item.priority >= 0 ? ` (priority: ${item.priority})` : " (global instruction)";
+        lines.push(`- **[${item.type}]**${priorityStr}${sceneStr}${scoreStr}`);
+        lines.push(`  ${item.content}`);
+        lines.push("");
+    }
+    return lines.join("\n");
+}

package/vendor/tdai/core/types.js

@@ -0,0 +1,12 @@
+/**
+ * TDAI Core — Host-neutral type definitions and abstract interfaces.
+ *
+ * These types define the boundary between TDAI Core (memory algorithms)
+ * and the host environment (OpenClaw, Hermes, standalone Gateway, etc.).
+ *
+ * Design principles:
+ * 1. TDAI Core depends ONLY on these interfaces — never on a specific host.
+ * 2. Each host provides its own implementation of HostAdapter + LLMRunnerFactory.
+ * 3. RuntimeContext is the single source of truth for session/user identity.
+ */
+export {};

package/vendor/tdai/utils/backup.js

@@ -0,0 +1,189 @@
+/**
+ * BackupManager: generic file/directory backup utility.
+ *
+ * Provides two backup modes:
+ *   - `backupFile(src, category, tag, maxKeep)` — copy a single file
+ *   - `backupDirectory(src, category, tag, maxKeep)` — copy an entire directory
+ *
+ * All backups land under `<backupRoot>/<category>/` with timestamped names.
+ * After each backup, entries beyond `maxKeep` are automatically pruned
+ * (oldest first, by lexicographic order on the timestamp-embedded name).
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+export class BackupManager {
+    backupRoot;
+    /**
+     * @param backupRoot - Absolute path to the root backup directory
+     *                     (e.g. `<dataDir>/.backup`).
+     */
+    constructor(backupRoot) {
+        this.backupRoot = backupRoot;
+    }
+    /**
+     * Backup a single file.
+     *
+     * Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>.<ext>`
+     *
+     * @param srcFile   - Absolute path to the source file
+     * @param category  - Logical grouping (e.g. "persona")
+     * @param tag       - Additional identifier (e.g. "offset42")
+     * @param maxKeep   - Max backup files to retain in this category (0 = unlimited)
+     */
+    async backupFile(srcFile, category, tag, maxKeep) {
+        try {
+            await fs.access(srcFile);
+        }
+        catch {
+            return; // Source file doesn't exist, nothing to backup
+        }
+        const destDir = path.join(this.backupRoot, category);
+        await fs.mkdir(destDir, { recursive: true });
+        const ext = path.extname(srcFile); // e.g. ".md"
+        const timestamp = formatTimestamp(new Date());
+        const destName = `${category}_${timestamp}_${tag}${ext}`;
+        await fs.copyFile(srcFile, path.join(destDir, destName));
+        if (maxKeep > 0) {
+            await pruneOldEntries(destDir, maxKeep, "file");
+        }
+    }
+    /**
+     * Backup an entire directory (shallow copy of all files).
+     *
+     * Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>/`
+     *
+     * @param srcDir    - Absolute path to the source directory
+     * @param category  - Logical grouping (e.g. "scene_blocks")
+     * @param tag       - Additional identifier (e.g. "offset42")
+     * @param maxKeep   - Max backup directories to retain in this category (0 = unlimited)
+     */
+    async backupDirectory(srcDir, category, tag, maxKeep) {
+        let entries;
+        try {
+            entries = await fs.readdir(srcDir, { withFileTypes: true });
+        }
+        catch {
+            return; // Source directory doesn't exist
+        }
+        // Only backup regular files (skip subdirectories to avoid EISDIR errors)
+        const files = entries.filter((e) => e.isFile()).map((e) => e.name);
+        if (files.length === 0)
+            return;
+        const parentDir = path.join(this.backupRoot, category);
+        const timestamp = formatTimestamp(new Date());
+        const destDir = path.join(parentDir, `${category}_${timestamp}_${tag}`);
+        await fs.mkdir(destDir, { recursive: true });
+        for (const file of files) {
+            await fs.copyFile(path.join(srcDir, file), path.join(destDir, file));
+        }
+        if (maxKeep > 0) {
+            await pruneOldEntries(parentDir, maxKeep, "directory");
+        }
+    }
+    /**
+     * Find the latest backup directory for a category.
+     *
+     * Backup directory names are `<category>_<timestamp>_<tag>` where the
+     * timestamp is `YYYYMMDD_HHmmss` (lexicographic order = chronological order),
+     * so the lexicographically largest entry is the most recent one.
+     *
+     * @param category - Logical grouping (e.g. "scene_blocks")
+     * @returns Absolute path to the latest backup directory, or undefined if none.
+     */
+    async findLatestBackup(category) {
+        const parentDir = path.join(this.backupRoot, category);
+        let entries;
+        try {
+            entries = await fs.readdir(parentDir, { withFileTypes: true });
+        }
+        catch {
+            return undefined; // No backup directory yet
+        }
+        const dirs = entries.filter((e) => e.isDirectory()).map((e) => e.name);
+        if (dirs.length === 0)
+            return undefined;
+        dirs.sort(); // ascending — oldest first; last = newest
+        return path.join(parentDir, dirs[dirs.length - 1]);
+    }
+    /**
+     * Restore the latest backup of `category` into `destDir`.
+     *
+     * Strategy:
+     *   1. Find the latest backup directory; if none exists, do nothing
+     *      (fail-soft: never clobber the destination when there is no
+     *      ground truth to restore from).
+     *   2. Wipe `destDir` and recreate it.
+     *   3. Copy every regular file from the backup directory into `destDir`.
+     *
+     * @param category - Logical grouping (e.g. "scene_blocks")
+     * @param destDir  - Absolute path to the directory to restore into
+     * @returns `{ restored: true, from }` when a backup was applied,
+     *          `{ restored: false }` when no backup was found.
+     * @throws  Lets fs errors during wipe/copy propagate so callers can decide
+     *          whether to fail-soft (log) or fail-hard.
+     */
+    async restoreLatestDirectory(category, destDir) {
+        const from = await this.findLatestBackup(category);
+        if (!from)
+            return { restored: false };
+        // Wipe the destination first so any partial LLM writes are removed,
+        // then recreate the directory and copy regular files back.
+        await fs.rm(destDir, { recursive: true, force: true });
+        await fs.mkdir(destDir, { recursive: true });
+        const entries = await fs.readdir(from, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isFile())
+                continue;
+            await fs.copyFile(path.join(from, entry.name), path.join(destDir, entry.name));
+        }
+        return { restored: true, from };
+    }
+}
+// ============================
+// Helpers
+// ============================
+function formatTimestamp(d) {
+    const pad = (n) => String(n).padStart(2, "0");
+    return [
+        d.getFullYear(),
+        pad(d.getMonth() + 1),
+        pad(d.getDate()),
+        "_",
+        pad(d.getHours()),
+        pad(d.getMinutes()),
+        pad(d.getSeconds()),
+    ].join("");
+}
+/**
+ * Keep only the newest `maxKeep` entries in a directory.
+ * Entries are sorted by name ascending (oldest first) since backup names
+ * embed timestamps, so lexicographic order = chronological order.
+ *
+ * @param dir     - Directory containing the backup entries
+ * @param maxKeep - Number of entries to retain
+ * @param kind    - "file" to unlink, "directory" to rm -rf
+ */
+async function pruneOldEntries(dir, maxKeep, kind) {
+    let entries;
+    try {
+        entries = await fs.readdir(dir);
+    }
+    catch {
+        return;
+    }
+    entries.sort(); // ascending — oldest first
+    const toRemove = entries.slice(0, Math.max(0, entries.length - maxKeep));
+    for (const name of toRemove) {
+        try {
+            if (kind === "file") {
+                await fs.unlink(path.join(dir, name));
+            }
+            else {
+                await fs.rm(path.join(dir, name), { recursive: true, force: true });
+            }
+        }
+        catch {
+            // best-effort
+        }
+    }
+}