dsh-tdai-memory 0.3.2 → 0.3.4

package/vendor/tdai/core/record/l1-reader.js

@@ -1,178 +1,178 @@
-/**
- * L1 Memory Reader: reads persisted L1 memory records.
- *
- * Provides two data paths:
- *
- * 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()`
- *    with composite indexes on (session_key, updated_time) and (session_id, updated_time)
- *    for efficient session-scoped and time-range queries.
- *
- * 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from
- *    `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded.
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-const TAG = "[memory-tdai] [l1-reader]";
-// ============================
-// SQLite-based queries (preferred)
-// ============================
-/**
- * Query L1 memory records from SQLite via VectorStore.
- *
- * This is the **preferred** read path — it uses the composite index
- * `idx_l1_session_updated(session_id, updated_time)` for efficient
- * session-scoped and time-range queries.
- *
- * All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write).
- *
- * Falls back to empty array if VectorStore is null or degraded.
- */
-export async function queryMemoryRecords(vectorStore, filter, logger) {
-    if (!vectorStore) {
-        logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`);
-        return [];
-    }
-    const rows = await vectorStore.queryL1Records(filter);
-    return rows.map(rowToMemoryRecord);
-}
-/**
- * Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records).
- */
-function rowToMemoryRecord(row) {
-    let metadata = {};
-    try {
-        metadata = JSON.parse(row.metadata_json);
-    }
-    catch {
-        // malformed JSON — use empty object
-    }
-    // Reconstruct timestamps array from timestamp_start / timestamp_end
-    const timestamps = [];
-    if (row.timestamp_str)
-        timestamps.push(row.timestamp_str);
-    if (row.timestamp_start && row.timestamp_start !== row.timestamp_str)
-        timestamps.push(row.timestamp_start);
-    if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) {
-        timestamps.push(row.timestamp_end);
-    }
-    return {
-        id: row.record_id,
-        content: row.content,
-        type: row.type,
-        priority: row.priority,
-        scene_name: row.scene_name,
-        source_message_ids: [], // not stored in SQLite (vector search doesn't need them)
-        metadata,
-        timestamps,
-        createdAt: row.created_time,
-        updatedAt: row.updated_time,
-        sessionKey: row.session_key,
-        sessionId: row.session_id,
-    };
-}
-// ============================
-// JSONL-based reads (fallback)
-// ============================
-/**
- * Read all memory records for a session from JSONL files.
- *
- * Current naming mode:
- * - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file)
- */
-export async function readMemoryRecords(sessionKey, baseDir, logger) {
-    const recordsDir = path.join(baseDir, "records");
-    const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
-    let entries;
-    try {
-        entries = await fs.readdir(recordsDir, { withFileTypes: true });
-    }
-    catch {
-        // Directory doesn't exist yet
-        return [];
-    }
-    const targetFiles = entries
-        .filter((entry) => entry.isFile() && dateFilePattern.test(entry.name))
-        .map((entry) => entry.name)
-        .sort();
-    if (targetFiles.length === 0) {
-        return [];
-    }
-    const records = [];
-    for (const fileName of targetFiles) {
-        const filePath = path.join(recordsDir, fileName);
-        let raw;
-        try {
-            raw = await fs.readFile(filePath, "utf-8");
-        }
-        catch {
-            logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`);
-            continue;
-        }
-        const lines = raw.split("\n").filter((line) => line.trim());
-        for (let i = 0; i < lines.length; i++) {
-            const line = lines[i];
-            try {
-                const parsed = JSON.parse(line);
-                if (parsed.sessionKey !== sessionKey) {
-                    continue;
-                }
-                records.push(parsed);
-            }
-            catch {
-                logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
-            }
-        }
-    }
-    records.sort((a, b) => {
-        const ta = a.updatedAt || a.createdAt || "";
-        const tb = b.updatedAt || b.createdAt || "";
-        return ta.localeCompare(tb);
-    });
-    return records;
-}
-/**
- * Read ALL memory records across all session JSONL files.
- */
-export async function readAllMemoryRecords(baseDir, logger) {
-    const recordsDir = path.join(baseDir, "records");
-    try {
-        const files = await fs.readdir(recordsDir);
-        const allRecords = [];
-        for (const file of files) {
-            if (!file.endsWith(".jsonl"))
-                continue;
-            const filePath = path.join(recordsDir, file);
-            try {
-                const raw = await fs.readFile(filePath, "utf-8");
-                const lines = raw.split("\n").filter((line) => line.trim());
-                for (const line of lines) {
-                    try {
-                        allRecords.push(JSON.parse(line));
-                    }
-                    catch {
-                        logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`);
-                    }
-                }
-            }
-            catch {
-                logger?.warn?.(`${TAG} Failed to read ${file}`);
-            }
-        }
-        allRecords.sort((a, b) => {
-            const ta = a.updatedAt || a.createdAt || "";
-            const tb = b.updatedAt || b.createdAt || "";
-            return ta.localeCompare(tb);
-        });
-        return allRecords;
-    }
-    catch {
-        // records/ directory doesn't exist yet
-        return [];
-    }
-}
-// ============================
-// Helpers
-// ============================
-function sanitizeFilename(name) {
-    return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_");
-}
+/**
+ * L1 Memory Reader: reads persisted L1 memory records.
+ *
+ * Provides two data paths:
+ *
+ * 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()`
+ *    with composite indexes on (session_key, updated_time) and (session_id, updated_time)
+ *    for efficient session-scoped and time-range queries.
+ *
+ * 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from
+ *    `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+const TAG = "[memory-tdai] [l1-reader]";
+// ============================
+// SQLite-based queries (preferred)
+// ============================
+/**
+ * Query L1 memory records from SQLite via VectorStore.
+ *
+ * This is the **preferred** read path — it uses the composite index
+ * `idx_l1_session_updated(session_id, updated_time)` for efficient
+ * session-scoped and time-range queries.
+ *
+ * All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write).
+ *
+ * Falls back to empty array if VectorStore is null or degraded.
+ */
+export async function queryMemoryRecords(vectorStore, filter, logger) {
+    if (!vectorStore) {
+        logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`);
+        return [];
+    }
+    const rows = await vectorStore.queryL1Records(filter);
+    return rows.map(rowToMemoryRecord);
+}
+/**
+ * Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records).
+ */
+function rowToMemoryRecord(row) {
+    let metadata = {};
+    try {
+        metadata = JSON.parse(row.metadata_json);
+    }
+    catch {
+        // malformed JSON — use empty object
+    }
+    // Reconstruct timestamps array from timestamp_start / timestamp_end
+    const timestamps = [];
+    if (row.timestamp_str)
+        timestamps.push(row.timestamp_str);
+    if (row.timestamp_start && row.timestamp_start !== row.timestamp_str)
+        timestamps.push(row.timestamp_start);
+    if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) {
+        timestamps.push(row.timestamp_end);
+    }
+    return {
+        id: row.record_id,
+        content: row.content,
+        type: row.type,
+        priority: row.priority,
+        scene_name: row.scene_name,
+        source_message_ids: [], // not stored in SQLite (vector search doesn't need them)
+        metadata,
+        timestamps,
+        createdAt: row.created_time,
+        updatedAt: row.updated_time,
+        sessionKey: row.session_key,
+        sessionId: row.session_id,
+    };
+}
+// ============================
+// JSONL-based reads (fallback)
+// ============================
+/**
+ * Read all memory records for a session from JSONL files.
+ *
+ * Current naming mode:
+ * - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file)
+ */
+export async function readMemoryRecords(sessionKey, baseDir, logger) {
+    const recordsDir = path.join(baseDir, "records");
+    const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
+    let entries;
+    try {
+        entries = await fs.readdir(recordsDir, { withFileTypes: true });
+    }
+    catch {
+        // Directory doesn't exist yet
+        return [];
+    }
+    const targetFiles = entries
+        .filter((entry) => entry.isFile() && dateFilePattern.test(entry.name))
+        .map((entry) => entry.name)
+        .sort();
+    if (targetFiles.length === 0) {
+        return [];
+    }
+    const records = [];
+    for (const fileName of targetFiles) {
+        const filePath = path.join(recordsDir, fileName);
+        let raw;
+        try {
+            raw = await fs.readFile(filePath, "utf-8");
+        }
+        catch {
+            logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`);
+            continue;
+        }
+        const lines = raw.split("\n").filter((line) => line.trim());
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed.sessionKey !== sessionKey) {
+                    continue;
+                }
+                records.push(parsed);
+            }
+            catch {
+                logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
+            }
+        }
+    }
+    records.sort((a, b) => {
+        const ta = a.updatedAt || a.createdAt || "";
+        const tb = b.updatedAt || b.createdAt || "";
+        return ta.localeCompare(tb);
+    });
+    return records;
+}
+/**
+ * Read ALL memory records across all session JSONL files.
+ */
+export async function readAllMemoryRecords(baseDir, logger) {
+    const recordsDir = path.join(baseDir, "records");
+    try {
+        const files = await fs.readdir(recordsDir);
+        const allRecords = [];
+        for (const file of files) {
+            if (!file.endsWith(".jsonl"))
+                continue;
+            const filePath = path.join(recordsDir, file);
+            try {
+                const raw = await fs.readFile(filePath, "utf-8");
+                const lines = raw.split("\n").filter((line) => line.trim());
+                for (const line of lines) {
+                    try {
+                        allRecords.push(JSON.parse(line));
+                    }
+                    catch {
+                        logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`);
+                    }
+                }
+            }
+            catch {
+                logger?.warn?.(`${TAG} Failed to read ${file}`);
+            }
+        }
+        allRecords.sort((a, b) => {
+            const ta = a.updatedAt || a.createdAt || "";
+            const tb = b.updatedAt || b.createdAt || "";
+            return ta.localeCompare(tb);
+        });
+        return allRecords;
+    }
+    catch {
+        // records/ directory doesn't exist yet
+        return [];
+    }
+}
+// ============================
+// Helpers
+// ============================
+function sanitizeFilename(name) {
+    return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_");
+}

package/vendor/tdai/core/record/l1-writer.js

@@ -1,141 +1,141 @@
-/**
- * L1 Memory Writer: writes extracted memories to JSONL files.
- *
- * File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged).
- * Each record includes sessionKey for traceability.
- *
- * Write strategy:
- * - JSONL is the append-only persistent store (source of truth for backup/recovery).
- * - VectorStore (SQLite) is the primary retrieval engine.
- * - On update/merge, old records are deleted from VectorStore in real-time;
- *   JSONL is append-only and cleaned up periodically by memory-cleaner.
- *
- * Supports store (append), update, merge, and skip operations.
- *
- * v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction),
- * numeric priority, scene_name, source_message_ids, metadata, timestamps.
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-import crypto from "node:crypto";
-import { formatLocalDate } from "../../utils/time.js";
-const TAG = "[memory-tdai][l1-writer]";
-// ============================
-// Core functions
-// ============================
-/**
- * Generate a unique memory ID.
- */
-export function generateMemoryId() {
-    return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
-}
-/**
- * Write a memory record according to the dedup decision.
- *
- * - store: append new record
- * - update: remove target records + append updated record
- * - merge: remove target records + append merged record
- * - skip: do nothing
- *
- * v3: supports multi-target removal for update/merge.
- * v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector).
- */
-export async function writeMemory(params) {
-    const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
-    if (decision.action === "skip") {
-        logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`);
-        return null;
-    }
-    const now = new Date().toISOString();
-    // Determine final content, type, priority based on action
-    let finalContent;
-    let finalType;
-    let finalPriority;
-    let finalTimestamps;
-    if (decision.action === "merge" || decision.action === "update") {
-        finalContent = decision.merged_content ?? memory.content;
-        finalType = decision.merged_type ?? memory.type;
-        finalPriority = decision.merged_priority ?? memory.priority;
-        finalTimestamps = decision.merged_timestamps ?? [now];
-    }
-    else {
-        // store
-        finalContent = memory.content;
-        finalType = memory.type;
-        finalPriority = memory.priority;
-        finalTimestamps = [now];
-    }
-    const record = {
-        id: decision.record_id || generateMemoryId(),
-        content: finalContent,
-        type: finalType,
-        priority: finalPriority,
-        scene_name: memory.scene_name,
-        source_message_ids: memory.source_message_ids,
-        metadata: memory.metadata,
-        timestamps: finalTimestamps,
-        createdAt: now,
-        updatedAt: now,
-        sessionKey,
-        sessionId: sessionId || "",
-    };
-    const recordsDir = path.join(baseDir, "records");
-    await fs.mkdir(recordsDir, { recursive: true });
-    const shardDate = formatLocalDate(new Date());
-    const filePath = path.join(recordsDir, `${shardDate}.jsonl`);
-    if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) {
-        // Remove target records from VectorStore (real-time deletion for retrieval accuracy).
-        // JSONL is append-only — old records remain in files and are cleaned up periodically
-        // by memory-cleaner (which reconciles against VectorStore as source of truth).
-        if (vectorStore) {
-            try {
-                await vectorStore.deleteL1Batch(decision.target_ids);
-                logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`);
-            }
-            catch (err) {
-                logger?.warn?.(`${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`);
-            }
-        }
-        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
-        logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`);
-    }
-    else {
-        // store: append a new line
-        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
-        logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`);
-    }
-    // === Vector Store dual-write ===
-    if (vectorStore) {
-        try {
-            logger?.debug?.(`${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` +
-                `content="${record.content.slice(0, 80)}..."`);
-            let embedding;
-            if (embeddingService) {
-                try {
-                    embedding = await embeddingService.embed(record.content);
-                    logger?.debug?.(`${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` +
-                        `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
-                }
-                catch (embedErr) {
-                    // Embedding failed — pass undefined to upsert() which writes
-                    // metadata + FTS only, skipping the vec0 table.
-                    logger?.warn(`${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` +
-                        `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`);
-                }
-            }
-            const upsertOk = await vectorStore.upsertL1(record, embedding);
-            logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`);
-        }
-        catch (err) {
-            // Vector write failure should NOT block the main JSONL write
-            logger?.warn?.(`${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    else {
-        logger?.debug?.(`${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`);
-    }
-    return record;
-}
-// ============================
-// Helpers
-// ============================
+/**
+ * L1 Memory Writer: writes extracted memories to JSONL files.
+ *
+ * File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged).
+ * Each record includes sessionKey for traceability.
+ *
+ * Write strategy:
+ * - JSONL is the append-only persistent store (source of truth for backup/recovery).
+ * - VectorStore (SQLite) is the primary retrieval engine.
+ * - On update/merge, old records are deleted from VectorStore in real-time;
+ *   JSONL is append-only and cleaned up periodically by memory-cleaner.
+ *
+ * Supports store (append), update, merge, and skip operations.
+ *
+ * v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction),
+ * numeric priority, scene_name, source_message_ids, metadata, timestamps.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import { formatLocalDate } from "../../utils/time.js";
+const TAG = "[memory-tdai][l1-writer]";
+// ============================
+// Core functions
+// ============================
+/**
+ * Generate a unique memory ID.
+ */
+export function generateMemoryId() {
+    return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
+}
+/**
+ * Write a memory record according to the dedup decision.
+ *
+ * - store: append new record
+ * - update: remove target records + append updated record
+ * - merge: remove target records + append merged record
+ * - skip: do nothing
+ *
+ * v3: supports multi-target removal for update/merge.
+ * v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector).
+ */
+export async function writeMemory(params) {
+    const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
+    if (decision.action === "skip") {
+        logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`);
+        return null;
+    }
+    const now = new Date().toISOString();
+    // Determine final content, type, priority based on action
+    let finalContent;
+    let finalType;
+    let finalPriority;
+    let finalTimestamps;
+    if (decision.action === "merge" || decision.action === "update") {
+        finalContent = decision.merged_content ?? memory.content;
+        finalType = decision.merged_type ?? memory.type;
+        finalPriority = decision.merged_priority ?? memory.priority;
+        finalTimestamps = decision.merged_timestamps ?? [now];
+    }
+    else {
+        // store
+        finalContent = memory.content;
+        finalType = memory.type;
+        finalPriority = memory.priority;
+        finalTimestamps = [now];
+    }
+    const record = {
+        id: decision.record_id || generateMemoryId(),
+        content: finalContent,
+        type: finalType,
+        priority: finalPriority,
+        scene_name: memory.scene_name,
+        source_message_ids: memory.source_message_ids,
+        metadata: memory.metadata,
+        timestamps: finalTimestamps,
+        createdAt: now,
+        updatedAt: now,
+        sessionKey,
+        sessionId: sessionId || "",
+    };
+    const recordsDir = path.join(baseDir, "records");
+    await fs.mkdir(recordsDir, { recursive: true });
+    const shardDate = formatLocalDate(new Date());
+    const filePath = path.join(recordsDir, `${shardDate}.jsonl`);
+    if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) {
+        // Remove target records from VectorStore (real-time deletion for retrieval accuracy).
+        // JSONL is append-only — old records remain in files and are cleaned up periodically
+        // by memory-cleaner (which reconciles against VectorStore as source of truth).
+        if (vectorStore) {
+            try {
+                await vectorStore.deleteL1Batch(decision.target_ids);
+                logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`);
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
+        logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`);
+    }
+    else {
+        // store: append a new line
+        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
+        logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`);
+    }
+    // === Vector Store dual-write ===
+    if (vectorStore) {
+        try {
+            logger?.debug?.(`${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` +
+                `content="${record.content.slice(0, 80)}..."`);
+            let embedding;
+            if (embeddingService) {
+                try {
+                    embedding = await embeddingService.embed(record.content);
+                    logger?.debug?.(`${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` +
+                        `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
+                }
+                catch (embedErr) {
+                    // Embedding failed — pass undefined to upsert() which writes
+                    // metadata + FTS only, skipping the vec0 table.
+                    logger?.warn(`${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` +
+                        `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`);
+                }
+            }
+            const upsertOk = await vectorStore.upsertL1(record, embedding);
+            logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`);
+        }
+        catch (err) {
+            // Vector write failure should NOT block the main JSONL write
+            logger?.warn?.(`${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    else {
+        logger?.debug?.(`${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`);
+    }
+    return record;
+}
+// ============================
+// Helpers
+// ============================