dsh-tdai-memory 0.1.0

package/vendor/tdai/core/record/l1-reader.js

@@ -0,0 +1,178 @@
+/**
+ * L1 Memory Reader: reads persisted L1 memory records.
+ *
+ * Provides two data paths:
+ *
+ * 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()`
+ *    with composite indexes on (session_key, updated_time) and (session_id, updated_time)
+ *    for efficient session-scoped and time-range queries.
+ *
+ * 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from
+ *    `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+const TAG = "[memory-tdai] [l1-reader]";
+// ============================
+// SQLite-based queries (preferred)
+// ============================
+/**
+ * Query L1 memory records from SQLite via VectorStore.
+ *
+ * This is the **preferred** read path — it uses the composite index
+ * `idx_l1_session_updated(session_id, updated_time)` for efficient
+ * session-scoped and time-range queries.
+ *
+ * All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write).
+ *
+ * Falls back to empty array if VectorStore is null or degraded.
+ */
+export async function queryMemoryRecords(vectorStore, filter, logger) {
+    if (!vectorStore) {
+        logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`);
+        return [];
+    }
+    const rows = await vectorStore.queryL1Records(filter);
+    return rows.map(rowToMemoryRecord);
+}
+/**
+ * Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records).
+ */
+function rowToMemoryRecord(row) {
+    let metadata = {};
+    try {
+        metadata = JSON.parse(row.metadata_json);
+    }
+    catch {
+        // malformed JSON — use empty object
+    }
+    // Reconstruct timestamps array from timestamp_start / timestamp_end
+    const timestamps = [];
+    if (row.timestamp_str)
+        timestamps.push(row.timestamp_str);
+    if (row.timestamp_start && row.timestamp_start !== row.timestamp_str)
+        timestamps.push(row.timestamp_start);
+    if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) {
+        timestamps.push(row.timestamp_end);
+    }
+    return {
+        id: row.record_id,
+        content: row.content,
+        type: row.type,
+        priority: row.priority,
+        scene_name: row.scene_name,
+        source_message_ids: [], // not stored in SQLite (vector search doesn't need them)
+        metadata,
+        timestamps,
+        createdAt: row.created_time,
+        updatedAt: row.updated_time,
+        sessionKey: row.session_key,
+        sessionId: row.session_id,
+    };
+}
+// ============================
+// JSONL-based reads (fallback)
+// ============================
+/**
+ * Read all memory records for a session from JSONL files.
+ *
+ * Current naming mode:
+ * - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file)
+ */
+export async function readMemoryRecords(sessionKey, baseDir, logger) {
+    const recordsDir = path.join(baseDir, "records");
+    const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
+    let entries;
+    try {
+        entries = await fs.readdir(recordsDir, { withFileTypes: true });
+    }
+    catch {
+        // Directory doesn't exist yet
+        return [];
+    }
+    const targetFiles = entries
+        .filter((entry) => entry.isFile() && dateFilePattern.test(entry.name))
+        .map((entry) => entry.name)
+        .sort();
+    if (targetFiles.length === 0) {
+        return [];
+    }
+    const records = [];
+    for (const fileName of targetFiles) {
+        const filePath = path.join(recordsDir, fileName);
+        let raw;
+        try {
+            raw = await fs.readFile(filePath, "utf-8");
+        }
+        catch {
+            logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`);
+            continue;
+        }
+        const lines = raw.split("\n").filter((line) => line.trim());
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed.sessionKey !== sessionKey) {
+                    continue;
+                }
+                records.push(parsed);
+            }
+            catch {
+                logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
+            }
+        }
+    }
+    records.sort((a, b) => {
+        const ta = a.updatedAt || a.createdAt || "";
+        const tb = b.updatedAt || b.createdAt || "";
+        return ta.localeCompare(tb);
+    });
+    return records;
+}
+/**
+ * Read ALL memory records across all session JSONL files.
+ */
+export async function readAllMemoryRecords(baseDir, logger) {
+    const recordsDir = path.join(baseDir, "records");
+    try {
+        const files = await fs.readdir(recordsDir);
+        const allRecords = [];
+        for (const file of files) {
+            if (!file.endsWith(".jsonl"))
+                continue;
+            const filePath = path.join(recordsDir, file);
+            try {
+                const raw = await fs.readFile(filePath, "utf-8");
+                const lines = raw.split("\n").filter((line) => line.trim());
+                for (const line of lines) {
+                    try {
+                        allRecords.push(JSON.parse(line));
+                    }
+                    catch {
+                        logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`);
+                    }
+                }
+            }
+            catch {
+                logger?.warn?.(`${TAG} Failed to read ${file}`);
+            }
+        }
+        allRecords.sort((a, b) => {
+            const ta = a.updatedAt || a.createdAt || "";
+            const tb = b.updatedAt || b.createdAt || "";
+            return ta.localeCompare(tb);
+        });
+        return allRecords;
+    }
+    catch {
+        // records/ directory doesn't exist yet
+        return [];
+    }
+}
+// ============================
+// Helpers
+// ============================
+function sanitizeFilename(name) {
+    return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_");
+}

package/vendor/tdai/core/record/l1-writer.js

@@ -0,0 +1,141 @@
+/**
+ * L1 Memory Writer: writes extracted memories to JSONL files.
+ *
+ * File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged).
+ * Each record includes sessionKey for traceability.
+ *
+ * Write strategy:
+ * - JSONL is the append-only persistent store (source of truth for backup/recovery).
+ * - VectorStore (SQLite) is the primary retrieval engine.
+ * - On update/merge, old records are deleted from VectorStore in real-time;
+ *   JSONL is append-only and cleaned up periodically by memory-cleaner.
+ *
+ * Supports store (append), update, merge, and skip operations.
+ *
+ * v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction),
+ * numeric priority, scene_name, source_message_ids, metadata, timestamps.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import crypto from "node:crypto";
+import { formatLocalDate } from "../../utils/time.js";
+const TAG = "[memory-tdai][l1-writer]";
+// ============================
+// Core functions
+// ============================
+/**
+ * Generate a unique memory ID.
+ */
+export function generateMemoryId() {
+    return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
+}
+/**
+ * Write a memory record according to the dedup decision.
+ *
+ * - store: append new record
+ * - update: remove target records + append updated record
+ * - merge: remove target records + append merged record
+ * - skip: do nothing
+ *
+ * v3: supports multi-target removal for update/merge.
+ * v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector).
+ */
+export async function writeMemory(params) {
+    const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
+    if (decision.action === "skip") {
+        logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`);
+        return null;
+    }
+    const now = new Date().toISOString();
+    // Determine final content, type, priority based on action
+    let finalContent;
+    let finalType;
+    let finalPriority;
+    let finalTimestamps;
+    if (decision.action === "merge" || decision.action === "update") {
+        finalContent = decision.merged_content ?? memory.content;
+        finalType = decision.merged_type ?? memory.type;
+        finalPriority = decision.merged_priority ?? memory.priority;
+        finalTimestamps = decision.merged_timestamps ?? [now];
+    }
+    else {
+        // store
+        finalContent = memory.content;
+        finalType = memory.type;
+        finalPriority = memory.priority;
+        finalTimestamps = [now];
+    }
+    const record = {
+        id: decision.record_id || generateMemoryId(),
+        content: finalContent,
+        type: finalType,
+        priority: finalPriority,
+        scene_name: memory.scene_name,
+        source_message_ids: memory.source_message_ids,
+        metadata: memory.metadata,
+        timestamps: finalTimestamps,
+        createdAt: now,
+        updatedAt: now,
+        sessionKey,
+        sessionId: sessionId || "",
+    };
+    const recordsDir = path.join(baseDir, "records");
+    await fs.mkdir(recordsDir, { recursive: true });
+    const shardDate = formatLocalDate(new Date());
+    const filePath = path.join(recordsDir, `${shardDate}.jsonl`);
+    if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) {
+        // Remove target records from VectorStore (real-time deletion for retrieval accuracy).
+        // JSONL is append-only — old records remain in files and are cleaned up periodically
+        // by memory-cleaner (which reconciles against VectorStore as source of truth).
+        if (vectorStore) {
+            try {
+                await vectorStore.deleteL1Batch(decision.target_ids);
+                logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`);
+            }
+            catch (err) {
+                logger?.warn?.(`${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
+        logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`);
+    }
+    else {
+        // store: append a new line
+        await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
+        logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`);
+    }
+    // === Vector Store dual-write ===
+    if (vectorStore) {
+        try {
+            logger?.debug?.(`${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` +
+                `content="${record.content.slice(0, 80)}..."`);
+            let embedding;
+            if (embeddingService) {
+                try {
+                    embedding = await embeddingService.embed(record.content);
+                    logger?.debug?.(`${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` +
+                        `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`);
+                }
+                catch (embedErr) {
+                    // Embedding failed — pass undefined to upsert() which writes
+                    // metadata + FTS only, skipping the vec0 table.
+                    logger?.warn(`${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` +
+                        `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`);
+                }
+            }
+            const upsertOk = await vectorStore.upsertL1(record, embedding);
+            logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`);
+        }
+        catch (err) {
+            // Vector write failure should NOT block the main JSONL write
+            logger?.warn?.(`${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    else {
+        logger?.debug?.(`${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`);
+    }
+    return record;
+}
+// ============================
+// Helpers
+// ============================

package/vendor/tdai/core/report/reporter.js

@@ -0,0 +1,86 @@
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+export const REPORT_CONST = {
+    PLUGIN: "plugin",
+};
+// ── Singleton ──
+let _reporter;
+export function initReporter(opts) {
+    if (_reporter)
+        return;
+    if (!opts.enabled)
+        return;
+    switch (opts.type) {
+        case "local":
+            _reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion);
+            break;
+        // TODO: add new reporter type
+        default:
+            opts.logger.debug?.(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`);
+            break;
+    }
+}
+export function setReporter(reporter) {
+    _reporter = reporter;
+}
+/**
+ * Reset the reporter singleton so that the next `initReporter` call takes effect.
+ * Must be called at plugin re-registration (hot-reload) to pick up config changes.
+ */
+export function resetReporter() {
+    _reporter = undefined;
+}
+export function report(event, data) {
+    if (!_reporter)
+        return;
+    try {
+        _reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data });
+    }
+    catch { /* never block business logic */ }
+}
+// ── LocalReporter (default) ──
+class LocalReporter {
+    logger;
+    instanceId;
+    pluginVersion;
+    constructor(logger, instanceId, pluginVersion) {
+        this.logger = logger;
+        this.instanceId = instanceId;
+        this.pluginVersion = pluginVersion;
+    }
+    reportFunc(category, payload) {
+        try {
+            this.logger.info(JSON.stringify({
+                tag: "METRIC",
+                category,
+                plugin: "memory-tdai",
+                instanceId: this.instanceId,
+                pluginVersion: this.pluginVersion,
+                ts: new Date().toISOString(),
+                ...payload,
+            }));
+        }
+        catch { /* swallow */ }
+    }
+}
+// ── Instance ID (persisted per-install) ──
+let _instanceIdCache;
+export async function getOrCreateInstanceId(pluginDataDir) {
+    if (_instanceIdCache)
+        return _instanceIdCache;
+    const idFile = path.join(pluginDataDir, ".metadata", "instance_id");
+    try {
+        const existing = (await fs.readFile(idFile, "utf-8")).trim();
+        if (existing) {
+            _instanceIdCache = existing;
+            return existing;
+        }
+    }
+    catch { /* file doesn't exist */ }
+    const newId = randomUUID();
+    await fs.mkdir(path.dirname(idFile), { recursive: true });
+    await fs.writeFile(idFile, newId, "utf-8");
+    _instanceIdCache = newId;
+    return newId;
+}

package/vendor/tdai/core/scene/filename-normalizer.js

@@ -0,0 +1,161 @@
+/**
+ * Scene filename normalizer.
+ *
+ * Defensive engineering layer that runs *after* the LLM writes scene_blocks/*.md
+ * and *before* syncSceneIndex(). Even though the prompt forbids spaces and
+ * punctuation in filenames, LLMs occasionally produce names like
+ * `Daily Rhythm in Shanghai.md`. Such names break:
+ *   - Markdown navigation refs that downstream tools parse with `\S+\.md`
+ *     (e.g. health-checker's scene reference detection).
+ *   - Shell-based tools that iterate scene files without quoting.
+ *   - URL/path encoding consumers (COS object keys etc).
+ *
+ * This module renames offenders to a canonical form on disk and lets every
+ * other consumer (PersonaGenerator, recall, profile-sync) read the already
+ * sanitized name from scene_index.json — no additional changes needed.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+/**
+ * Normalize a single scene filename.
+ *
+ * Rules:
+ *   - Preserves the `.md` extension (case-insensitive match, lowercased).
+ *   - Whitespace runs (spaces / tabs) → single hyphen.
+ *   - Strips quotes, brackets, and ASCII punctuation that breaks shell/markdown.
+ *   - Collapses consecutive separators (`-`, `_`, `.`).
+ *   - Trims leading / trailing separators.
+ *   - Falls back to `"scene"` if the stem becomes empty.
+ *
+ * Allowed character set after normalization (informally):
+ *   Unicode letters/numbers, hyphen, underscore, dot.
+ *
+ * Examples:
+ *   "Daily Rhythm in Shanghai.md"  → "Daily-Rhythm-in-Shanghai.md"
+ *   "日常生活 健康管理.md"          → "日常生活-健康管理.md"
+ *   "Coffee (Yirgacheffe).md"      → "Coffee-Yirgacheffe.md"
+ *   "  spaced  .md"                → "spaced.md"
+ *   ".MD"                          → "scene.md"
+ *   "已经规范.md"                   → "已经规范.md" (no-op)
+ */
+export function normalizeSceneFilename(name) {
+    if (!name)
+        return "scene.md";
+    // Strip directory components defensively — we only normalize the basename.
+    const base = name.replace(/^.*[\\/]/, "");
+    // Detect & strip `.md` (case-insensitive). Always re-emit lowercase `.md`.
+    const lower = base.toLowerCase();
+    const hasMd = lower.endsWith(".md");
+    const stem = hasMd ? base.slice(0, -3) : base;
+    const safe = stem
+        // Replace whitespace runs (incl. NBSP, full-width space) with `-`
+        .replace(/[\s\u00A0\u3000]+/g, "-")
+        // Drop quotes, brackets, and punctuation known to break shells/markdown.
+        // Keep Unicode letters/numbers and the safe separators `-`, `_`, `.`.
+        .replace(/[()[\]{}<>'"`,;:!?*|/\\=&%$#@^~+]/g, "")
+        // Collapse consecutive separators.
+        .replace(/-{2,}/g, "-")
+        .replace(/_{2,}/g, "_")
+        .replace(/\.{2,}/g, ".")
+        // Trim leading / trailing separators.
+        .replace(/^[-_.]+|[-_.]+$/g, "");
+    return (safe || "scene") + ".md";
+}
+/**
+ * Return whether a filename already matches its normalized form.
+ * Faster than computing the normalized form when callers only need a yes/no.
+ */
+export function isNormalizedSceneFilename(name) {
+    return normalizeSceneFilename(name) === name;
+}
+/**
+ * Resolve a non-conflicting target path inside `dir` for the desired filename.
+ *
+ * If `desired` (e.g. `Daily-Rhythm.md`) already exists in `dir`, append a
+ * numeric suffix `-2`, `-3`, ... before the `.md` extension until a free slot
+ * is found. Caller may also pass `excludePath` to ignore a known existing file
+ * (e.g. the source path of an in-flight rename, when source != target).
+ */
+export async function resolveUniqueScenePath(dir, desired, excludePath) {
+    const target = path.join(dir, desired);
+    if (!(await pathExists(target)) || target === excludePath)
+        return target;
+    const ext = ".md";
+    const stem = desired.endsWith(ext) ? desired.slice(0, -ext.length) : desired;
+    // Bound the search to keep this defensive (LLMs rarely produce hundreds of
+    // colliding names; if they do, surface the failure rather than spin).
+    for (let i = 2; i < 1000; i++) {
+        const candidate = path.join(dir, `${stem}-${i}${ext}`);
+        if (!(await pathExists(candidate)) || candidate === excludePath) {
+            return candidate;
+        }
+    }
+    throw new Error(`resolveUniqueScenePath: could not find a free slot for ${desired} in ${dir} after 1000 attempts`);
+}
+async function pathExists(p) {
+    try {
+        await fs.access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Walk a scene_blocks directory and rename any `.md` file whose basename does
+ * not match `normalizeSceneFilename(basename)`.
+ *
+ * Safe to call multiple times: subsequent invocations are no-ops once names
+ * have stabilized.
+ *
+ * Notes:
+ *   - Non-`.md` files are ignored (the LLM tool surface is restricted to .md,
+ *     but the directory may contain transient artifacts).
+ *   - Empty / soft-deleted files are not pre-filtered here; the SceneExtractor
+ *     cleanup pass handles those before / after this call as appropriate.
+ *   - Failures on individual entries are logged via the optional logger and
+ *     do not abort the loop — index sync should still see the remaining files.
+ */
+export async function normalizeSceneFilenames(blocksDir, logger) {
+    const result = { renamed: 0, skipped: 0, renames: [] };
+    let entries;
+    try {
+        entries = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md"));
+    }
+    catch {
+        return result;
+    }
+    for (const file of entries) {
+        const normalized = normalizeSceneFilename(file);
+        if (normalized === file) {
+            result.skipped++;
+            continue;
+        }
+        const from = path.join(blocksDir, file);
+        let to;
+        try {
+            to = await resolveUniqueScenePath(blocksDir, normalized, from);
+        }
+        catch (err) {
+            logger?.warn?.(`[filename-normalizer] could not resolve unique target for ${file}: ${err instanceof Error ? err.message : String(err)}`);
+            result.skipped++;
+            continue;
+        }
+        if (to === from) {
+            // Filesystem already matched (e.g. case-insensitive FS where source and
+            // target collapse to the same inode); treat as a no-op.
+            result.skipped++;
+            continue;
+        }
+        try {
+            await fs.rename(from, to);
+            result.renamed++;
+            result.renames.push({ from: file, to: path.basename(to) });
+            logger?.debug?.(`[filename-normalizer] renamed: ${file} → ${path.basename(to)}`);
+        }
+        catch (err) {
+            logger?.warn?.(`[filename-normalizer] rename failed (${file} → ${path.basename(to)}): ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return result;
+}