@memory-river/core 0.2.0

package/dist/store/status-manager.js

@@ -0,0 +1,235 @@
+/**
+ * StatusManager — 記憶狀態單一所有人（P0-3）
+ *
+ * 所有 status 變更（active / superseded / deprecated / archived / trashed）必須經過此模組。
+ * 內部同時更新 metadata.status + row.status（雙欄位同步），並寫入 audit log。
+ *
+ * 設計原則：
+ * - 純 class，只依賴 MemoryStore
+ * - changeStatus() 走 store.update()（自動進 WAL exactly-once 保護）
+ * - 所有成功/失敗都記 audit row
+ * - changeStatusBatch() 逐筆 try/catch，單筆失敗不中斷整 batch
+ */
+import { randomUUID } from "node:crypto";
+export class StatusManager {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    // ========================================================================
+    // 主寫入 API — 所有 status 變更走這
+    // ========================================================================
+    async changeStatus(req) {
+        const auditRowId = randomUUID();
+        let fromStatus = null;
+        try {
+            // 1. 取出當前記憶
+            const entry = await this.store.getById(req.memoryId, true);
+            if (!entry) {
+                // 記憶不存在 — 記 audit（失敗）並回傳
+                await this.safeRecordAudit({
+                    memoryId: req.memoryId,
+                    fromStatus: null,
+                    toStatus: req.toStatus,
+                    reason: req.reason,
+                    source: req.source,
+                    supersededBy: req.supersededBy ?? null,
+                    meta: req.meta ? JSON.stringify(req.meta) : null,
+                    canonicalKey: null,
+                    partial: false,
+                }, auditRowId);
+                return {
+                    ok: false,
+                    memoryId: req.memoryId,
+                    fromStatus: null,
+                    toStatus: req.toStatus,
+                    auditRowId,
+                    error: 'memory_not_found',
+                };
+            }
+            // 2. 解析當前 metadata，取得 fromStatus
+            const meta = this.parseMetadata(entry.metadata);
+            fromStatus = meta.status ?? null;
+            // 3. 更新 metadata 欄位
+            meta.status = req.toStatus;
+            // 設置時間戳和額外欄位（保持與現有寫入者行為一致）
+            if (req.toStatus === 'superseded') {
+                meta.supersededAt = Date.now();
+                if (req.supersededBy) {
+                    meta.supersededBy = req.supersededBy;
+                }
+            }
+            else if (req.toStatus === 'deprecated') {
+                meta.deprecatedAt = Date.now();
+                if (req.supersededBy) {
+                    meta.supersededBy = req.supersededBy;
+                }
+            }
+            else if (req.toStatus === 'trashed') {
+                meta.trashedAt = Date.now();
+            }
+            // 合併額外 meta 資訊（如 consolidationReason）
+            if (req.meta) {
+                for (const [key, value] of Object.entries(req.meta)) {
+                    meta[key] = value;
+                }
+            }
+            // 4. 同時更新 metadata + row.status（單一 store.update() 呼叫 → 單一 WAL entry）
+            let partial = false;
+            try {
+                await this.store.update(req.memoryId, {
+                    metadata: JSON.stringify(meta),
+                    status: req.toStatus,
+                });
+            }
+            catch (updateErr) {
+                // row.status 補寫可能失敗（例如 LanceDB schema 不含 status column）
+                // 退回只更新 metadata 的安全路徑
+                console.warn(`[StatusManager] Dual-field update failed; falling back to metadata-only: ${updateErr.message}`);
+                try {
+                    await this.store.update(req.memoryId, {
+                        metadata: JSON.stringify(meta),
+                    });
+                    partial = true; // metadata 成功但 row.status 未寫入
+                }
+                catch (fallbackErr) {
+                    // 兩條路徑都失敗 — 記 audit 並回傳錯誤
+                    await this.safeRecordAudit({
+                        memoryId: req.memoryId,
+                        fromStatus: fromStatus,
+                        toStatus: req.toStatus,
+                        reason: req.reason,
+                        source: req.source,
+                        supersededBy: req.supersededBy ?? null,
+                        meta: JSON.stringify({ error: fallbackErr.message, ...(req.meta ?? {}) }),
+                        canonicalKey: null,
+                        partial: false,
+                    }, auditRowId);
+                    return {
+                        ok: false,
+                        memoryId: req.memoryId,
+                        fromStatus,
+                        toStatus: req.toStatus,
+                        auditRowId,
+                        error: `update_failed: ${fallbackErr.message}`,
+                    };
+                }
+            }
+            // 5. 寫入 audit log（成功）
+            await this.safeRecordAudit({
+                memoryId: req.memoryId,
+                fromStatus: fromStatus,
+                toStatus: req.toStatus,
+                reason: req.reason,
+                source: req.source,
+                supersededBy: req.supersededBy ?? null,
+                meta: req.meta ? JSON.stringify(req.meta) : null,
+                canonicalKey: null,
+                partial,
+            }, auditRowId);
+            return {
+                ok: true,
+                memoryId: req.memoryId,
+                fromStatus,
+                toStatus: req.toStatus,
+                auditRowId,
+            };
+        }
+        catch (err) {
+            // 未預期錯誤 — 嘗試記 audit
+            await this.safeRecordAudit({
+                memoryId: req.memoryId,
+                fromStatus: fromStatus,
+                toStatus: req.toStatus,
+                reason: req.reason,
+                source: req.source,
+                supersededBy: req.supersededBy ?? null,
+                meta: JSON.stringify({ error: err.message, ...(req.meta ?? {}) }),
+                canonicalKey: null,
+                partial: false,
+            }, auditRowId);
+            return {
+                ok: false,
+                memoryId: req.memoryId,
+                fromStatus,
+                toStatus: req.toStatus,
+                auditRowId,
+                error: err.message,
+            };
+        }
+    }
+    // ========================================================================
+    // 批次 API — NightConsolidator 用
+    // ========================================================================
+    /**
+     * 逐筆呼叫 changeStatus()，單筆失敗不中斷整 batch。
+     * 每 20 筆輸出 progress log，避免凌晨 batch 卡住時無感。
+     */
+    async changeStatusBatch(reqs) {
+        const results = [];
+        const total = reqs.length;
+        const mainReason = reqs[0]?.reason ?? 'unknown';
+        for (let i = 0; i < total; i++) {
+            try {
+                const result = await this.changeStatus(reqs[i]);
+                results.push(result);
+            }
+            catch (err) {
+                // changeStatus 內部已有 try/catch，理論上不會走到這
+                // 但防禦性程式設計：記錄失敗結果，繼續下一筆
+                results.push({
+                    ok: false,
+                    memoryId: reqs[i].memoryId,
+                    fromStatus: null,
+                    toStatus: reqs[i].toStatus,
+                    auditRowId: '',
+                    error: `batch_unexpected: ${err.message}`,
+                });
+            }
+            // 每 20 筆輸出 progress log
+            if ((i + 1) % 20 === 0 || i === total - 1) {
+                const succeeded = results.filter(r => r.ok).length;
+                const failed = results.length - succeeded;
+                console.log(`[StatusManager] batch progress: ${i + 1}/${total} (reason=${mainReason}, ok=${succeeded}, fail=${failed})`);
+            }
+        }
+        return results;
+    }
+    // ========================================================================
+    // 查詢 audit log（觀測用）
+    // ========================================================================
+    async queryAuditLog(opts) {
+        return this.store.queryStatusAudit(opts);
+    }
+    /**
+     * 新記憶建立時，row.status 與 metadata.status 已在 store.store() 一次寫入；
+     * 這裡只補 audit log，避免再次觸發 LanceDB update metadata parser。
+     */
+    async recordCreation(req) {
+        return this.store.recordCreationAudit(req);
+    }
+    // ========================================================================
+    // 內部工具
+    // ========================================================================
+    /**
+     * 安全寫入 audit log — 失敗只 warn，不影響主流程
+     */
+    async safeRecordAudit(audit, id) {
+        try {
+            await this.store.recordStatusAudit({ ...audit, id });
+        }
+        catch (err) {
+            console.warn(`[StatusManager] Failed to write audit log (memoryId=${audit.memoryId}): ${err.message}`);
+        }
+    }
+    parseMetadata(metaStr) {
+        if (!metaStr)
+            return {};
+        try {
+            return typeof metaStr === 'string' ? JSON.parse(metaStr) : metaStr;
+        }
+        catch {
+            return {};
+        }
+    }
+}

package/dist/store/store-v4.d.ts

@@ -0,0 +1,339 @@
+/**
+ * LanceDB Store - RAM + SSD Dual-Write Architecture with WAL
+ * memory-river v4
+ *
+ * 核心原則:
+ * - RAM Disk (/dev/shm) 為主要讀寫目標(極速)
+ * - SSD 為異步備份(持久化)
+ * - WAL (Write-Ahead Log) 保護 update/delete 一致性
+ * - store() WAL + RAM 同步，SSD 異步寫入
+ * - update/delete 必須 WAL 先行 → 雙寫 → WAL commit
+ * - 讀取全部走 RAM(速度優先)
+ * - crash recovery replays every WAL change at-least-once, including unacknowledged deletes
+ */
+import type { ConcentratorStat, MemoryEntry, MemorySearchResult, MemoryHealth, SkillCapsule, StatusAuditRow, SubsystemEffectivenessEvent, SubsystemEffectivenessQueryFilter, SubsystemEffectivenessRow, TranscriptWatermarkRow } from "../types.js";
+export declare class SchemaViolationError extends Error {
+    readonly violations: string[];
+    constructor(message: string, violations: string[]);
+}
+declare const DEFAULT_HEALTH_CONFIG: {
+    initialScore: number;
+    coreCategories: string[];
+    coreImportanceThreshold: number;
+    skillDecayFactor: number;
+};
+export declare function sqlStringLiteral(value: string): string;
+export declare function normalizeLanceUpdateValues(values: Record<string, unknown>): Record<string, unknown>;
+type DecayOptions = {
+    coreCategories?: string[];
+    coreImportanceThreshold?: number;
+    skillCapsuleProtection?: boolean;
+    dryRun?: boolean;
+    deleteWith?: (id: string) => Promise<boolean>;
+    maxDelete?: number;
+    maxDecay?: number;
+};
+type StoreEntryInput = Omit<MemoryEntry, "id" | "createdAt" | "updatedAt" | "textTokens"> & {
+    creationAuditSource?: string | null;
+    creationAuditMeta?: Record<string, unknown>;
+};
+type ConcentratorStatInput = Omit<ConcentratorStat, "id" | "createdAt"> & {
+    id?: string;
+    createdAt?: number;
+};
+type ConflictStatInput = {
+    ts?: number;
+    operationName: string;
+    callerPath?: string | null;
+    attempt: number;
+    finalOutcome: string;
+    fragmentId?: string | number | null;
+};
+type NightConsolidationStatInput = {
+    id?: string;
+    runId: string;
+    phase: string;
+    ts?: number;
+    outcome?: string | null;
+    durationMs?: number | null;
+    candidateCount?: number | null;
+    scannedCount?: number | null;
+    decisionCount?: number | null;
+    mergeCount?: number | null;
+    deleteCount?: number | null;
+    deprecatedCount?: number | null;
+    updateCount?: number | null;
+    keepCount?: number | null;
+    attemptedCount?: number | null;
+    failedCount?: number | null;
+    batchIndex?: number | null;
+    batchSize?: number | null;
+    driftMs?: number | null;
+    scheduledFor?: number | null;
+    errorMessage?: string | null;
+    metadata?: string | Record<string, unknown> | null;
+};
+export declare class MemoryStore {
+    private readonly dbPath;
+    private readonly ramDbPath;
+    private readonly vectorDim;
+    private readonly ssdRecoveryProbeIntervalMs;
+    private ramDb;
+    private ramTable;
+    private ssdDb;
+    private ssdTable;
+    private subsystemEffectivenessRamTable;
+    private subsystemEffectivenessSsdTable;
+    private concentratorStatsRamTable;
+    private concentratorStatsSsdTable;
+    private conflictStatsRamTable;
+    private conflictStatsSsdTable;
+    private nightConsolidationStatsRamTable;
+    private nightConsolidationStatsSsdTable;
+    private walMetadataRamTable;
+    private walMetadataSsdTable;
+    private statusAuditLogRamTable;
+    private statusAuditLogSsdTable;
+    private transcriptWatermarkRamTable;
+    private transcriptWatermarkSsdTable;
+    private shutdownHooks;
+    private initPromise;
+    private healthConfig;
+    private readonly _embedder?;
+    private readonly walDir;
+    private readonly walPath;
+    private walRecovered;
+    private walTxnCounter;
+    private lastCheckpointTxnId;
+    private walCheckpointInitialized;
+    private walCheckpointUpdateQueue;
+    private ssdAvailable;
+    private ssdConsecutiveFailures;
+    private ssdRecoveryProbeTimer;
+    private ssdRecoveryProbeInFlight;
+    private ftsAvailable;
+    private readonly ssdFallback;
+    constructor(dbPath: string, // SSD 持久化路徑
+    ramDbPath: string, // RAM Disk 路徑
+    vectorDim: number, walFileOrHealthConfig?: string | typeof DEFAULT_HEALTH_CONFIG, healthConfigOrEmbedder?: typeof DEFAULT_HEALTH_CONFIG | {
+        embed(text: string): Promise<number[]>;
+    }, embedder?: {
+        embed(text: string): Promise<number[]>;
+    }, ssdRecoveryProbeIntervalMs?: number);
+    get db(): any;
+    /** SSD 持久化連接（供 GraphStore 共用） */
+    get ssd(): any;
+    onShutdown(fn: () => Promise<void>): void;
+    ensureInitialized(): Promise<void>;
+    private doInitialize;
+    private initTable;
+    private ensureHasHooksColumnAndIndex;
+    private ensureFtsIndex;
+    /**
+     * 寫入一筆 WAL 條目,自動附加單調遞增的 txnId。
+     * ⚠️ txnId 是精確的 replay 順序控制依據,不可重複使用。
+     */
+    private appendWal;
+    private restoreWalTxnCounter;
+    /**
+     * 更新 wal_metadata 表的 last_committed_txn_id。
+     * 寫入後即表示這筆 txnId 及之前的操作已安全落地。
+     * ⚠️ 必須在 WAL commit line 寫入磁碟成功後才能更新(嚴格 ordered)。
+     */
+    private updateWalMetadata;
+    /**
+     * 查詢目前已 commit 的最大 txnId(用於 recovery 起點)。
+     * 回傳 0 表示尚無任何 commit 記錄。
+     */
+    private getLastCommittedTxnId;
+    private commitWal;
+    /**
+     * WAL Recovery - 從上次已知的安全 checkpoint 開始 replay。
+     *
+     * At-least-once recovery:
+     * 1. 所有已進 WAL 的變更都以冪等方式 replay（包括尚未 ack 的操作）
+     * 2. 每筆 replay 成功後,立即更新 last_committed_txn_id(寫入 wal_metadata)
+     * 3. 若 replay 到一半再次當機,下次重啟會重試保留的 WAL 條目
+     *
+     * ⚠️ txnId 小的先 replay(嚴格ordered),避免因果鏈順序錯亂。
+     */
+    private recoverFromWal;
+    private rewriteWal;
+    private clearWal;
+    private handleSsdSuccess;
+    private handleSsdError;
+    private startSsdRecoveryProbe;
+    private probeSsdRecovery;
+    private stopSsdRecoveryProbe;
+    private ensureConcentratorStatsTable;
+    private ensureConcentratorStatsTables;
+    private ensureConflictStatsTable;
+    private ensureConflictStatsTables;
+    private ensureNightConsolidationStatsTable;
+    private ensureNightConsolidationStatsTables;
+    private ensureWalMetadataTable;
+    private ensureWalMetadataTables;
+    private cleanupLegacyWalMetadataRow;
+    private tokenizeChinese;
+    private toJsVector;
+    private parseMetadata;
+    private hasHooksFromMetadata;
+    /**
+     * 🛡️ LanceDB Optimistic Concurrency 緩解器
+     * 遇到 'Commit conflict' 時,隨機等待後重試 (Jitter Backoff)
+     * 這對於雙寫與頻繁背景任務至關重要。
+     */
+    private lancedbRetry;
+    private extractFragmentId;
+    private extractCallerPath;
+    private extractCallerPathFrames;
+    private recordConflictStatBestEffort;
+    private recordConflictStatRow;
+    recordConflictStat(stat: ConflictStatInput): Promise<void>;
+    recordNightConsolidationStat(stat: NightConsolidationStatInput): Promise<void>;
+    private validateEntrySchema;
+    private rejectSchemaViolation;
+    /**
+     * store() - append-only,風險最低
+     * RAM 同步寫(快),SSD 異步寫(fire-and-forget)
+     * WAL 在回應前同步落地，SSD 若未完成可於重啟時補寫
+     */
+    private static readonly MAX_TEXT_LENGTH;
+    store(entry: StoreEntryInput): Promise<MemoryEntry>;
+    /**
+     * update() - 危險操作,WAL 先行
+     * 1. append WAL(先寫 log)
+     * 2. RAM + SSD 同時更新
+     * 3. WAL commit
+     */
+    update(id: string, updates: Partial<Omit<MemoryEntry, "id" | "createdAt">>, newVector?: number[]): Promise<boolean>;
+    /**
+     * delete() - 危險操作,WAL 先行
+     * 1. WAL 先行
+     * 2. RAM + SSD 同時刪
+     * 3. WAL commit
+     */
+    delete(id: string): Promise<boolean>;
+    private static readonly UUID_RE;
+    private validateId;
+    getById(id: string, includeAllStatus?: boolean): Promise<MemoryEntry | null>;
+    getByIds(ids: string[], includeAllStatus?: boolean): Promise<MemoryEntry[]>;
+    count(): Promise<number>;
+    private ensureSubsystemEffectivenessTable;
+    initSubsystemEffectivenessTable(): Promise<void>;
+    recordSubsystemEffectiveness(event: SubsystemEffectivenessEvent): Promise<void>;
+    querySubsystemEffectiveness(filter?: SubsystemEffectivenessQueryFilter): Promise<SubsystemEffectivenessRow[]>;
+    recordConcentratorStat(stat: ConcentratorStatInput): Promise<void>;
+    queryConcentratorStats(opts?: {
+        since?: number;
+        provider?: ConcentratorStat["provider"];
+        outcome?: ConcentratorStat["outcome"];
+        canonicalKey?: string;
+        limit?: number;
+    }): Promise<ConcentratorStat[]>;
+    getRecentConcentratorStats(limit?: number): Promise<ConcentratorStat[]>;
+    private ensureStatusAuditLogTable;
+    private ensureStatusAuditLogTables;
+    /**
+     * P0-3 Schema Migration: 確保 memories table 有 `status` column (Utf8, nullable)。
+     * 舊資料全部預設為 'active'。使用 LanceDB addColumns API，idempotent。
+     */
+    private ensureStatusColumn;
+    recordStatusAudit(audit: Omit<StatusAuditRow, "id" | "timestamp"> & {
+        id?: string;
+        timestamp?: number;
+    }): Promise<string>;
+    recordCreationAudit(req: {
+        memoryId: string;
+        source: string;
+        meta?: Record<string, unknown>;
+    }): Promise<string>;
+    /**
+     * 查詢 status_audit_log（觀測用）。
+     *
+     * @note volatile — 只讀 RAM，重啟後遺失歷史。
+     *   長期 audit 查詢需另開 P1 任務支援 readFromSsd 選項。
+     */
+    queryStatusAudit(opts?: {
+        memoryId?: string;
+        since?: number;
+        source?: string;
+        limit?: number;
+    }): Promise<StatusAuditRow[]>;
+    private initializeCreationStatus;
+    private safeRecordCreationAudit;
+    private ensureTranscriptWatermarkTable;
+    private ensureTranscriptWatermarkTables;
+    setTranscriptWatermark(canonicalKey: string, sessionId: string | null, lineCount: number): Promise<void>;
+    getTranscriptWatermark(canonicalKey: string): Promise<(TranscriptWatermarkRow & {
+        sessionId: string | null;
+    }) | null>;
+    vectorSearch(vector: number[], limit?: number): Promise<MemorySearchResult[]>;
+    ftsSearch(query: string, limit?: number): Promise<MemorySearchResult[]>;
+    /**
+     * hybridVectorSearch - 統一混合搜尋(向量 + FTS + RRF Fusion)
+     * 替代所有純向量搜尋調用
+     */
+    hybridVectorSearch(query: string, limit?: number): Promise<MemorySearchResult[]>;
+    private _embed;
+    /**
+     * hybridSkillCapsuleSearch - 技能膠囊混合搜尋(hybridVectorSearch + keyword match)
+     * 用於 autoRecall 結果組裝階段
+     */
+    hybridSkillCapsuleSearch(query: string, limit?: number, filters?: {
+        capsuleVersion?: number;
+        status?: string;
+    }): Promise<SkillCapsule[]>;
+    query(predicate: string, limit?: number, includeAllStatus?: boolean): Promise<MemoryEntry[]>;
+    queryAll(limit?: number): Promise<MemoryEntry[]>;
+    queryHookBearing(): Promise<MemoryEntry[]>;
+    queryAllWithMeta(limit?: number): Promise<Array<MemoryEntry & {
+        metadataObj: Record<string, any>;
+    }>>;
+    recordMemoryRecalls(entries: Array<MemoryEntry | {
+        id: string;
+    }>, recalledAt?: number): Promise<void>;
+    getRecallStats(memoryId: string): Promise<{
+        lastRecalledAt: number | null;
+        recallCount: number;
+        ageInDays: number;
+        dormancyInDays: number | null;
+    } | null>;
+    /**
+     * searchBySlotKey - 精準查詢同 slotKey 的所有版本
+     * 用於 Structured Slot 的 supersedes 鏈查找
+     */
+    searchBySlotKey(slotKey: string): Promise<MemoryEntry[]>;
+    boostHealth(id: string): Promise<boolean>;
+    /**
+     * 批次更新多筆記錄的 metadata(不走一筆一筆 WAL,直接寫 table + 單筆 batch WAL entry)。
+     * 用於 decayMemories 批次收集完後一次性寫入,減少 O(N) WAL I/O。
+     *
+     * ⚠️ 犧牲了逐筆 WAL entry 的精細度,但換來批次效能。
+     *   萬一寫入中途當機,batch 內部分記錄可能未落地,需靠下次 recovery 重跑。
+     */
+    batchUpdateMemories(updates: Array<{
+        id: string;
+        metadata: string;
+    }>): Promise<void>;
+    decayMemories(decayPerRun?: number, deleteThreshold?: number, options?: DecayOptions): Promise<{
+        decayed: number;
+        deleted: number;
+        coreProtected: number;
+        wouldDecay: number;
+        wouldDelete: number;
+        deferredDecay: number;
+        deferredDelete: number;
+        deleteCandidateSummary: {
+            count: number;
+            firstId: string | null;
+            lastId: string | null;
+            minCreatedAt: number | null;
+            maxCreatedAt: number | null;
+            createdAtByDay: Record<string, number>;
+        };
+    }>;
+    getHealthStats(): Promise<any>;
+    shutdown(): Promise<void>;
+}
+export type { MemoryEntry, MemorySearchResult, MemoryHealth, SkillCapsule };