@modusensus/dsh-mneme 0.3.8 → 0.4.0

package/src/store.js

@@ -13,6 +13,8 @@ CREATE TABLE IF NOT EXISTS memories (
   archived    INTEGER NOT NULL DEFAULT 0,
   source      TEXT,
   embedding   TEXT,
+  last_accessed_at  TEXT,
+  _full_content     TEXT,
   created_at  TEXT NOT NULL,
   updated_at  TEXT NOT NULL
 );
@@ -38,7 +40,8 @@ CREATE TABLE IF NOT EXISTS dream_runs (
   applied        INTEGER NOT NULL DEFAULT 0,
   summary_stored INTEGER NOT NULL DEFAULT 0,
   receipt        TEXT NOT NULL,
-  policy_epoch   INTEGER NOT NULL DEFAULT 0  -- 裁决规则版本：规则升级后旧裁决降级为历史证据
+  policy_epoch   INTEGER NOT NULL DEFAULT 0,  -- 裁决规则版本：规则升级后旧裁决降级为历史证据
+  run_type       TEXT NOT NULL DEFAULT 'auto'  -- auto | sleep：睡眠周期的审计区分
 );
 CREATE INDEX IF NOT EXISTS idx_dream_runs_created ON dream_runs(created_at);
 
@@ -180,13 +183,13 @@ CREATE TABLE IF NOT EXISTS mirror_state (
   last_error TEXT,                   -- 最近失败原因
   last_attempt TEXT,                 -- 最近尝试时间（ISO）
   success_at TEXT,                   -- 最近成功时间（ISO）
-  generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0 AND generation <= 9007199254740991),         -- 期望的同步轮次（desired）
-  applied_generation INTEGER NOT NULL DEFAULT 0 CHECK (applied_generation >= 0 AND applied_generation <= 9007199254740991), -- 已成功应用的轮次
+  generation INTEGER NOT NULL DEFAULT 0 CHECK (generation >= 0 AND generation <= 9007199254740991 AND generation = CAST(generation AS INTEGER)),         -- 期望的同步轮次（desired）
+  applied_generation INTEGER NOT NULL DEFAULT 0 CHECK (applied_generation >= 0 AND applied_generation <= 9007199254740991 AND applied_generation = CAST(applied_generation AS INTEGER)), -- 已成功应用的轮次
   type_status TEXT                               -- JSON: 逐 type 状态 {type: {dirty, applied_gen, last_error}}
 );
 `;
 
-const TYPES = new Set(["preference", "project", "decision", "history", "summary"]);
+const TYPES = new Set(["preference", "project", "decision", "history", "summary", "pattern"]);
 
 // Per-type mirror sync receipts (peer blocker 4): a type is either committed
 // (file written + fence applied), failed (last sync round errored for it), or
@@ -227,7 +230,9 @@ function toRow(row) {
     archived: row.archived === 1,
     source: row.source ?? undefined,
     created_at: row.created_at,
-    updated_at: row.updated_at
+    updated_at: row.updated_at,
+    last_accessed_at: row.last_accessed_at ?? undefined,
+    _full_content: row._full_content ?? undefined
   };
 }
 
@@ -248,7 +253,8 @@ function toDreamRun(row) {
     applied: row.applied,
     summary_stored: row.summary_stored === 1,
     receipt: row.receipt,
-    policy_epoch: row.policy_epoch ?? 0
+    policy_epoch: row.policy_epoch ?? 0,
+    run_type: row.run_type ?? "auto"
   };
 }
 
@@ -391,11 +397,14 @@ function parseJsonArray(raw) {
 
 export function createStore(path) {
   const db = new DatabaseSync(path);
-  db.exec("PRAGMA journal_mode = WAL;");
-  // Concurrent writers (peer probe: 8 independent processes) must wait for the
-  // write lock instead of failing immediately with SQLITE_BUSY — otherwise the
-  // atomic generation increment loses whole writes, not just increments.
+  // Set busy_timeout BEFORE the journal-mode switch (audit peer: 8-process WAL
+  // init). Switching a fresh DB to WAL takes an exclusive lock; when several
+  // processes open the same path simultaneously, that lock can fail with
+  // SQLITE_BUSY before the timeout is armed. With the timeout installed first,
+  // the WAL transition (and every later write) blocks and retries instead of
+  // failing outright, so concurrent init converges to a stable 447/447.
   db.exec("PRAGMA busy_timeout = 5000;");
+  db.exec("PRAGMA journal_mode = WAL;");
   db.exec(SCHEMA);
 
   // Schema migrations for legacy databases (idempotent).
@@ -406,12 +415,21 @@ export function createStore(path) {
   if (!columns.includes("embedding")) {
     db.exec("ALTER TABLE memories ADD COLUMN embedding TEXT");
   }
+  if (!columns.includes("last_accessed_at")) {
+    db.exec("ALTER TABLE memories ADD COLUMN last_accessed_at TEXT");
+  }
+  if (!columns.includes("_full_content")) {
+    db.exec("ALTER TABLE memories ADD COLUMN _full_content TEXT");
+  }
 
   // Legacy dream_runs without policy_epoch → backfill with the default epoch.
   const dreamCols = db.prepare("PRAGMA table_info(dream_runs)").all().map((c) => c.name);
   if (!dreamCols.includes("policy_epoch")) {
     db.exec("ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
   }
+  if (!dreamCols.includes("run_type")) {
+    db.exec("ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'");
+  }
 
   // Legacy mirror_state without v0.3.6 generation columns → add each missing
   // column idempotently (old DBs open cleanly, no data loss).
@@ -426,6 +444,24 @@ export function createStore(path) {
     db.exec("ALTER TABLE mirror_state ADD COLUMN type_status TEXT");
   }
 
+  // Audit peer F: a legacy DB may hold a non-integer generation/applied_generation
+  // (pre-v0.3.9 the JS gate truncated with Math.trunc and SQLite's CHECK only
+  // enforced >= 0). Such a value is ambiguous — it cannot map to a real applied
+  // round — so surface it as a hard error on open instead of silently reading it
+  // as a coherent generation. Fail-closed: the operator must repair or reset the
+  // state row rather than continue with a lie.
+  for (const col of ["generation", "applied_generation"]) {
+    const bad = db.prepare(
+      `SELECT id FROM mirror_state WHERE ${col} IS NOT NULL AND ${col} != CAST(${col} AS INTEGER) LIMIT 1`
+    ).get();
+    if (bad) {
+      throw new RangeError(
+        `mirror_state.${col} holds a non-integer value (legacy dirty state); ` +
+        `repair or reset the row before opening this database`
+      );
+    }
+  }
+
   // Per-instance monotonic timestamp guard: consecutive writes within the same
   // millisecond must still produce strictly increasing timestamps (test asserts
   // updated_at != created_at). State lives in the store closure, not module scope.
@@ -552,24 +588,35 @@ export function createStore(path) {
     const embedding = patch.embedding !== undefined
       ? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
       : existing.embedding ?? null;
-    const result = db.prepare(
-      `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=?
-       WHERE id=? AND updated_at=?`
-    ).run(
-      type,
-      patch.title ?? existing.title,
-      patch.content ?? existing.content,
-      JSON.stringify(patch.tags ?? existing.tags),
-      Number.isInteger(patch.importance) ? patch.importance : existing.importance,
-      patch.source !== undefined ? patch.source : (existing.source ?? null),
-      embedding,
-      now,
-      id,
-      expectedUpdatedAt
-    );
-    if (result.changes === 0) return undefined; // CAS miss: a concurrent write won
-    // Only bump desired generation on a successful CAS — a miss writes nothing.
-    runAtomically(() => { incrementGeneration(); });
+    // The CAS UPDATE and the desired-generation bump must commit together (audit
+    // peer A): if the UPDATE autocommits first and the process dies before the
+    // increment, the store is mutated while generation == applied_generation and
+    // dirty == false — recoverMirror sees no debt and the mirror stays stale.
+    // Wrapping both in one transaction means a CAS miss rolls back cleanly too
+    // (no write, no generation bump).
+    let applied = false;
+    runAtomically(() => {
+      const result = db.prepare(
+        `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=?
+         WHERE id=? AND updated_at=?`
+      ).run(
+        type,
+        patch.title ?? existing.title,
+        patch.content ?? existing.content,
+        JSON.stringify(patch.tags ?? existing.tags),
+        Number.isInteger(patch.importance) ? patch.importance : existing.importance,
+        patch.source !== undefined ? patch.source : (existing.source ?? null),
+        embedding,
+        now,
+        id,
+        expectedUpdatedAt
+      );
+      if (result.changes === 0) return; // CAS miss: a concurrent write won
+      // Only bump desired generation on a successful CAS — a miss writes nothing.
+      incrementGeneration();
+      applied = true;
+    });
+    if (!applied) return undefined;
     return getById(id);
   }
 
@@ -591,6 +638,69 @@ export function createStore(path) {
     return getById(id);
   }
 
+  // --- sleep-mode storage support (v0.4.0) ---------------------------------
+  // touchLastAccess stamps the read time on recall/inject paths. It deliberately
+  // does NOT bump the mirror generation: reads must not mark the mirror dirty.
+  function touchLastAccess(id, at) {
+    if (!getById(id)) return false;
+    db.prepare("UPDATE memories SET last_accessed_at = ? WHERE id = ?")
+      .run(at ?? nowIso(), id);
+    return true;
+  }
+
+  // Shrink an aged memory to `summary`, parking its full body in _full_content.
+  // Idempotent: an already-demoted memory (non-null _full_content) is left
+  // untouched. minRefTimeMs guards the fast path — if last_accessed_at moved
+  // after the caller's snapshot (>= minRefTimeMs), the memory is hot again and
+  // is skipped. Returns the updated memory, or undefined when skipped/absent.
+  function demoteToSummary(id, summary, { minRefTimeMs } = {}) {
+    let changed = false;
+    runAtomically(() => {
+      const row = db.prepare("SELECT last_accessed_at, content, _full_content FROM memories WHERE id = ?").get(id);
+      if (!row || row._full_content) return;
+      if (minRefTimeMs !== undefined && row.last_accessed_at) {
+        const lastMs = Date.parse(row.last_accessed_at);
+        if (lastMs >= minRefTimeMs) return; // touched after snapshot — still hot
+      }
+      db.prepare(
+        "UPDATE memories SET content = ?, _full_content = ?, updated_at = ? WHERE id = ?"
+      ).run(summary, row.content, nowIso(), id);
+      incrementGeneration();
+      changed = true;
+    });
+    return changed ? getById(id) : undefined;
+  }
+
+  // Undo demoteToSummary: pull the parked body back into content.
+  function restoreContent(id) {
+    let changed = false;
+    runAtomically(() => {
+      const row = db.prepare("SELECT content, _full_content FROM memories WHERE id = ?").get(id);
+      if (!row || !row._full_content) return;
+      db.prepare(
+        "UPDATE memories SET content = ?, _full_content = NULL, updated_at = ? WHERE id = ?"
+      ).run(row._full_content, nowIso(), id);
+      incrementGeneration();
+      changed = true;
+    });
+    return changed ? getById(id) : undefined;
+  }
+
+  // Live memories that have not been touched since `cutMs` (never-touched ones
+  // fall back to created_at). Ordered by last access ascending — the coldest
+  // first. Used by sleep phase 2 to pick archival-demotion candidates.
+  function getUnrecalledSince(cutMs, { limit = 500 } = {}) {
+    const cutIso = new Date(cutMs).toISOString();
+    const rows = db.prepare(
+      `SELECT * FROM memories
+       WHERE forgotten = 0 AND archived = 0
+         AND (last_accessed_at IS NULL OR last_accessed_at < ?)
+       ORDER BY COALESCE(last_accessed_at, created_at) ASC, id
+       LIMIT ?`
+    ).all(cutIso, limit);
+    return rows.map(toRow);
+  }
+
   function list({ type, limit = 50, offset = 0, includeForgotten = false, includeArchived = false } = {}) {
     const clauses = [];
     const params = [];
@@ -716,16 +826,17 @@ export function createStore(path) {
     const id = run.id ?? randomUUID();
     const now = nowIso();
     const policyEpoch = Number.isInteger(run.policy_epoch) ? run.policy_epoch : 0;
+    const runType = run.run_type ?? "auto";
     db.prepare(
       `INSERT INTO dream_runs (id, created_at, status, error, provider, model, snapshot_hash,
-        input_count, input, decisions, outcome, applied, summary_stored, receipt, policy_epoch)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        input_count, input, decisions, outcome, applied, summary_stored, receipt, policy_epoch, run_type)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        ON CONFLICT(id) DO UPDATE SET
          created_at=excluded.created_at, status=excluded.status, error=excluded.error,
          provider=excluded.provider, model=excluded.model, snapshot_hash=excluded.snapshot_hash,
          input_count=excluded.input_count, input=excluded.input, decisions=excluded.decisions,
          outcome=excluded.outcome, applied=excluded.applied, summary_stored=excluded.summary_stored,
-         receipt=excluded.receipt, policy_epoch=excluded.policy_epoch`
+         receipt=excluded.receipt, policy_epoch=excluded.policy_epoch, run_type=excluded.run_type`
     ).run(
       id,
       run.created_at ?? now,
@@ -741,7 +852,8 @@ export function createStore(path) {
       run.applied ?? 0,
       run.summary_stored ? 1 : 0,
       run.receipt,
-      policyEpoch
+      policyEpoch,
+      runType
     );
     return getDreamRun(id);
   }
@@ -1183,6 +1295,16 @@ export function createStore(path) {
     ).all(entityId, entityId).map(toRelation);
   }
 
+  /** All entities (optionally name-filtered, newest first). Used by sleep phase 4
+   *  orphan detection: an entity with zero relations is a candidate for relation
+   *  completion. */
+  function listEntities({ limit = 1000 } = {}) {
+    const rows = db.prepare(
+      "SELECT * FROM entities ORDER BY last_seen DESC, name ASC LIMIT ?"
+    ).all(limit);
+    return rows.map(toEntity);
+  }
+
   // --- mirror sync state (F-NEW-03) -----------------------------------------
 
   /**
@@ -1223,8 +1345,15 @@ export function createStore(path) {
       if (key === "dirty") {
         value = value ? 1 : 0;
       } else if (key === "generation" || key === "applied_generation") {
-        value = Math.trunc(Number(value));
-        if (!Number.isFinite(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) {
+        // Fail-closed integer enforcement (audit peer F): never truncate. A
+        // fractional value like 1.5 previously passed the JS gate via
+        // Math.trunc while SQLite's CHECK (>= 0) silently accepted it too, so a
+        // dirty legacy row could carry a non-integer generation that reads as a
+        // coherent applied round. Reject non-integers outright — the caller must
+        // pass a whole number, and a stale dirty value stays visible instead of
+        // being "repaired" into a misleading clean integer.
+        value = Number(value);
+        if (!Number.isInteger(value) || value < 0 || value > Number.MAX_SAFE_INTEGER) {
           throw new RangeError(`mirror_state.${key} out of range: ${value}`);
         }
       } else if (key === "type_status" && value != null && typeof value !== "string") {
@@ -1374,6 +1503,10 @@ export function createStore(path) {
     remove,
     setForget,
     setArchived,
+    touchLastAccess,
+    demoteToSummary,
+    restoreContent,
+    getUnrecalledSince,
     list,
     all,
     search,
@@ -1402,6 +1535,7 @@ export function createStore(path) {
     createEntity,
     findEntityByName,
     findEntityById,
+    listEntities,
     updateEntity,
     saveAttr,
     invalidateOldAttr,

package/test/mirror-generation.test.js

@@ -1,6 +1,6 @@
 import test from "node:test";
 import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { EventEmitter } from "node:events";
@@ -464,3 +464,36 @@ test("V0.3.6-F1: 旧库（v0.3.5 5 列）打开自动 ALTER 加 3 列，不丢
     rmSync(dir, { recursive: true, force: true });
   }
 });
+
+test("v0.3.9-D: mirror 逐 type 物理终态——兄弟 type 失败不误标已提交 type", () => {
+  const { dir, store, mirror, service } = setup();
+  try {
+    // 注入逐 type 故障：project 写成功、decision 抛错（模拟 EISDIR），其余 type 正常。
+    // 对应审计 D：project 文件已物理提交、decision 失败，状态必须逐 type 记录——
+    // 不能像旧逻辑那样整体批量标 failed。
+    const original = mirror.sync.bind(mirror);
+    mirror.sync = (memories) => {
+      const results = original(memories);
+      results.decision = { ok: false, error: "EISDIR: decision.md is a directory" };
+      return results;
+    };
+    // project 与 decision 都有真实记忆 → 触发逐 type 渲染
+    store.save({ id: "mem-project", type: "project", title: "已提交", content: "物理写入", importance: 3, tags: [] });
+    store.save({ id: "mem-decision", type: "decision", title: "写失败", content: "此 type 失败", importance: 3, tags: [] });
+    // 直接调 service 内部 syncMirror（通过一次写触发）
+    service.saveWithDedupe({ type: "project", title: "触发", content: "sync", importance: 3 });
+
+    const ts = store.getTypeStatus();
+    // project 物理提交 → 必须 committed，不能被 decision 失败拖成 failed
+    assert.equal(ts.project.status, "committed", "物理已提交的 type 必须标记 committed");
+    assert.equal(ts.decision.status, "failed", "失败的 type 必须标记 failed");
+    // 部分失败 = 未完全收敛 → dirty 必须持久
+    assert.equal(store.getMirrorState().dirty, true, "部分 type 失败必须持久 dirty");
+    // 镜像里 project 文件真实存在（物理终态已落地，不是整体失败）
+    const projectFile = mirror.filePath("project");
+    assert.ok(existsSync(projectFile), "project 镜像文件必须已物理写入");
+  } finally {
+    store.close();
+    rmSync(dir, { recursive: true, force: true });
+  }
+});

package/test/peer-blockers.test.js

@@ -146,3 +146,45 @@ test("peer-D: generation 上界与负数拒绝", () => {
     rmSync(dir, { recursive: true, force: true });
   }
 });
+
+test("v0.3.9-A: compareAndUpdate 的 CAS UPDATE 与 generation 同事务——miss 不得递增", () => {
+  const { dir, store } = setup();
+  try {
+    const saved = store.save({ type: "project", title: "CAS 原子", content: "v0" });
+    const before = store.getById(saved.id);
+    const genBefore = store.getMirrorState().generation;
+
+    // 成功 CAS：业务写入 + generation 递增必须一次提交（同事务）
+    const updated = store.compareAndUpdate(saved.id, before.updated_at, { content: "v1" });
+    assert.ok(updated, "当前版本 CAS 必须成功");
+    const genAfterOk = store.getMirrorState().generation;
+    assert.equal(genAfterOk, genBefore + 1, "成功 CAS 必须恰好递增一次 generation");
+
+    // miss CAS：不写任何东西，generation 也不得递增
+    const stale = store.getById(saved.id).updated_at; // v1 的 token
+    store.compareAndUpdate(saved.id, before.updated_at, { content: "v2" }); // 用旧 token → miss
+    assert.equal(store.getById(saved.id).content, "v1", "miss 不得改数据");
+    assert.equal(store.getMirrorState().generation, genAfterOk,
+      "CAS miss 不得递增 generation（UPDATE 与 increment 必须同事务）");
+  } finally {
+    store.close();
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("v0.3.9-F: generation 非整数必须拒绝（不得截断）", () => {
+  const { dir, store } = setup();
+  try {
+    // 审计 peer F：1.5 这类小数此前被 Math.trunc 截断 + SQLite CHECK 接受 → 静默脏值。
+    // fail-closed：JS 与 SQL 统一只接受整数。
+    assert.throws(() => store.setMirrorState({ generation: 1.5 }), RangeError, "小数 generation 必须拒绝");
+    assert.throws(() => store.setMirrorState({ applied_generation: -1.5 }), RangeError, "负数小数必须拒绝");
+    assert.throws(() => store.setMirrorState({ generation: Number.MAX_SAFE_INTEGER + 0.5 }), RangeError, "超界小数必须拒绝");
+    // 整数仍正常
+    const s = store.setMirrorState({ generation: 7 });
+    assert.equal(s.generation, 7, "整数 generation 正常");
+  } finally {
+    store.close();
+    rmSync(dir, { recursive: true, force: true });
+  }
+});