dsh-engram 0.2.0 → 0.3.0

package/lib/store.js

@@ -17,8 +17,11 @@
  */
 
 import { z } from "zod";
+import { existsSync } from "node:fs";
+import path from "node:path";
 import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
-import { hashText, byRecency, uuid, now, tokenize as tokenizeLocal, scoreRecord as scoreRecordLocal, truncate as truncateText } from "./util.js";
+import { hashText, byRecency, uuid, now, truncate as truncateText, bm25Rank, tokenize as tokenizeLocal, dedupeBySession } from "./util.js";
+import { deriveObservations } from "./obs.js";
 
 export const DOMAIN_VERSION = 1;
 
@@ -52,6 +55,10 @@ const MemorySchema = z.object({
   updatedAt: z.number().int().nonnegative(),
   /** Recall hit count — used to promote low-signal auto-captures into the index. */
   hits: z.number().int().nonnegative().default(0),
+  /** Last recall/access touch (epoch ms); null = never accessed. Retention signal for GC. */
+  lastAccessAt: z.number().int().nonnegative().nullable().default(null),
+  /** Optional file anchor (absolute path). Sharpens error revival and recall display. */
+  filePath: z.string().nullable().default(null),
   /** 0..1 static signal; index threshold is minIndexSignal. */
   signal: z.number().min(0).max(1).default(0.5),
   status: z.enum(["active", "archived"]).default("active"),
@@ -78,6 +85,12 @@ const TaskSchema = z.object({
   /** Set by GC when a stable task passed its retention window. Archived tasks
    *  leave the [ESR] surface but stay retrievable (id + name kept). */
   archivedAt: z.number().int().nullable().default(null),
+  /** Dependency edges (Beads-inspired mini graph): blocks = this task waits on
+   *  target, parent-of = target is this task's child, relates-to = no blocking. */
+  deps: z.array(z.object({ id: z.string().min(1), kind: z.enum(["blocks", "relates-to", "parent-of"]).default("blocks") })).default([]),
+  /** Claimer (agent id) — ownership fence for concurrent agents. */
+  assignee: z.string().nullable().default(null),
+  claimedAt: z.number().int().nullable().default(null),
 });
 
 const LinkSchema = z.object({
@@ -123,10 +136,71 @@ const UsageSchema = z.object({
   updatedAt: z.number().int().nonnegative(),
 });
 
+const ObservationSchema = z.object({
+  /** Stable id (table key), obs_<...>. */
+  id: z.string().min(1),
+  workspace: z.string().min(1),
+  /** Consolidated belief text (refined, never overwritten). */
+  text: z.string().min(1),
+  /** belief | pattern (derived from the triggering memory kind, no LLM). */
+  kind: z.enum(["belief", "pattern"]).default("belief"),
+  /** Unique supporting memory ids + their count. */
+  proof: z.object({ count: z.number().int().nonnegative(), sources: z.array(z.string()).default([]) }).default({ count: 0, sources: [] }),
+  span: z.object({ first_seen_at: z.number().int().nonnegative(), last_seen_at: z.number().int().nonnegative() }),
+  /** Opposite-polarity repeats weaken the belief. */
+  negations: z.number().int().nonnegative().default(0),
+  /** new | strengthening | stable | weakening | stale (algorithmic). */
+  trend: z.enum(["new", "strengthening", "stable", "weakening", "stale"]).default("new"),
+  tags: z.array(z.string()).default([]),
+  entity: z.string().nullable().default(null),
+  updated_at: z.number().int().nonnegative(),
+});
+
+const ModelSchema = z.object({
+  /** Workspace key ("" = global model). Reportedly any non-string crept in
+   *  once (URLSearchParams via an old readQuery misuse) and broke open — so
+   *  the shape is strictly string, and getModel/putModel normalize below. */
+  ws: z.string(),
+  /** Precomputed standing answer (markdown), zero-LLM aggregate. */
+  content: z.string().min(1),
+  generated_at: z.number().int().nonnegative(),
+  /** Bumped by any esr_* write; getModel recomputes when dirty. */
+  dirty: z.boolean().default(false),
+  /** Hash of the aggregation inputs (catches bumps we missed). */
+  sources_hash: z.string().default(""),
+});
+
 const GlobalSchema = z.object({
   initialized: z.boolean(),
 });
 
+/**
+ * Deterministic repeat-failure detector: two error texts share the memory if
+ * they overlap on >= 2 tokens AND the intersection covers >= 60% of the
+ * smaller token set (CJK bigrams + ASCII words). This drives the "failure
+ * memory revival" path — a recurring failure re-warms its old memory instead
+ * of piling up duplicates (PROJECTMEM's warn-before-repeat, done with zero
+ * LLM).
+ */
+const REPEAT_OVERLAP = 0.6;
+/** Looser threshold when the SAME filePath anchors both errors — a same-file
+ *  re-run that drifts in wording is still the same root cause. */
+const SAME_PATH_OVERLAP = 0.35;
+
+function repeatOverlapRatio(previous, incoming) {
+  const a = new Set(tokenizeLocal(String(previous)));
+  const b = new Set(tokenizeLocal(String(incoming)));
+  let inter = 0;
+  for (const t of a) if (b.has(t)) inter += 1;
+  if (inter < 2) return 0;
+  const min = Math.min(a.size, b.size);
+  return min > 0 ? inter / min : 0;
+}
+
+function repeatOverlap(previous, incoming) {
+  return repeatOverlapRatio(previous, incoming) >= REPEAT_OVERLAP;
+}
+
 /** Identity validation (mirrors shipped defineDomain; fails loud at load). */
 export const engramDomainSpec = defineDomain({
   name: "dsh_engram",
@@ -138,6 +212,8 @@ export const engramDomainSpec = defineDomain({
     links: domainTable(LinkSchema),
     entities: domainTable(EntitySchema),
     usage: domainTable(UsageSchema),
+    observations: domainTable(ObservationSchema),
+    models: domainTable(ModelSchema),
   },
 });
 
@@ -165,6 +241,8 @@ export async function openEngramDomain(facility) {
   const links = opened.table("links");
   const entities = opened.table("entities");
   const usage = opened.table("usage");
+  const observations = opened.table("observations");
+  const models = opened.table("models");
 
   const allMemories = (workspace) =>
     [...memories.entries()]
@@ -222,16 +300,7 @@ export async function openEngramDomain(facility) {
       if (kind) pool = pool.filter((m) => m.kind === kind);
       if (status) pool = pool.filter((m) => m.status === status);
       if (q !== void 0 && String(q).trim().length > 0) {
-        const trimmed = String(q).trim();
-        const tokens = tokenizeLocal(trimmed);
-        const phraseBoost = /\s/.test(trimmed);
-        const folded = trimmed.toLowerCase();
-        return pool
-          .map((m) => ({ m, score: scoreRecordLocal(m, tokens, folded, phraseBoost) }))
-          .filter((e) => e.score > 0)
-          .sort((a, b) => b.score - a.score || byRecency(a.m, b.m))
-          .slice(0, bounded)
-          .map((e) => e.m);
+        return bm25Rank(pool, String(q).trim(), bounded);
       }
       return pool.sort(byRecency).slice(0, bounded);
     },
@@ -243,7 +312,8 @@ export async function openEngramDomain(facility) {
       let taskCount = 0;
       let linkCount = 0;
       let nodeCount = 0;
-      const zeroCounts = () => ({ memories: 0, tasks: 0, links: 0, nodes: 0 });
+      let observationCount = 0;
+      const zeroCounts = () => ({ memories: 0, tasks: 0, links: 0, nodes: 0, observations: 0 });
       for (const [, m] of memories.entries()) {
         const ws = m.workspace;
         workspaces[ws] ??= zeroCounts();
@@ -270,7 +340,13 @@ export async function openEngramDomain(facility) {
         workspaces[e.workspace].nodes += 1;
         nodeCount += 1;
       }
-      return { workspaces, kinds, totals: { memories: memoryCount, tasks: taskCount, links: linkCount, nodes: nodeCount } };
+      for (const [, m] of memories.entries()) {
+        if (m.status !== "active" || (m.hits ?? 0) < 2) continue;
+        workspaces[m.workspace] ??= zeroCounts();
+        workspaces[m.workspace].observations += 1;
+        observationCount += 1;
+      }
+      return { workspaces, kinds, totals: { memories: memoryCount, tasks: taskCount, links: linkCount, nodes: nodeCount, observations: observationCount } };
     },
     /** Hard delete (GUI only; provenance stays in the session log). */
     async deleteMemory(workspace, id) {
@@ -279,9 +355,11 @@ export async function openEngramDomain(facility) {
       await memories.delete(id);
       return true;
     },
+
     /**
      * Store one memory. Exact-duplicate text in the same workspace refreshes
-     * `updatedAt` (and returns the existing id) instead of piling up.
+     * `updatedAt` (and returns the existing id) instead of piling up, and a
+     * near-repeat of a past ERROR re-warms that entry (failure revival).
      */
     async storeMemory(entry, config) {
       const text = String(entry.text).trim();
@@ -291,9 +369,29 @@ export async function openEngramDomain(facility) {
       }
       const dup = allMemories(entry.workspace).find((m) => hashText(m.text) === hashText(text) && m.kind === (entry.kind ?? "fact"));
       if (dup !== void 0) {
-        const next = { ...dup, updatedAt: now() };
+        // Repeat of the exact same failure is itself a signal: refresh recency
+        // AND climb the hit counter (errors only) so the [ENGRAM] index can
+        // eventually surface it via promoteHits.
+        const next = { ...dup, updatedAt: now(), ...(dup.kind === "error" ? { hits: (dup.hits ?? 0) + 1 } : {}) };
         await memories.put(dup.id, next);
-        return { stored: next, duplicated: true, id: dup.id };
+        return { stored: next, duplicated: true, id: dup.id, revived: dup.kind === "error" };
+      }
+      if ((entry.kind ?? "fact") === "error") {
+        // Failure-memory revival: a near-repeat of a past error re-warms the
+        // existing entry (recency + hits) and does not create a new row, so
+        // recurring failures stay one point of truth and climb toward an
+        // [ENGRAM] line instead of flooding the pool.
+        for (const prev of allMemories(entry.workspace)) {
+          if (prev.kind !== "error" || hashText(prev.text) === hashText(text)) continue;
+          const ratio = repeatOverlapRatio(prev.text, text);
+          const samePath =
+            prev.filePath !== null && entry.filePath !== null && prev.filePath === entry.filePath;
+          if (!samePath && ratio < REPEAT_OVERLAP) continue;
+          if (samePath && ratio < SAME_PATH_OVERLAP) continue;
+          const revived = { ...prev, updatedAt: now(), hits: (prev.hits ?? 0) + 1, filePath: prev.filePath ?? entry.filePath ?? null };
+          await memories.put(prev.id, revived);
+          return { stored: revived, duplicated: true, id: prev.id, revived: true };
+        }
       }
       if (allMemories(entry.workspace).length >= config.maxMemoriesPerWorkspace) {
         throw new EngramError("ENGRAM_CAP_EXCEEDED", `workspace already holds ${config.maxMemoriesPerWorkspace} active memories (the cap)`);
@@ -316,6 +414,8 @@ export async function openEngramDomain(facility) {
         createdAt: now(),
         updatedAt: now(),
         hits: 0,
+        lastAccessAt: null,
+        filePath: entry.filePath ?? null,
         signal: typeof entry.signal === "number" ? Math.max(0, Math.min(1, entry.signal)) : 0.5,
         status: "active",
         expiresAt,
@@ -326,25 +426,27 @@ export async function openEngramDomain(facility) {
     async touchMemory(workspace, id) {
       const m = memories.get(id);
       if (m === void 0 || m.workspace !== workspace || !notExpiredNow(m)) return void 0;
-      const next = { ...m, hits: (m.hits ?? 0) + 1, updatedAt: m.updatedAt };
+      const next = { ...m, hits: (m.hits ?? 0) + 1, lastAccessAt: now(), updatedAt: m.updatedAt };
       await memories.put(id, next);
       return next;
     },
-    /** Deterministic recall: tag-exact > text-substring > fuzzy, recency ties. */
-    recall(workspace, query, limit = 20) {
+    /** Pure access touch: updates lastAccessAt only (retention signal) without
+     *  touching the proof-count `hits`. Deterministic, non-appending. */
+    async markAccessed(workspace, id, at = now()) {
+      const m = memories.get(id);
+      if (m === void 0 || m.workspace !== workspace || !notExpiredNow(m)) return void 0;
+      const next = { ...m, lastAccessAt: at, updatedAt: m.updatedAt };
+      await memories.put(id, next);
+      return next;
+    },
+    /** Deterministic BM25 recall (tag-exact/fuzzy boosts, phrase boost); recency ties. */
+    recall(workspace, query, limit = 20, { maxPerSession } = {}) {
       const active = allMemories(workspace).filter(notExpiredNow);
       const bounded = Math.max(1, Math.min(50, Math.trunc(limit) || 20));
       if (query !== void 0 && String(query).trim().length > 0) {
-        const trimmed = String(query).trim();
-        const tokens = tokenizeLocal(trimmed);
-        const phraseBoost = /\s/.test(trimmed);
-        const folded = trimmed.toLowerCase();
-        return active
-          .map((m) => ({ m, score: scoreRecordLocal(m, tokens, folded, phraseBoost) }))
-          .filter((e) => e.score > 0)
-          .sort((a, b) => b.score - a.score || byRecency(a.m, b.m))
-          .slice(0, bounded)
-          .map((e) => e.m);
+        const ranked = bm25Rank(active, String(query).trim(), bounded);
+        if (maxPerSession === void 0 || maxPerSession === null) return ranked;
+        return dedupeBySession(ranked, maxPerSession); // 候选① per-session cap
       }
       return active.sort(byRecency).slice(0, bounded);
     },
@@ -394,6 +496,156 @@ export async function openEngramDomain(facility) {
       await tasks.put(task.id, task);
       return task;
     },
+    /**
+     * Single source of truth for the ESR evidence gate, shared by the esr_close
+     * tool and the web API so both never drift.
+     *
+     * Reuses what DSH already knows: the workspace key IS the session cwd, so a
+     * relative artifact resolves against it and is checked against the host's
+     * real filesystem when `config.verifyArtifact` is on (URLs and explicit
+     * `force` skip the disk check). Empty gates are still gaps regardless.
+     *
+     * @returns {{ gaps: string[], artifactReason?: string }} gaps (artifact /
+     *   evaluation / memory_ref) plus a human reason when the artifact was
+     *   provided but could not be confirmed on disk.
+     */
+    evidenceGate(workspace, task, config = {}) {
+      const gaps = [];
+      let artifactReason;
+      const artifact = typeof task.artifact === "string" ? task.artifact.trim() : "";
+      if (artifact.length === 0) {
+        gaps.push("artifact");
+      } else if (config.verifyArtifact !== false && !/^https?:\/\//i.test(artifact)) {
+        const resolved = path.isAbsolute(artifact) ? artifact : path.resolve(workspace, artifact);
+        try {
+          if (!existsSync(resolved)) artifactReason = `artifact not found on disk: ${resolved}`;
+        } catch {
+          artifactReason = `artifact unverifiable: ${resolved}`;
+        }
+        if (artifactReason !== void 0) gaps.push("artifact");
+      }
+      const evaluation = typeof task.evaluation === "string" ? task.evaluation.trim() : "";
+      if (evaluation.length === 0) gaps.push("evaluation");
+      if (!Array.isArray(task.memoryRefs) || task.memoryRefs.length === 0) gaps.push("memory_ref");
+      return { gaps, artifactReason };
+    },
+    /** Outgoing dependency edges of a task. */
+    taskDeps(workspace, id) {
+      const t = tasks.get(id);
+      return t === void 0 || t.workspace !== workspace ? [] : (t.deps ?? []);
+    },
+    /**
+     * Add a dependency edge with self-reference + directed-cycle guards.
+     * `blocks`/`parent-of` edges participate in blocking; `relates-to` never does.
+     */
+    async addDep(workspace, id, depId, kind = "blocks") {
+      const t = tasks.get(id);
+      const d = tasks.get(depId);
+      if (t === void 0 || t.workspace !== workspace) throw new EngramError("ENGRAM_NOT_FOUND", `no task "${id}" in this workspace`);
+      if (d === void 0 || d.workspace !== workspace) throw new EngramError("ENGRAM_NOT_FOUND", `no dep task "${depId}" in this workspace`);
+      if (id === depId) throw new EngramError("ENGRAM_INVALID_ARGS", "a task cannot depend on itself");
+      const deps = t.deps ?? [];
+      if (deps.some((x) => x.id === depId && x.kind === kind)) return t;
+      if (this.reaches(workspace, depId, id)) {
+        throw new EngramError("ENGRAM_CYCLE", `adding ${depId} as a dep of ${id} would form a cycle`);
+      }
+      const next = { ...t, deps: [...deps, { id: depId, kind }], updatedAt: now() };
+      await tasks.put(id, next);
+      return next;
+    },
+    /** Directed reachability over blocking edges (blocks + parent-of). */
+    reaches(workspace, from, target) {
+      const seen = new Set();
+      const walk = (cur) => {
+        if (cur === target) return true;
+        if (seen.has(cur)) return false;
+        seen.add(cur);
+        const node = tasks.get(cur);
+        for (const dep of node?.deps ?? []) {
+          if (dep.kind === "relates-to") continue;
+          if (walk(dep.id)) return true;
+        }
+        return false;
+      };
+      return walk(from);
+    },
+    /** Closed = stable or archived (blocker released). */
+    taskClosed(id) {
+      const t = tasks.get(id);
+      return t === void 0 || t.state === "stable" || (t.archivedAt !== null && t.archivedAt !== void 0);
+    },
+    /**
+     * Derived blocked state (Beads BlockedStateInvariant, read-time fixpoint):
+     * blocked exactly when a blocks/parent-of dep is not closed.
+     */
+    isBlocked(workspace, id) {
+      const t = tasks.get(id);
+      if (t === void 0 || t.workspace !== workspace) return false;
+      if (this.taskClosed(id)) return false;
+      for (const dep of t.deps ?? []) {
+        if (dep.kind === "relates-to") continue;
+        if (!this.taskClosed(dep.id)) return true;
+      }
+      return false;
+    },
+    /** Claimable queue: active/draft, no open blocker, nobody claimed it. */
+    readyTasks(workspace, limit = 10) {
+      return [...tasks.entries()]
+        .map(([, t]) => t)
+        .filter((t) => t.workspace === workspace && (t.state === "active" || t.state === "draft") && (t.archivedAt === null || t.archivedAt === void 0))
+        .filter((t) => !this.isBlocked(workspace, t.id))
+        .filter((t) => t.assignee === null || t.assignee === void 0)
+        .sort(byRecency)
+        .slice(0, Math.max(1, Math.min(100, limit)));
+    },
+    /** Atomic claim with an anti-yank fence: foreign holder refuses unless forced. */
+    async claimTask(workspace, id, agent, { force = false } = {}) {
+      const t = tasks.get(id);
+      if (t === void 0 || t.workspace !== workspace) throw new EngramError("ENGRAM_NOT_FOUND", `no task "${id}" in this workspace`);
+      if (this.taskClosed(id)) throw new EngramError("ENGRAM_INVALID_ARGS", `task ${id} is already closed and cannot be claimed`);
+      if (t.assignee !== null && t.assignee !== void 0 && t.assignee !== agent && !force) {
+        throw new EngramError("ENGRAM_CONFLICT", `task ${id} is claimed by ${t.assignee}`);
+      }
+      const next = { ...t, assignee: agent, claimedAt: now(), state: t.state === "draft" ? "active" : t.state, updatedAt: now() };
+      await tasks.put(id, next);
+      return next;
+    },
+    /** Release ownership; foreign holder needs force. */
+    async unclaimTask(workspace, id, agent, { force = false } = {}) {
+      const t = tasks.get(id);
+      if (t === void 0 || t.workspace !== workspace) throw new EngramError("ENGRAM_NOT_FOUND", `no task "${id}" in this workspace`);
+      if (t.assignee !== null && t.assignee !== void 0 && t.assignee !== agent && !force) {
+        throw new EngramError("ENGRAM_CONFLICT", `task ${id} is claimed by ${t.assignee}`);
+      }
+      const next = { ...t, assignee: null, claimedAt: null, updatedAt: now() };
+      await tasks.put(id, next);
+      return next;
+    },
+    /**
+     * Reversible compaction (Beads compactor.go, rule-only, zero LLM): when a
+     * task transitions to stable with a long description, keep a rule summary
+     * in `summary` and archive the full original text in `snapshot` — the kanban
+     * reads the summary, the original stays retrievable. No-op for shorts.
+     */
+    compactOnClose(task) {
+      const desc = (task.description ?? "").trim();
+      const already = (task.summary ?? null) !== null || (task.snapshot ?? null) !== null;
+      if (desc.length <= 240 || already) {
+        return { ...task, summary: task.summary ?? null, snapshot: task.snapshot ?? null };
+      }
+      const firstLine = (desc.split("\n")[0] ?? "").trim();
+      const summary = (firstLine.length > 0 ? firstLine : desc).slice(0, 140);
+      return {
+        ...task,
+        summary,
+        snapshot: {
+          description: desc,
+          artifact: task.artifact ?? null,
+          evaluation: task.evaluation ?? null,
+          memoryRefs: (task.memoryRefs ?? []).slice(),
+        },
+      };
+    },
     /** Cap is enforced by the tools layer (needs config); expose the count here. */
     activeTaskCount(workspace) {
       return activeTasks(workspace).length;
@@ -409,8 +661,9 @@ export async function openEngramDomain(facility) {
      * working-set-protected, archive-only (never hard-deletes — everything
      * archived stays re-fetchable, honouring pointer-salience):
      *   - TTL-expired memories beyond their `expiresAt` are archived;
-     *   - over-cap workspaces evict the lowest-value non-protected memories
-     *     (hits asc, signal asc, age asc) until at the cap;
+     *   - over-cap workspaces evict the lowest-retention non-protected memories
+     *     (lastAccess asc with createdAt fallback, hits asc, signal asc, age asc)
+     *     until at the cap;
      *   - stable tasks past `config.gcStableRetentionDays` become `archived`;
      *   - links whose BOTH endpoints are gone (no active memory entity, no
      *     surviving task id) are removed (dangling graph edges).
@@ -475,9 +728,19 @@ export async function openEngramDomain(facility) {
         const survivable = activeMem.length - report.archivedMemories.filter((e) => e.workspace === ws).length;
         if (survivable > cap) {
           const over = survivable - cap;
+          // Retention-first eviction (candidate ②): least-recently-accessed
+          // non-protected memories go first (fallback createdAt for never-accessed),
+          // then hits/signal/createdAt — identical to the pre-retention order for
+          // memories that never had lastAccessAt, so old data is untouched.
           const evict = candidates
             .slice()
-            .sort((a, b) => (a.hits ?? 0) - (b.hits ?? 0) || (a.signal ?? 0) - (b.signal ?? 0) || (a.createdAt ?? 0) - (b.createdAt ?? 0))
+            .sort(
+              (a, b) =>
+                (a.lastAccessAt ?? a.createdAt ?? 0) - (b.lastAccessAt ?? b.createdAt ?? 0) ||
+                (a.hits ?? 0) - (b.hits ?? 0) ||
+                (a.signal ?? 0) - (b.signal ?? 0) ||
+                (a.createdAt ?? 0) - (b.createdAt ?? 0),
+            )
             .slice(0, over);
           for (const m of evict) {
             if (!dryRun) await memories.put(m.id, { ...m, status: "archived", updatedAt: t0 });
@@ -527,6 +790,25 @@ export async function openEngramDomain(facility) {
       const e = entities.get(id);
       if (e === void 0 || e.workspace !== workspace) return void 0;
       return e;
+    },
+        /** Evidence-grounded beliefs for the workspace — derived, not persisted.
+     *  A memory with hits>=2 IS an observation; proof = hits, span = created→updated. */
+    listObservations(workspace) {
+      return deriveObservations(allMemories(workspace));
+    },
+    /** Precomputed standing-answer cache (mental model). */
+    getModel(ws) {
+      return models.get(typeof ws === "string" ? ws : "");
+    },
+    async putModel(ws, row) {
+      const key = typeof ws === "string" ? ws : "";
+      const rec = { ...row, ws: typeof ws === "string" ? ws : "" };
+      await models.put(key, rec);
+      return rec;
+    },
+    async markModelDirty(ws) {
+      const cur = models.get(ws);
+      if (cur !== void 0) await models.put(ws, { ...cur, dirty: true });
     },
     listEntities(workspace) {
       return allEntities(workspace).sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0));