dsh-engram 0.2.0 → 0.3.0

package/lib/usage.js

@@ -1,16 +1,24 @@
 /**
- * dsh-engram: real-data agent-behaviour observability.
+ * dsh-engram: agent-behaviour observability — computed ON DEMAND from the
+ * session log stream, nothing recorded per tool call.
  *
- * Every engram_* / esr_* tool call made by the model is recorded into the
- * store's per-(workspace, day) `usage` rollup: counts per tool, failures,
- * and recall mechanics. The GUI (`/api/dsh-engram/stats`) turns that into the
- * "proactive ESR ratio" and "recall hit rate" numbers — real measurements
- * from real sessions, not a static benchmark.
+ * Assessment P3 (2026-08-22): the old design had TWO observability stacks —
+ * a write-path `UsageTracker` that bumped a per-(workspace,day) `usage` table
+ * on every engram/esr tool call, and a separate session-log scan that counted
+ * tool calls (to include the built-in `todo` tool). Both are now ONE module:
+ * both /stats and /toolstats answer from the canonical log stream
+ * `~/.dsh/sessions/<bucket>/<sid>/session.jsonl.zstd`, decompressed on demand
+ * with a short TTL cache. No tool call pays a write anymore (pure GUI-side
+ * observability), and the log stream is the single source of truth.
  *
- * Zero-dependency, fire-and-forget: recording must never disturb the tool
- * pipeline, so callers use `void record(...).catch(() => {})`.
+ * Zero-dependency: the host already carries the `zstd` CLI.
  */
 
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
 import { workspaceKey as wk } from "./util.js";
 
 /** Local YYYY-MM-DD, so daily rollups land on calendar days for the user. */
@@ -20,6 +28,34 @@ export function dayKey(now = Date.now()) {
   return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
 }
 
+/** A big session log far exceeds Node's 1MB default exec buffer. */
+const MAX_BYTES = 300 * 1024 * 1024;
+const CACHE_TTL_MS = 60_000;
+const DAY_MS = 86_400_000;
+
+const cache = new Map(); // `${root}|${days}|${kind}` -> { at, payload }
+
+/** Default session-log root (the same tree the session index reads). */
+export function sessionsRoot() {
+  return path.join(os.homedir(), ".dsh", "sessions");
+}
+
+function decodeZstd(file) {
+  try {
+    return execFileSync("zstd", ["-dc", file], { maxBuffer: MAX_BYTES }).toString("utf8");
+  } catch {
+    return "";
+  }
+}
+
+function listDirOrEmpty(dir) {
+  try {
+    return fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+}
+
 /**
  * Count recalled memory rows from a engram_recall output string. The tool
  * renders one `- <id> …` line per item (or a `no active memories…` / `no
@@ -34,63 +70,191 @@ export function recallStatsFromOutput(out) {
   return { withHits: hitsTotal > 0 ? 1 : 0, hitsTotal };
 }
 
-/** Session-local drill-through window: a engram_detail within N events of a hit recall counts as a follow-up. */
-const DETAIL_FOLLOW_WINDOW = 8;
+/** Concatenate all text blocks of a `tool-result` content array. */
+function blockText(content) {
+  if (!Array.isArray(content)) return "";
+  let out = "";
+  for (const c of content) {
+    if (typeof c?.text === "string") out += c.text;
+  }
+  return out;
+}
 
 /**
- * One tracker per plugin instance. Holds the cross-call state used to infer
- * "recall → detail drill-through" from the raw tool stream.
+ * Count tool calls across recent session logs.
+ * @param opts - { root, days, now } — root is injectable for tests.
+ * @returns { days, files, events, tools, buckets, cachedAt }
+ *   `tools` = tool name -> call count (most active first, across all buckets),
+ *   `buckets` = bucket dir -> per-tool counts.
  */
-export function makeUsageTracker() {
-  const recallFresh = new Map(); // sessionId -> event seq of last hit recall
-
-  return {
-    /**
-     * Record one tool-call outcome. `opts`: { ok, recallOutput? }.
-     * Resolves the caller from `exec` exactly like tools.js `callerOf`. Returns
-     * a promise; callers should fire-and-forget it.
-     */
-    async record(service, exec, name, opts) {
-      const agent = exec?.agent;
-      if (agent === void 0) return;
-      const rawCwd = agent.session?.header?.cwd;
-      const sessionId = agent.session?.id;
-      if (rawCwd === void 0 || rawCwd.length === 0 || sessionId === void 0) return;
-      const workspace = wk(rawCwd);
-      const seq = agent.session?.events?.length ?? 0;
-
-      const counts = { [name]: 1 };
-      const recall = {};
-      if (name === "engram_recall") {
-        if (opts.ok) {
-          const { withHits, hitsTotal } = recallStatsFromOutput(opts.recallOutput);
-          recall.queries = 1;
-          recall.withHits = withHits;
-          recall.hitsTotal = hitsTotal;
-          if (withHits > 0) recallFresh.set(sessionId, seq);
-          else recallFresh.delete(sessionId);
-        } else {
-          recallFresh.delete(sessionId);
-        }
-      } else if (name === "engram_detail" && opts.ok && recallFresh.has(sessionId)) {
-        const at = recallFresh.get(sessionId);
-        if (seq - at <= DETAIL_FOLLOW_WINDOW) {
-          recall.detailFollows = 1;
-          recallFresh.delete(sessionId);
+export function collectToolCounts({ root = sessionsRoot(), days = 14, now = Date.now() } = {}) {
+  const cacheKey = `${root}|${days}|tools`;
+  const hit = cache.get(cacheKey);
+  if (hit && now - hit.at < CACHE_TTL_MS) return hit.payload;
+
+  const byTool = new Map();
+  const byBucket = new Map();
+  const cutoff = now - days * DAY_MS;
+  let files = 0;
+  let events = 0;
+
+  const buckets = listDirOrEmpty(root).filter((b) => b.startsWith("--"));
+  for (const bucket of buckets) {
+    const bdir = path.join(root, bucket);
+    for (const sid of listDirOrEmpty(bdir)) {
+      const file = path.join(bdir, sid, "session.jsonl.zstd");
+      let st;
+      try {
+        st = fs.statSync(file);
+      } catch {
+        continue;
+      }
+      if (st.mtimeMs < cutoff) continue;
+      files += 1;
+      const raw = decodeZstd(file);
+      const bucketCounts = byBucket.get(bucket) ?? new Map();
+      for (const line of raw.split("\n")) {
+        if (line.length === 0) continue;
+        let j;
+        try {
+          j = JSON.parse(line);
+        } catch {
+          continue;
         }
+        if (j?.type !== "tool/call") continue;
+        const name = j.data?.name;
+        if (typeof name !== "string" || name.length === 0) continue;
+        events += 1;
+        byTool.set(name, (byTool.get(name) ?? 0) + 1);
+        bucketCounts.set(name, (bucketCounts.get(name) ?? 0) + 1);
       }
+      if (bucketCounts.size > 0) byBucket.set(bucket, bucketCounts);
+    }
+  }
+
+  const tools = Object.fromEntries([...byTool.entries()].sort((a, b) => b[1] - a[1]));
+  const bucketsOut = {};
+  for (const [b, m] of byBucket) {
+    bucketsOut[b] = Object.fromEntries([...m.entries()].sort((a, b) => b[1] - a[1]));
+  }
 
+  const payload = { days, files, events, tools, buckets: bucketsOut, cachedAt: now };
+  cache.set(cacheKey, { at: now, payload });
+  return payload;
+}
+
+/** Resolve the workspace key of a session from its `session` header event. */
+function workspaceFromSession(raw) {
+  for (const line of raw.split("\n")) {
+    if (line.length === 0) continue;
+    let j;
+    try {
+      j = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (j?.type === "session" && typeof j.cwd === "string" && j.cwd.length > 0) {
+      return wk(j.cwd);
+    }
+    return null; // session header is the first event
+  }
+  return null;
+}
+
+/** Session-local drill-through window: a engram_detail within N tool events of a hit recall counts as a follow-up. */
+const DETAIL_FOLLOW_WINDOW = 8;
+
+/**
+ * Rebuild the per-(workspace, day) usage rollup straight from the session log
+ * stream. Same shape as the old persisted rows, so the /stats aggregation is
+ * unchanged: each row is { workspace, day, counts, failures, recall } where
+ * `counts` = tool name -> calls, `failures` = errored results, `recall` =
+ * { queries, withHits, hitsTotal, detailFollows }.
+ *
+ * Workspace is resolved from each session's `type:session` header cwd; day
+ * from the file mtime (calendar-day granularity is all the GUI needs).
+ */
+export function collectUsageStats({ root = sessionsRoot(), days = 14, now = Date.now() } = {}) {
+  const cacheKey = `${root}|${days}|usage`;
+  const hit = cache.get(cacheKey);
+  if (hit && now - hit.at < CACHE_TTL_MS) return hit.payload;
+
+  const rows = new Map(); // `${workspace}|${day}` -> row
+  const cutoff = now - days * DAY_MS;
+  let files = 0;
+  let events = 0;
+  let sessions = 0;
+
+  const buckets = listDirOrEmpty(root).filter((b) => b.startsWith("--"));
+  for (const bucket of buckets) {
+    for (const sid of listDirOrEmpty(path.join(root, bucket))) {
+      const file = path.join(root, bucket, sid, "session.jsonl.zstd");
+      let st;
       try {
-        const domain = await service.getDomain();
-        await domain.bumpUsage(workspace, dayKey(), {
-          counts,
-          failures: opts.ok ? 0 : 1,
-          recall,
-        });
-      } catch (error) {
-        // observability is best-effort; a store hiccup must not surface here
-        service?.log?.warn?.(`engram usage record failed: ${String(error)}`);
+        st = fs.statSync(file);
+      } catch {
+        continue;
+      }
+      if (st.mtimeMs < cutoff) continue;
+      const raw = decodeZstd(file);
+      files += 1;
+      const workspace = workspaceFromSession(raw) ?? "?";
+      const day = dayKey(st.mtimeMs);
+      sessions += 1;
+
+      // Results come AFTER their call in the stream, so resolve them in a
+      // first pass and walk the calls in a second (index keeps stream order).
+      const results = new Map(); // callId -> { isError, text }
+      for (const line of raw.split("\n")) {
+        if (line.length === 0) continue;
+        let j;
+        try {
+          j = JSON.parse(line);
+        } catch {
+          continue;
+        }
+        if (j?.type !== "tool/result") continue;
+        const block = j.data?.message?.content?.find((c) => c?.type === "tool-result");
+        if (block?.toolCallId) {
+          results.set(block.toolCallId, { isError: block.isError === true, text: blockText(block.content) });
+        }
       }
-    },
-  };
-}
+      let windowLeft = -1; // stream index of last hit-recall
+      let idx = 0;
+      for (const line of raw.split("\n")) {
+        if (line.length === 0) continue;
+        let j;
+        try {
+          j = JSON.parse(line);
+        } catch {
+          continue;
+        }
+        if (j?.type !== "tool/call") continue;
+        const name = j.data?.name;
+        if (typeof name !== "string" || name.length === 0) continue;
+        idx += 1;
+        events += 1;
+        const result = j.data?.callId ? results.get(j.data.callId) : void 0;
+        const key = `${workspace}|${day}`;
+        const row = rows.get(key) ?? { workspace, day, counts: {}, failures: 0, recall: {} };
+        row.counts[name] = (row.counts[name] ?? 0) + 1;
+        if (result?.isError) row.failures += 1;
+        if (name === "engram_recall") {
+          row.recall.queries = (row.recall.queries ?? 0) + 1;
+          const rs = recallStatsFromOutput(result ? result.text : "");
+          row.recall.withHits = (row.recall.withHits ?? 0) + rs.withHits;
+          row.recall.hitsTotal = (row.recall.hitsTotal ?? 0) + rs.hitsTotal;
+          windowLeft = rs.withHits > 0 ? idx : -1;
+        } else if (name === "engram_detail" && windowLeft >= 0 && idx - windowLeft <= DETAIL_FOLLOW_WINDOW) {
+          row.recall.detailFollows = (row.recall.detailFollows ?? 0) + 1;
+          windowLeft = -1;
+        }
+        rows.set(key, row);
+      }
+    }
+  }
+
+  const payload = { days, files, events, sessions, rows: [...rows.values()], cachedAt: now };
+  cache.set(cacheKey, { at: now, payload });
+  return payload;
+}

package/lib/util.js

@@ -4,6 +4,7 @@
 
 import { createHash, randomUUID } from "node:crypto";
 import { basename, resolve } from "node:path";
+import { evidenceBoost } from "./rerank.js";
 
 /** Random id for memories / tasks / links. */
 export function uuid() {
@@ -117,3 +118,109 @@ export function scoreRecord(record, tokens, fullQuery, phraseBoost) {
 export function byRecency(left, right) {
   return (right.updatedAt ?? 0) - (left.updatedAt ?? 0) || String(left.id).localeCompare(String(right.id));
 }
+
+/**
+ * BM25-style ranking over an in-memory record corpus. Deterministic, zero
+ * dependencies — we deliberately do NOT build a SQLite/FTS index here: DSH
+ * already owns the cross-session full-text index (`ctx.sessionQuery`), which
+ * engram_recall's `search_sessions` piggybacks on; this ranks only the small
+ * in-domain memory pool, where an in-process IDF pass is cheap and keeps the
+ * store self-contained.
+ *
+ * Weights: BM25 (k1=1.5, b=0.75) over text+tags, plus the proven small boosts —
+ * exact tag match, fuzzy tag overlap, and a phrase boost for multi-word queries,
+ * and a gentle multiplicative recency factor (half-life 14 days, max +50%):
+ * freshness lifts near-ties and stale-but-still-matching results, but never
+ * overrides strong lexical relevance (Zep/decay-weighted recall direction).
+ * `now` is injectable so the ranking stays deterministic in tests.
+ * Returns up to `limit` records with score > 0, best first, recency as tiebreak.
+ */
+export function bm25Rank(records, query, limit = 20, now = Date.now()) {
+  const DAY = 86400000;
+  const clamped = Math.max(0, typeof now === "number" ? now : Date.now());
+  const trimAgeDays = (r) => Math.max(0, (clamped - (r.updatedAt ?? r.createdAt ?? 0)) / DAY);
+  const trimmed = String(query ?? "").trim().toLowerCase();
+  const bounded = Math.max(1, Math.min(50, Math.trunc(limit) || 20));
+  if (trimmed.length === 0) return records.slice(0, bounded);
+  const tokens = tokenize(trimmed);
+  if (tokens.length === 0) return records.slice(0, bounded);
+
+  const N = Math.max(1, records.length);
+  const df = new Map();
+  const lens = new Array(records.length);
+  const blobs = records.map((r, i) => {
+    const text = String(r.text ?? "").toLowerCase();
+    const tags = (r.tags ?? []).map((t) => String(t).toLowerCase());
+    const blob = `${text} ${tags.join(" ")}`;
+    const terms = blob.split(/\s+/).filter(Boolean);
+    lens[i] = Math.max(1, terms.length);
+    for (const term of new Set(terms)) df.set(term, (df.get(term) ?? 0) + 1);
+    return { blob, tags, text };
+  });
+  const avgdl = lens.reduce((a, b) => a + b, 0) / N;
+  const K1 = 1.5;
+  const B = 0.75;
+
+  const scored = records
+    .map((r, i) => {
+      const { blob, tags, text } = blobs[i];
+      const len = lens[i];
+      let score = 0;
+      for (const token of tokens) {
+        const dfT = df.get(token) ?? 0;
+        const idf = Math.log(1 + (N - dfT + 0.5) / (dfT + 0.5));
+        let count = 0;
+        let from = 0;
+        for (;;) {
+          const at = blob.indexOf(token, from);
+          if (at === -1) break;
+          count += 1;
+          from = at + token.length;
+        }
+        const tf = count * (K1 + 1) / (count + K1 * (1 - B + (B * len) / avgdl));
+        score += idf * tf;
+        if (tags.includes(token)) score += 2;
+        else if (tags.some((tag) => tag.includes(token) || token.includes(tag))) score += 1;
+      }
+      if (/\s/.test(trimmed) && trimmed.length > 1 && text.includes(trimmed)) score += 3;
+      // Gentle time-awareness: 1 + 0.5·exp(-ageDays/14). New memories get up to
+      // +50%; after 14 days still +18%, after 60 days ~+1%. Never zeroes a hit.
+      score *= 1 + 0.5 * Math.exp(-trimAgeDays(r) / 14);
+      // Evidence strength (cumulative recall hits = proof count analogue, fed
+      // by the failure-revival path): conservative ±5% at alpha 0.1, saturating
+      // after 5 hits so no single memory can drown the lexical signal.
+      score *= evidenceBoost(r.hits ?? 0);
+      return { r, score };
+    })
+    .filter((e) => e.score > 0)
+    .sort((a, b) => b.score - a.score || byRecency(a.r, b.r));
+  return scored.slice(0, bounded).map((e) => e.r);
+}
+
+/**
+ * Session-diversified dedup (agentmemory 候选①): keep the input order
+ * (stable) but cap how many records the same sessionId may contribute.
+ *
+ * Rationale (AML coding board + agentmemory deep-dive): "few and precise"
+ * evidence beats letting one session flood the recall list — the platform
+ * answerer consumes a bounded evidence set, and single-session domination
+ * squeezes out cross-session relevance. Deterministic: pure function, no
+ * randomness, no reading beyond `records`.
+ *
+ * @param {Array<{sessionId?: string|null, id?: string}>} records ranked list, best first
+ * @param {number} [maxPerSession=3] max records kept per sessionId (>=1)
+ * @returns {Array} subset of records, order preserved
+ */
+export function dedupeBySession(records, maxPerSession = 3) {
+  const cap = Math.max(1, Math.trunc(maxPerSession) || 3);
+  const seen = new Map();
+  const out = [];
+  for (const record of records) {
+    const key = record?.sessionId ?? record?.id ?? `__noid:${out.length}`;
+    const n = seen.get(key) ?? 0;
+    if (n >= cap) continue;
+    seen.set(key, n + 1);
+    out.push(record);
+  }
+  return out;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-engram",
   "description": "Minimalist long-term memory for DeepSeek Harness, distilling symbolic-index + pi-esr ideas: zero-LLM auto-capture, a symbolic [ENGRAM] index with progressive disclosure, and an ESR-lite evidence-closure protocol (esr_task / esr_node / esr_close / esr_link). Real agent-behaviour telemetry (usage rollup + /stats) and a deterministic offline recall benchmark (npm run eval). Hot path is model-free; storage on ctx.storageDomain; web memory viewer, observability panel and config card through DSH's native settings slots.",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -38,8 +38,7 @@
     "typecheck": "echo ok"
   },
   "publishConfig": {
-    "access": "public",
-    "registry": "https://registry.npmjs.org"
+    "access": "public"
   },
   "dsh": {
     "bundle": {
@@ -85,8 +84,5 @@
     "long-term-memory",
     "symbolic-index",
     "pi-esr"
-  ],
-  "publishConfig": {
-    "access": "public"
-  }
+  ]
 }