@try-works/dsh-memory 0.1.20 → 0.1.21

package/STATUS.md

@@ -61,6 +61,7 @@ Observed in the running host. Each row names the effect that was seen, not a cla
 | Injection metric counts injections | `events.inject` advances by 1 per injection; the old character total is preserved under `legacy.inject` |
 | `memory_recover` is wired | calling it no longer answers "unavailable in this context"; it reaches output validation |
 | Session logs are searchable in principle | zstd, not encrypted; 368 logs, 740MB compressed, ~2.96GB plaintext |
+| `memory/map` events carry `ignorable: true` | the live session log's own event is marked, and both dsh-memory logs pass the harness's `validateStoredEvents` — the exact check that refused session-cc4795a7 with "unknown to this harness and not marked ignorable" |
 
 ## WRITTEN but NOT LOADED
 
@@ -71,6 +72,7 @@ Implemented and tested on disk. The running host predates them, so none of this
 | `due()` priority ordering | retry → abandon → work → recover, so a bounded batch reaches a due retry | measured: the retry sat at position 101 of 111 and is now position 1 |
 | `hostExtras(host)` as one source | `index.js` and the wiring test shared a copied list that drifted | the test now derives extras from the host object |
 | `memory_recover` report omits its `entries` field rather than blanking it | the old line added `entries: undefined`, which is NOT valid JSON, so the harness rejected the whole result | the new guard round-trips every tool output; reintroducing it fails with `value.entries is undefined` |
+| Publisher stops when the harness does not stamp `ignorable` (0.1.21) | ≤0.1.19 appended `memory/map` WITHOUT the marker into ten real session logs, so every later load of those sessions failed ("unknown to this harness and not marked ignorable"). The guard checks the RETURN VALUE of `session.append` and suppresses publishing for that session at the first refusal, instead of poisoning the log | 403 tests pass; the new test drives a harness-shaped stub that returns an unstamped event and asserts the next turn appends nothing, even with a changed store and a passed throttle |
 
 How to tell loaded from not: ask the host. `memory_recover` used to reply *"unavailable in this
 context"* and now replies *"returned invalid output: value is not lossless JSON"*. The change of
@@ -145,6 +147,11 @@ the flag on did not change what the user sees.
 
 ### Blocked on a restart
 
+- [ ] **Publish 0.1.21, install, restart.** The guard against unmarked `memory/map` events is
+      written and tested (403 pass) but the publish needs npm credentials that were not available
+      in this session (`npm publish` → ENEEDAUTH). Installed 0.1.20 already appends WITH the
+      marker, so nothing is at risk while this waits; the guard only protects against a harness
+      older than the `append(type, data, { ignorable: true })` option.
 - [x] **Restart, then check the tasks layer — DONE 2026-10-09.** The restart confirmed it. See the
       tasks-layer section above for the two log entries and the counts. It writes to ANOTHER plugin's storage through
       that plugin's service, so it deserves a real observation rather than a passing suite. After

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@try-works/dsh-memory",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Git-backed, workspace-scoped memory for the DeepSeek Harness (global layer + per-workspace layers)",
   "type": "module",
   "main": "src/index.js",

package/scripts/inspect-log.ts

@@ -0,0 +1,40 @@
+/* eslint-disable no-console */
+// One-shot diagnostic: decompress a session.v4.jsonl.zstd artifact with the
+// harness's own frame primitives and report memory/map rows and their envelope.
+import { readFileSync } from 'node:fs'
+import { scanZstdFrames, decompressZstdFrame } from 'file:///D:/deepseek-harness/packages/session/session-persistence-jsonl/src/zstd.ts'
+
+const file = process.argv[2]
+if (!file) throw new Error('usage: inspect-log.ts <session.v4.jsonl.zstd>')
+
+const buf = readFileSync(file)
+const scan = scanZstdFrames(buf)
+const lines: string[] = []
+for (const frame of scan.frames) {
+  const plain = await decompressZstdFrame(buf.subarray(frame.start, frame.end))
+  lines.push(...plain.toString('utf8').split('\n').filter((l) => l.trim() !== ''))
+}
+console.log(`frames=${scan.frames.length} torn=${scan.tornStart ?? '-'} rows=${lines.length}`)
+if (scan.tornStart !== undefined) console.log(`tornStart=${scan.tornStart}`)
+
+const header = lines[0]
+console.log(`header: ${header?.slice(0, 300) ?? '(none)'}`)
+
+const hits = lines.filter((l) => l.startsWith('{"type":"memory/map"'))
+console.log(`\nmemory/map EVENT rows: ${hits.length}`)
+for (const h of hits) {
+  let env: { seq?: number; time?: number; ignorable?: unknown } = {}
+  try { env = JSON.parse(h) } catch { /* not json */ }
+  console.log(`seq=${env.seq} time=${env.time} ignorable=${JSON.stringify(env.ignorable)} keys=${Object.keys(env).join(',')}`)
+}
+
+const unmarked = hits.filter((l) => !l.includes('"ignorable":true'))
+console.log(`\nUNMARKED memory/map events: ${unmarked.length}`)
+for (const h of unmarked.slice(0, 10)) console.log(h.slice(0, 300))
+
+console.log('\nlast 4 rows:')
+for (const l of lines.slice(-4)) {
+  let env: { type?: string; seq?: number; time?: number } = {}
+  try { env = JSON.parse(l) } catch { /* not json */ }
+  console.log(`type=${env.type} seq=${env.seq} time=${env.time}`)
+}

package/scripts/repair-ignorable.ts

@@ -0,0 +1,130 @@
+/* eslint-disable no-console */
+// One-shot repair for the dsh-memory "unknown event type" incident.
+//
+// Plugin <=0.1.19 appended `{"type":"memory/map",...}` events WITHOUT the
+// `ignorable: true` envelope marker. The harness's fail-closed storage contract
+// (session-persistence/src/storage-contract.ts) refuses such logs on every
+// load: "unknown to this harness and not marked ignorable". This rewrites each
+// poisoned log with the marker inserted into the memory/map rows, preserving
+// every other byte, the header-only first frame, frame checksums and seq
+// contiguity, then verifies the result through the harness's own validator.
+//
+// Idempotent: files with zero unmarked memory/map events are left untouched.
+// Usage: node --import tsx/esm scripts/repair-ignorable.ts [session-dir ...]
+//        (no args = scan the default sessions root and repair every broken log)
+import { copyFileSync, existsSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { join, basename } from 'node:path'
+import { scanZstdFrames, decompressZstdFrame, decompressZstdPrefix, compressZstdFrame } from 'file:///D:/deepseek-harness/packages/session/session-persistence-jsonl/src/zstd.ts'
+import { validateStoredEvents } from 'file:///D:/deepseek-harness/packages/session/session-persistence/src/storage-contract.ts'
+
+const ROOT = 'C:/Users/erikb/.dsh/sessions'
+const CUT_MS = Date.parse('2026-10-09T20:53:00Z')
+
+/** Insert the marker into one memory/map row, preserving every other byte of the line. */
+function markLine(line) {
+  if (!line.startsWith('{"type":"memory/map"') || line.includes('"ignorable":true')) return { line, marked: false }
+  const edited = line.replace(/^(\{"type":"memory\/map","seq":\d+,"time":\d+,)/, '$1"ignorable":true,')
+  if (edited === line) throw new Error(`memory/map row did not match the envelope pattern: ${line.slice(0, 80)}`)
+  return { line: edited, marked: true }
+}
+
+function markText(text) {
+  let marked = 0
+  const out = text.split('\n').map((line) => {
+    const r = markLine(line)
+    if (r.marked) marked += 1
+    return r.line
+  }).join('\n')
+  return { text: out, marked }
+}
+
+/** Repair one file in place; returns a report or throws. */
+async function repairFile(file) {
+  const buf = readFileSync(file)
+  const scan = scanZstdFrames(buf)
+  const frames = []
+  for (const frame of scan.frames) {
+    const plain = await decompressZstdFrame(buf.subarray(frame.start, frame.end))
+    frames.push(plain.toString('utf8'))
+  }
+  if (scan.tornStart !== undefined) {
+    const prefix = await decompressZstdPrefix(buf.subarray(scan.tornStart))
+    frames.push(prefix.toString('utf8'))
+  }
+  let marked = 0
+  const edited = []
+  for (const text of frames) {
+    const r = markText(text)
+    marked += r.marked
+    edited.push(r.text)
+  }
+  if (marked === 0) return { file, repaired: false, reason: 'no unmarked memory/map events' }
+
+  // Compress each frame back as its own checksummed frame: frame 1 stays the
+  // header-only frame, batch boundaries are preserved, and a recovered torn
+  // prefix becomes a complete final frame.
+  const parts = []
+  for (const text of edited) parts.push(await compressZstdFrame(Buffer.from(text, 'utf8')))
+  const out = Buffer.concat(parts)
+
+  const backup = `${file}.bak-pre-ignorable`
+  if (!existsSync(backup)) copyFileSync(file, backup)
+  const tmp = `${file}.repair.tmp`
+  writeFileSync(tmp, out)
+  renameSync(tmp, file)
+  return { file, repaired: true, marked }
+}
+
+/** Verify a file the way the gateway will read it: frames decode, frame 1 is
+ *  header-only, every row parses, seq is contiguous, and the storage contract
+ *  (the exact validator that refused these logs) accepts every event. */
+async function verifyFile(file) {
+  const buf = readFileSync(file)
+  const scan = scanZstdFrames(buf)
+  if (scan.frames.length === 0) throw new Error('no frames')
+  const header = (await decompressZstdFrame(buf.subarray(scan.frames[0].start, scan.frames[0].end))).toString('utf8')
+  if (header.indexOf('\n') !== header.length - 1) throw new Error('first frame is not exactly one header line')
+  const lines = [header.trimEnd()]
+  for (const frame of scan.frames.slice(1)) {
+    lines.push(...(await decompressZstdFrame(buf.subarray(frame.start, frame.end))).toString('utf8').split('\n').filter((l) => l.trim() !== ''))
+  }
+  if (scan.tornStart !== undefined) {
+    lines.push(...(await decompressZstdPrefix(buf.subarray(scan.tornStart))).toString('utf8').split('\n').filter((l) => l.trim() !== ''))
+  }
+  const rows = lines.map((l) => JSON.parse(l))
+  const meta = rows[0]
+  const events = rows.slice(1)
+  // Relative contiguity: a fork inherits a prefix, so the log may start at a nonzero seq.
+  for (let i = 1; i < events.length; i += 1) {
+    if (events[i].seq !== events[0].seq + i) throw new Error(`seq gap at index ${i}: ${events[i].seq}`)
+  }
+  const unmarked = events.filter((e) => e.type === 'memory/map' && e.ignorable !== true)
+  if (unmarked.length > 0) throw new Error(`${unmarked.length} memory/map events still unmarked`)
+  validateStoredEvents(meta, events, { kind: 'jsonl', path: file })
+  return { rows: rows.length, events: events.length, memoryMaps: events.filter((e) => e.type === 'memory/map').length }
+}
+
+function collectFiles() {
+  const dirs = process.argv.slice(2).filter((a) => !a.startsWith('-'))
+  if (dirs.length > 0) return dirs.map((d) => join(d, 'session.v4.jsonl.zstd'))
+  const out = []
+  const walk = (dir) => {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+      const p = join(dir, e.name)
+      if (e.isDirectory()) walk(p)
+      else if (e.name.endsWith('.zstd') && !e.name.includes('.bak') && statSync(p).mtimeMs >= CUT_MS) out.push(p)
+    }
+  }
+  walk(ROOT)
+  return out
+}
+
+let repaired = 0
+for (const file of collectFiles()) {
+  const r = await repairFile(file)
+  if (!r.repaired) continue
+  repaired += 1
+  const v = await verifyFile(file)
+  console.log(`repaired ${basename(file)}: marked ${r.marked} events; verified ${v.events} events, ${v.memoryMaps} memory/map rows, storage contract OK`)
+}
+console.log(`\nrepaired ${repaired} file(s); backups kept as <file>.bak-pre-ignorable`)

package/scripts/scan-sessions.ts

@@ -0,0 +1,43 @@
+/* eslint-disable no-console */
+// One-shot diagnostic: scan every session log modified since the first
+// memory/map publish and report unmarked memory/map events per file.
+import { readdirSync, readFileSync, statSync } from 'node:fs'
+import { join, basename } from 'node:path'
+import { scanZstdFrames, decompressZstdFrame, decompressZstdPrefix } from 'file:///D:/deepseek-harness/packages/session/session-persistence-jsonl/src/zstd.ts'
+
+const root = process.argv[2] ?? 'C:/Users/erikb/.dsh/sessions'
+const cut = Date.parse(process.argv[3] ?? '2026-10-09T20:53:00Z')
+
+function walk(dir: string): string[] {
+  const out: string[] = []
+  for (const e of readdirSync(dir, { withFileTypes: true })) {
+    const p = join(dir, e.name)
+    if (e.isDirectory()) out.push(...walk(p))
+    else if (e.name.endsWith('.zstd') && !e.name.includes('.bak') && statSync(p).mtimeMs >= cut) out.push(p)
+  }
+  return out
+}
+
+let scanned = 0
+for (const file of walk(root)) {
+  scanned += 1
+  let text = ''
+  try {
+    const buf = readFileSync(file)
+    const scan = scanZstdFrames(buf)
+    for (const f of scan.frames) text += (await decompressZstdFrame(buf.subarray(f.start, f.end))).toString('utf8')
+    if (scan.tornStart !== undefined) text += (await decompressZstdPrefix(buf.subarray(scan.tornStart))).toString('utf8')
+  } catch (err) {
+    console.log(`${basename(file)}: DECOMPRESS ERROR ${String(err).slice(0, 120)}`)
+    continue
+  }
+  let events = 0
+  let unmarked = 0
+  for (const line of text.split('\n')) {
+    if (!line.startsWith('{"type":"memory/map"')) continue
+    events += 1
+    if (!line.includes('"ignorable":true')) unmarked += 1
+  }
+  if (events > 0) console.log(`${unmarked > 0 ? 'BROKEN ' : 'ok     '} events=${events} unmarked=${unmarked} ${file}`)
+}
+console.log(`scanned ${scanned} files`)

package/scripts/verify-log.ts

@@ -0,0 +1,26 @@
+/* eslint-disable no-console */
+// Read-only: run the harness's own storage-contract validator over a live log.
+import { readFileSync } from 'node:fs'
+import { scanZstdFrames, decompressZstdFrame, decompressZstdPrefix } from 'file:///D:/deepseek-harness/packages/session/session-persistence-jsonl/src/zstd.ts'
+import { validateStoredEvents } from 'file:///D:/deepseek-harness/packages/session/session-persistence/src/storage-contract.ts'
+
+for (const file of process.argv.slice(2)) {
+  try {
+    const buf = readFileSync(file)
+    const scan = scanZstdFrames(buf)
+    const header = (await decompressZstdFrame(buf.subarray(scan.frames[0].start, scan.frames[0].end))).toString('utf8')
+    if (header.indexOf('\n') !== header.length - 1) throw new Error('first frame is not exactly one header line')
+    const lines = [header.trimEnd()]
+    for (const frame of scan.frames.slice(1)) {
+      lines.push(...(await decompressZstdFrame(buf.subarray(frame.start, frame.end))).toString('utf8').split('\n').filter((l) => l.trim() !== ''))
+    }
+    if (scan.tornStart !== undefined) {
+      lines.push(...(await decompressZstdPrefix(buf.subarray(scan.tornStart))).toString('utf8').split('\n').filter((l) => l.trim() !== ''))
+    }
+    const rows = lines.map((l) => JSON.parse(l))
+    validateStoredEvents(rows[0], rows.slice(1), { kind: 'jsonl', path: file })
+    console.log(`OK ${rows.length} rows, ${rows.slice(1).filter((e) => e.type === 'memory/map').length} memory/map: ${file}`)
+  } catch (err) {
+    console.log(`FAIL ${file}: ${String(err).slice(0, 300)}`)
+  }
+}

package/src/host.js

@@ -74,6 +74,7 @@ export function wireHost(ctx, store, cfg) {
   let restoreAttempted = false
   const lastMapPublish = new Map()
   const lastMapSignature = new Map()
+  const mapPublishSuppressed = new Set()
   let lastHistoryAttempt = 0
   let syncInFlight = false
 
@@ -721,6 +722,9 @@ export function wireHost(ctx, store, cfg) {
     try {
       if (!session || typeof session.append !== 'function' || !session.id) return
       if (isSubagentSession(session)) return
+      // A session whose harness did not honor ignorable is poisoned by every further append;
+      // the decision to stop is made once, below, and honored on every later attempt.
+      if (mapPublishSuppressed.has(session.id)) return
       const now = Date.now()
       if (!force && now - (lastMapPublish.get(session.id) ?? 0) < MAP_PUBLISH_MS) return
       lastMapPublish.set(session.id, now)
@@ -737,7 +741,21 @@ export function wireHost(ctx, store, cfg) {
       const signature = JSON.stringify({ ...payload, generatedAt: '' })
       if (!force && lastMapSignature.get(session.id) === signature) return
       lastMapSignature.set(session.id, signature)
-      session.append('memory/map', payload)
+      // Out-of-repo event types are absent from KNOWN_SESSION_EVENT_TYPES; without
+      // ignorable:true the persistence read path refuses the whole session log.
+      // The append returns the logged event: if the harness did NOT stamp
+      // ignorable onto it (a harness older than the option), the very next load
+      // of this session would fail with "unknown to this harness and not marked
+      // ignorable". Detecting that from the return value lets us stop before a
+      // log is poisoned, instead of discovering it when history fails to load.
+      const logged = session.append('memory/map', payload, { ignorable: true })
+      if (logged?.ignorable !== true) {
+        if (!mapPublishSuppressed.has(session.id)) {
+          mapPublishSuppressed.add(session.id)
+          logRun(store.root, 'map-publish', { ok: false, error: 'harness did not stamp ignorable; publishing suppressed for this session' })
+        }
+        return
+      }
       logRun(store.root, 'map-publish', { ok: true, scope, key: key ?? '', bytes: signature.length })
     } catch (error) {
       // Never let a view break the conversation it is describing.