@try-works/dsh-memory 0.1.28 → 0.1.30

package/STATUS.md

@@ -3,8 +3,9 @@
 A tracker and planner: what exists, what is verified working, what is written but not
 loaded, and what is left. Kept as a record of MEASURED state, not intention.
 
-**Last measured: 2026-10-17** (408 tests passing; plugin at 0.1.27, published and installed
-byte-identical and confirmed LOADED by the running host; queue terminal-clean).
+**Last measured: 2026-10-17** (408 tests passing; plugin at 0.1.28, published and installed
+byte-identical and confirmed LOADED by the running harness at 0.2.1-rc.2; queue terminal-clean;
+the Memory tab reads the published map under the new event name).
 
 ## How to read this
 
@@ -67,6 +68,7 @@ Observed in the running host. Each row names the effect that was seen, not a cla
 | Boot tasks backup + restore ran live | at boot (2026-10-10T22:03:14Z) the log records `tasks-backup ok:true written:0 refreshed:5` and `tasks-restore ok:true owner:session-6e8d9a10 present:5 added:0 paused:0` — the restore reached the schedule service, found everything present, and duplicated nothing |
 | Git sync works live | after the 0.1.22+ boot the log shows real `sync` events, `state/sync.json` records `ok:true remote:https://github.com/try-works/dsh-memory-store` (2026-10-11T10:34:54Z), and `git log` in the store holds `memory(store): update [dsh-memory]` commits — the first syncs this plugin has ever performed |
 | History ingest works live | `history-check ok=true` entries after the boot, including `count=1` at 09:37Z — a commit was actually ingested; the hourly `null.run` error era is over |
+| Map publishing works on harness 0.2.1 | after the 0.1.28 boot the log shows `map-publish ok:true` on every cadence, the live session log carries `plugin:memory/map` rows with `ignorable:true`, the harness's own `validateStoredEvents` accepts the log, and the Memory tab renders (confirmed in the UI). The 0.2.1 contract is different from what 0.1.27 assumed: `Session.append` takes options only for surface event types, so an out-of-repo record is written with `appendPluginRecord` under the `plugin:` namespace |
 
 ## Shipped and LOADED
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@try-works/dsh-memory",
-  "version": "0.1.28",
+  "version": "0.1.30",
   "description": "Git-backed, workspace-scoped memory for the DeepSeek Harness (global layer + per-workspace layers)",
   "type": "module",
   "main": "src/index.js",

package/src/fts.js

@@ -54,11 +54,43 @@ export function buildTrigramQuery(input) {
 /**
  * Create both indexes when missing. The word table serves normal queries; the trigram table
  * serves substring queries (no-space scripts), which the word tokenizer cannot express.
+ *
+ * MANDATORY, both tables. The trigram table is the ONLY route for a no-space script such as
+ * Chinese or Japanese, and the fallback for a partial-word query that the word table missed, so
+ * failing to create it must not be swallowed: a store without it silently makes a whole class of
+ * queries unanswerable, and the caller sees "no results" rather than an error. Throw here, where
+ * the cause is still visible, instead of degrading search quietly.
  */
 function ensureTables(db) {
   db.exec('CREATE VIRTUAL TABLE IF NOT EXISTS mem_fts USING fts5(file, line, section, content)')
-  try { db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS " + SUBSTRING_TABLE + " USING fts5(file, line, section, content, tokenize='trigram')") }
-  catch { /* a build without trigram still works, just without substring search */ }
+  try {
+    db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS " + SUBSTRING_TABLE + " USING fts5(file, line, section, content, tokenize='trigram')")
+  } catch (error) {
+    throw new Error('the substring (trigram) index is required and could not be created: ' + String((error && error.message) || error))
+  }
+}
+
+/**
+ * Run a block of writes inside ONE transaction.
+ *
+ * Without this, every INSERT is its own implicit transaction, so a row costs a commit and an
+ * fsync. A full rebuild of this store writes ~292,000 rows (word table + trigram table), and a
+ * measured rebuild did not finish in ten minutes. Batched, the same work is seconds - which is
+ * what makes "the index is derived, delete it and rebuild" a safe recovery step instead of an
+ * outage for every query.
+ * @param run - the work to perform inside the transaction.
+ * @returns whatever `run` returned, after the transaction commits.
+ */
+function inTransaction(db, run) {
+  db.exec('BEGIN')
+  try {
+    const result = run()
+    db.exec('COMMIT')
+    return result
+  } catch (error) {
+    try { db.exec('ROLLBACK') } catch { /* a failed rollback must not mask the real error */ }
+    throw error
+  }
 }
 
 /**
@@ -122,11 +154,13 @@ export function updateIndex(root, files) {
   let dropTri
   try { dropTri = db.prepare('DELETE FROM ' + SUBSTRING_TABLE + ' WHERE file = ?') } catch { dropTri = undefined }
   let rows = 0
-  for (const file of list) {
-    drop.run(relOf(root, file))
-    if (dropTri) try { dropTri.run(relOf(root, file)) } catch {}
-    try { rows += indexFile(db, root, file) } catch {}
-  }
+  inTransaction(db, () => {
+    for (const file of list) {
+      drop.run(relOf(root, file))
+      if (dropTri) try { dropTri.run(relOf(root, file)) } catch {}
+      try { rows += indexFile(db, root, file) } catch {}
+    }
+  })
   db.close()
   return { rows, files: list.length }
 }
@@ -159,8 +193,11 @@ export function buildIndex(root) {
   db.exec('DROP TABLE IF EXISTS mem_fts')
   db.exec('DROP TABLE IF EXISTS ' + SUBSTRING_TABLE)
   ensureTables(db)
-  let rows = 0
-  for (const file of indexedFiles(root)) rows += indexFile(db, root, file)
+  const rows = inTransaction(db, () => {
+    let total = 0
+    for (const file of indexedFiles(root)) total += indexFile(db, root, file)
+    return total
+  })
   db.close()
   return { dbPath, rows }
 }

package/src/inject.js

@@ -1,9 +1,9 @@
 // Session-start injection text: goal + working head + recent learnings + footer,
 // char-capped and scope-aware.
-import { existsSync, readFileSync } from 'node:fs'
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { layerDir } from './store.js'
-import { buildLedger } from './ledger.js'
+import { buildLedgerFor } from './ledger.js'
 import { readLayerTopics } from './layers.js'
 import { freshnessMarker, readState } from './state.js'
 import { readWorking, workingMarker } from './working.js'
@@ -12,7 +12,11 @@ import { relevanceQuery, selectEntries } from './select.js'
 export function buildInjectionText(store, { scope = 'global', key, goal, maxChars = 3600, subagent = false } = {}) {
   const dir = layerDir(store.root, scope, key)
   // Working memory is the AGENT's intent (task, recent activity, next steps); repository
-  const ledger = buildLedger(store.root)
+  //
+  // Only the two layers this card actually quotes are read. Building the whole ledger here meant
+  // walking every indexed file - ~8,000 of them, history shards included - on every injection,
+  // which measured ~17 s and dwarfed the text it produced.
+  const ledger = buildLedgerFor(store.root, layerFilesFor(dir, ['learnings', 'decisions']))
   const prefix = scope === 'global' ? 'global/' : 'workspaces/' + key + '/'
   // The pools are LISTED here and CHOSEN later, because choosing needs the question, and the
   // question comes from working memory - which is read below.
@@ -106,3 +110,24 @@ export function buildInjectionText(store, { scope = 'global', key, goal, maxChar
   const body = parts.join('\n\n')
   return (body.length <= maxChars ? body : body.slice(0, maxChars)) + '\n\n' + footer
 }
+
+/**
+ * The absolute file paths of the named layer directories in one scope.
+ *
+ * Layer directories are flat (one topic file per entry), so listing them is cheap. This exists
+ * because the injection card quotes only learnings and decisions: reading every OTHER layer -
+ * semantic, episodic, procedural, codebase, history - was the expensive part of the old
+ * whole-store ledger, and none of it could ever appear in the card.
+ * @param dir - the scope directory (a layer dir already), from {@link layerDir}.
+ * @param layers - directory names under it, e.g. 'learnings' and 'decisions'.
+ * @returns absolute paths, in directory order.
+ */
+function layerFilesFor(dir, layers) {
+  const files = []
+  for (const name of layers) {
+    const layerPath = join(dir, name)
+    if (!existsSync(layerPath)) continue
+    for (const file of readdirSync(layerPath)) if (file.endsWith('.md')) files.push(join(layerPath, file))
+  }
+  return files
+}

package/src/ledger.js

@@ -1,19 +1,47 @@
-// Derived entry ledger: per-entry metadata (file, layer, line span, importance,
-// dates) rebuilt from the markdown in one step. Feeds ranking + freshness.
-import { readFileSync } from 'node:fs'
-import { relative } from 'node:path'
-import { indexedFiles, layerOfFile } from './files.js'
-import { parseEntries } from './entry.js'
-
-export function buildLedger(root) {
-  const entries = []
-  for (const file of indexedFiles(root)) {
-    const rel = relative(root, file).replace(/\\/g, '/')
-    const layer = layerOfFile(root, file)
-    if (!layer) continue
-    for (const e of parseEntries(readFileSync(file, 'utf8'))) {
-      entries.push({ ...e, file: rel, layer })
-    }
-  }
-  return entries
-}
+// Derived entry ledger: per-entry metadata (file, layer, line span, importance,
+// dates) rebuilt from the markdown in one step. Feeds ranking + freshness.
+import { readFileSync } from 'node:fs'
+import { isAbsolute, join, relative, sep } from 'node:path'
+import { indexedFiles, layerOfFile } from './files.js'
+import { parseEntries } from './entry.js'
+
+export function buildLedger(root) {
+  const entries = []
+  for (const file of indexedFiles(root)) {
+    const rel = relative(root, file).replace(/\\/g, '/')
+    const layer = layerOfFile(root, file)
+    if (!layer) continue
+    for (const e of parseEntries(readFileSync(file, 'utf8'))) {
+      entries.push({ ...e, file: rel, layer })
+    }
+  }
+  return entries
+}
+
+/**
+ * Build entries for a KNOWN SET of files instead of the whole store.
+ *
+ * The full ledger reads every indexed file, and on this store that is ~8,000 files and ~17 s —
+ * far too expensive for a call that only needs a handful. Both callers below know which files
+ * can possibly matter: ranking needs the files that produced a hit, and the injection card needs
+ * only the two layers it quotes. Reading the rest was pure I/O.
+ * @param files - paths, absolute or root-relative with forward slashes; duplicates are ignored.
+ * @returns the same entry shape as {@link buildLedger}, for the files that resolved to a layer.
+ */
+export function buildLedgerFor(root, files) {
+  const entries = []
+  const seen = new Set()
+  for (const file of files) {
+    if (!file) continue
+    const abs = isAbsolute(file) ? file : join(root, String(file).split('/').join(sep))
+    const rel = relative(root, abs).replace(/\\/g, '/')
+    if (seen.has(rel)) continue
+    seen.add(rel)
+    const layer = layerOfFile(root, abs)
+    if (!layer) continue
+    let text
+    try { text = readFileSync(abs, 'utf8') } catch { continue }
+    for (const e of parseEntries(text)) entries.push({ ...e, file: rel, layer })
+  }
+  return entries
+}

package/src/search.js

@@ -7,7 +7,7 @@
 import { existsSync } from 'node:fs'
 import { DatabaseSync } from 'node:sqlite'
 import { buildFtsQuery, buildIndex, buildLikeQuery, buildTrigramQuery, indexPath, needsSubstring, SUBSTRING_TABLE } from './fts.js'
-import { buildLedger } from './ledger.js'
+import { buildLedgerFor } from './ledger.js'
 import { rankBlend } from './rank.js'
 
 /** Rows pulled from the word index before ranking. */
@@ -67,7 +67,9 @@ export function search(store, query, { limit = 10, scope, key } = {}) {
     if (scope === 'global') filtered = rows.filter((r) => r.file.startsWith('global/'))
     else if (scope === 'workspace') filtered = rows.filter((r) => r.file.startsWith('workspaces/' + (key ?? '') + '/'))
     const hits = filtered.map((r) => ({ file: r.file, line: r.line, section: r.section, snippet: String(r.content).trim() }))
-    return rankBlend(hits, buildLedger(store.root)).slice(0, limit)
+    // Ranking needs importance and recency only for the files that matched. Building the whole
+    // ledger here read ~8,000 files for a handful of hits and made one query cost ~15 s.
+    return rankBlend(hits, buildLedgerFor(store.root, hits.map((h) => h.file))).slice(0, limit)
       .map((h) => ({ file: h.file, line: h.line, section: h.section ?? '', snippet: h.snippet, id: h.id ?? '', score: Math.round(h.score * 1000) / 1000 }))
   } finally {
     db.close()

package/scripts/probe-open.ts

@@ -1,13 +0,0 @@
-import { Context } from "@deepseek-ai/cordis"
-import { SessionStore } from "@deepseek-ai/dsh-session"
-import JsonlSessionPersistence from "@deepseek-ai/dsh-session-persistence-jsonl"
-const ctx = new Context()
-await ctx.plugin(SessionStore)
-await ctx.plugin(JsonlSessionPersistence, { root: "C:/Users/erikb/.dsh/sessions", compression: "zstd" })
-const p = ctx.get("sessionPersistence")
-const id = "session-b21ecc7e-7743-4b06-b07f-453541dbcde5"
-try {
-  const h = await p.open(id, "read")
-  const snap = await h.read()
-  console.log("REAL OPEN OK — events:", snap?.events?.length ?? snap?.log?.length ?? "(shape)")
-} catch (e) { console.log("REAL OPEN FAILED:", String(e.message).slice(0, 300)) }

package/scripts/probe-rows.ts

@@ -1,16 +0,0 @@
-import { readFileSync } from "node:fs"
-import { scanZstdFrames, decompressZstdFrame, decompressZstdPrefix } from "file:///D:/deepseek-harness/packages/session/session-persistence-jsonl/src/zstd.ts"
-import { validateStoredEvents } from "file:///D:/deepseek-harness/packages/session/session-persistence/src/storage-contract.ts"
-const buf = readFileSync(process.argv[2])
-const scan = scanZstdFrames(buf)
-let text = ""
-for (const f of scan.frames) text += (await decompressZstdFrame(buf.subarray(f.start, f.end))).toString("utf8")
-if (scan.tornStart !== undefined) text += (await decompressZstdPrefix(buf.subarray(scan.tornStart))).toString("utf8")
-const rows = text.split("\n").filter(Boolean).map(l => JSON.parse(l))
-const meta = rows[0]; const events = rows.slice(1)
-const dups = new Map(); for (const e of events) dups.set(e.seq, (dups.get(e.seq) ?? 0) + 1)
-console.log("events:", events.length, "| duplicate seqs:", [...dups].filter(([,n]) => n > 1).slice(0, 5))
-const bad = events.filter(e => e.type === "memory/map" && e.ignorable !== true)
-console.log("memory/map without marker:", bad.length)
-try { validateStoredEvents(meta, events, { kind: "jsonl", path: process.argv[2] }); console.log("VALIDATOR: OK") }
-catch (e) { console.log("VALIDATOR THREW:", String(e.message).slice(0, 300)) }