@try-works/dsh-memory 0.1.29 → 0.1.31

package/STATUS.md

@@ -3,9 +3,26 @@
 A tracker and planner: what exists, what is verified working, what is written but not
 loaded, and what is left. Kept as a record of MEASURED state, not intention.
 
-**Last measured: 2026-10-17** (408 tests passing; plugin at 0.1.28, published and installed
-byte-identical and confirmed LOADED by the running harness at 0.2.1-rc.2; queue terminal-clean;
-the Memory tab reads the published map under the new event name).
+**Last measured: 2026-11-01** (409 tests passing; plugin at 0.1.30, published and installed
+byte-identical on harness 0.2.1-rc.2; queue terminal-clean; the Memory tab reads the published
+map under the new event name; the FTS index is 148.6 MB after a rebuild that used to be 436 MB,
+search measures 46-274 ms and an injection 3.3 s).
+
+## Store performance, 2026-11-01
+
+Three measurements that shaped 0.1.29/0.1.30, kept because each one is a design constraint
+rather than a one-off number:
+
+| what | before | after | why it mattered |
+|---|---|---|---|
+| full FTS rebuild | did not finish in 10 min | 43-50 s | every row was its own transaction (~292,000 commits). Batching made "the index is derived, rebuild it" safe |
+| `memory_search` | ~15,000 ms | 46-274 ms | `search()` built the WHOLE ledger (~8,000 files) just to rank the few files that matched; it now reads only the hit files |
+| injection | ~17,000 ms | 3,300 ms | the card quotes only learnings and decisions, but built the ledger for every layer including 425 history shards and 3,010 semantic entries |
+| index file | 436.2 MB | 148.6 MB | free-page bloat from incremental deletes; a rebuild reclaims it and the trigram table (345 MB of the old file) is mandatory, so this is the floor, not a saving to chase |
+
+Reading this store costs ~2.4 ms per file (8.5 MB takes ~19 s), so any code path that walks all
+~8,000 files is a multi-second operation by construction. That is the rule the two fixes above
+came from.
 
 ## How to read this
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@try-works/dsh-memory",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "description": "Git-backed, workspace-scoped memory for the DeepSeek Harness (global layer + per-workspace layers)",
   "type": "module",
   "main": "src/index.js",

package/src/capture.js

@@ -10,6 +10,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync,
 import { join } from 'node:path'
 import { projectDigest, classifyJunk, extractCandidates, renderDigestHeader, totalToolCalls, dateOf } from './digest.js'
 import { summarizeSession, renderSummarySection } from './summarize.js'
+import { compactDigest } from './compact.js'
 import { advance, fail, isSettled, readQueue, REASONS, seed, STATES } from './queue.js'
 
 /** Digest file format. A stored state with a different value forces a full re-capture. */
@@ -127,6 +128,11 @@ export async function captureSession(store, services, session, options = {}) {
       : { bullets: [], source: "thin" })
   if (summary.bullets.length > 0) {
     appendFileSync(path, '\n' + renderSummarySection(summary.bullets, { stamp: meta.capturedAt, source: summary.source }), 'utf8')
+    // A long-lived session adds a section on every pass, and consolidation reads only the first
+    // and last 20 KB of a digest - so without this the middle grew forever and was never read.
+    // Folding is safe: entries already written are durable, and the recap keeps one line of every
+    // folded section. Bounded to the config budget so a raise is a deliberate choice.
+    try { compactDigest(path, { keepBytes: cfg.compactKeepBytes ?? 14000, maxRecapBytes: cfg.compactMaxRecapBytes ?? 4000 }) } catch {}
   }
 
   // SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID

package/src/compact.js

@@ -0,0 +1,126 @@
+// Fold the OLD summary sections of one digest into a bounded recap.
+//
+// A digest is append-only by nature: every capture pass adds one `## <stamp> - summary (llm)`
+// section, because each pass summarizes only the new events. A long-lived session therefore
+// accumulates hundreds of sections - one was measured at 381 sections / 572 KB - while
+// consolidation reads at most `maxDigestBytes` (20 KB) of it, kept as 60% head and 40% tail. So
+// the middle of a large digest was NEVER read, and every later pass re-read the same head.
+//
+// Compaction keeps the header, folds the older sections into one recap of sampled bullets, and
+// leaves the newest sections intact, so a digest fits in ONE consolidation pass. It never grows
+// the file and never drops a section without keeping a line of it in the recap.
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+
+/** A summary section heading. */
+const SUMMARY_HEADING = /^## .+ - summary \(.+\)$/
+/** The recap this module writes, which is part of the header, never a foldable section. */
+const RECAP_BLOCK = /^## compacted recap[^\n]*\n(?:- [^\n]*\n?)*/m
+
+/**
+ * Split a digest into its header and its summary sections.
+ *
+ * The recap is deliberately part of the header: a recap is already the compressed form of older
+ * sections, so folding it again would re-compress it on every pass and the file would never settle.
+ * @returns header (everything before the first summary section) and the summary sections.
+ */
+function splitSections(text) {
+  const lines = text.split('\n')
+  const starts = []
+  for (let i = 0; i < lines.length; i += 1) {
+    if (SUMMARY_HEADING.test(lines[i])) starts.push(i)
+  }
+  if (starts.length === 0) return { header: text, sections: [] }
+  const header = lines.slice(0, starts[0]).join('\n') + (starts[0] > 0 ? '\n' : '')
+  const sections = starts.map((start, index) => {
+    const end = index + 1 < starts.length ? starts[index + 1] : lines.length
+    return { text: lines.slice(start, end).join('\n') }
+  })
+  return { header, sections }
+}
+
+/** The timestamp a section is stamped with: `## <stamp> - summary`, or a recap's own range. */
+function stampOf(section) {
+  const line = section.text.split('\n', 1)[0]
+  const match = /^## (.+?) - summary \(/.exec(line)
+  if (match) return match[1].trim()
+  const range = /^## compacted recap[^(]*\(([^.]*)\.\./.exec(line)
+  return range ? range[1].trim() : ''
+}
+
+/** The first bullet of a section, which is the line most likely to carry its point. */
+function firstBullet(section) {
+  for (const line of section.text.split('\n')) {
+    if (line.startsWith('- ')) return line.slice(2).trim()
+  }
+  return ''
+}
+
+/**
+ * Render the recap that replaces the folded sections.
+ *
+ * Bullets are SAMPLED evenly rather than taken from the front, so the recap spans the whole
+ * folded range instead of over-representing its first few summaries. A previous recap, when one
+ * exists, contributes its bullets first: they describe older material than the sections being
+ * folded now, and keeping them is what makes repeated compactions accumulate context.
+ * @param folded - the sections being replaced, oldest first.
+ * @param options.maxRecapBytes - hard ceiling for the rendered recap.
+ * @param options.prior - the previous recap block, if the header had one.
+ * @returns the recap block.
+ */
+function renderRecap(folded, { maxRecapBytes, prior = '' }) {
+  const first = stampOf(folded[0])
+  const last = stampOf(folded[folded.length - 1])
+  let out = '## compacted recap - ' + folded.length + ' earlier summaries' + (first || last ? ' (' + first + '..' + last + ')' : '') + '\n'
+  if (prior) {
+    for (const row of prior.split('\n')) {
+      if (!row.startsWith('- ')) continue
+      if (out.length + row.length + 1 > maxRecapBytes) break
+      out += row + '\n'
+    }
+  }
+  const step = Math.max(1, Math.ceil(folded.length / 80))
+  for (let i = 0; i < folded.length; i += step) {
+    const bullet = firstBullet(folded[i])
+    if (!bullet) continue
+    const row = '- ' + bullet.slice(0, 180) + '\n'
+    if (out.length + row.length > maxRecapBytes) break
+    out += row
+  }
+  return out + '\n'
+}
+
+/**
+ * Compact one digest in place.
+ * @param path - the digest file.
+ * @param options.keepBytes - byte budget for the newest sections kept verbatim.
+ * @param options.maxRecapBytes - ceiling for the recap that replaces the folded ones.
+ * @returns what it did: `folded` is 0 when the digest already fits.
+ */
+export function compactDigest(path, { keepBytes = 14000, maxRecapBytes = 4000 } = {}) {
+  if (!existsSync(path)) return { folded: 0 }
+  const before = readFileSync(path, 'utf8')
+  const { header, sections } = splitSections(before)
+  if (sections.length === 0) return { folded: 0 }
+  // An earlier recap lives in the header. Its bullets are carried into the new recap rather than
+  // dropped, so repeated compactions accumulate context instead of erasing it one level at a time.
+  const priorMatch = RECAP_BLOCK.exec(header)
+  const prior = priorMatch ? priorMatch[0] : ''
+  const headerBase = prior ? header.replace(prior, '').replace(/\n{3,}/g, '\n\n') : header
+  // Keep the NEWEST sections that fit the budget, and always at least one.
+  const kept = []
+  let keptBytes = 0
+  for (let i = sections.length - 1; i >= 0; i -= 1) {
+    const size = sections[i].text.length
+    if (kept.length > 0 && keptBytes + size > keepBytes) break
+    kept.unshift(sections[i])
+    keptBytes += size
+  }
+  const folded = sections.slice(0, sections.length - kept.length)
+  if (folded.length === 0) return { folded: 0, kept: kept.length }
+  const recap = renderRecap(folded, { maxRecapBytes, prior })
+  const after = headerBase + recap + kept.map((s) => s.text).join('')
+  // A compaction that does not shrink the file is not worth the write; leave it alone.
+  if (after.length >= before.length) return { folded: 0, kept: kept.length }
+  writeFileSync(path, after, 'utf8')
+  return { folded: folded.length, kept: kept.length, bytesBefore: before.length, bytesAfter: after.length }
+}

package/src/inject.js

@@ -1,9 +1,9 @@
 // Session-start injection text: goal + working head + recent learnings + footer,
 // char-capped and scope-aware.
-import { existsSync, readFileSync } from 'node:fs'
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { layerDir } from './store.js'
-import { buildLedger } from './ledger.js'
+import { buildLedgerFor } from './ledger.js'
 import { readLayerTopics } from './layers.js'
 import { freshnessMarker, readState } from './state.js'
 import { readWorking, workingMarker } from './working.js'
@@ -12,7 +12,11 @@ import { relevanceQuery, selectEntries } from './select.js'
 export function buildInjectionText(store, { scope = 'global', key, goal, maxChars = 3600, subagent = false } = {}) {
   const dir = layerDir(store.root, scope, key)
   // Working memory is the AGENT's intent (task, recent activity, next steps); repository
-  const ledger = buildLedger(store.root)
+  //
+  // Only the two layers this card actually quotes are read. Building the whole ledger here meant
+  // walking every indexed file - ~8,000 of them, history shards included - on every injection,
+  // which measured ~17 s and dwarfed the text it produced.
+  const ledger = buildLedgerFor(store.root, layerFilesFor(dir, ['learnings', 'decisions']))
   const prefix = scope === 'global' ? 'global/' : 'workspaces/' + key + '/'
   // The pools are LISTED here and CHOSEN later, because choosing needs the question, and the
   // question comes from working memory - which is read below.
@@ -106,3 +110,24 @@ export function buildInjectionText(store, { scope = 'global', key, goal, maxChar
   const body = parts.join('\n\n')
   return (body.length <= maxChars ? body : body.slice(0, maxChars)) + '\n\n' + footer
 }
+
+/**
+ * The absolute file paths of the named layer directories in one scope.
+ *
+ * Layer directories are flat (one topic file per entry), so listing them is cheap. This exists
+ * because the injection card quotes only learnings and decisions: reading every OTHER layer -
+ * semantic, episodic, procedural, codebase, history - was the expensive part of the old
+ * whole-store ledger, and none of it could ever appear in the card.
+ * @param dir - the scope directory (a layer dir already), from {@link layerDir}.
+ * @param layers - directory names under it, e.g. 'learnings' and 'decisions'.
+ * @returns absolute paths, in directory order.
+ */
+function layerFilesFor(dir, layers) {
+  const files = []
+  for (const name of layers) {
+    const layerPath = join(dir, name)
+    if (!existsSync(layerPath)) continue
+    for (const file of readdirSync(layerPath)) if (file.endsWith('.md')) files.push(join(layerPath, file))
+  }
+  return files
+}

package/src/ledger.js

@@ -1,19 +1,47 @@
-// Derived entry ledger: per-entry metadata (file, layer, line span, importance,
-// dates) rebuilt from the markdown in one step. Feeds ranking + freshness.
-import { readFileSync } from 'node:fs'
-import { relative } from 'node:path'
-import { indexedFiles, layerOfFile } from './files.js'
-import { parseEntries } from './entry.js'
-
-export function buildLedger(root) {
-  const entries = []
-  for (const file of indexedFiles(root)) {
-    const rel = relative(root, file).replace(/\\/g, '/')
-    const layer = layerOfFile(root, file)
-    if (!layer) continue
-    for (const e of parseEntries(readFileSync(file, 'utf8'))) {
-      entries.push({ ...e, file: rel, layer })
-    }
-  }
-  return entries
-}
+// Derived entry ledger: per-entry metadata (file, layer, line span, importance,
+// dates) rebuilt from the markdown in one step. Feeds ranking + freshness.
+import { readFileSync } from 'node:fs'
+import { isAbsolute, join, relative, sep } from 'node:path'
+import { indexedFiles, layerOfFile } from './files.js'
+import { parseEntries } from './entry.js'
+
+export function buildLedger(root) {
+  const entries = []
+  for (const file of indexedFiles(root)) {
+    const rel = relative(root, file).replace(/\\/g, '/')
+    const layer = layerOfFile(root, file)
+    if (!layer) continue
+    for (const e of parseEntries(readFileSync(file, 'utf8'))) {
+      entries.push({ ...e, file: rel, layer })
+    }
+  }
+  return entries
+}
+
+/**
+ * Build entries for a KNOWN SET of files instead of the whole store.
+ *
+ * The full ledger reads every indexed file, and on this store that is ~8,000 files and ~17 s —
+ * far too expensive for a call that only needs a handful. Both callers below know which files
+ * can possibly matter: ranking needs the files that produced a hit, and the injection card needs
+ * only the two layers it quotes. Reading the rest was pure I/O.
+ * @param files - paths, absolute or root-relative with forward slashes; duplicates are ignored.
+ * @returns the same entry shape as {@link buildLedger}, for the files that resolved to a layer.
+ */
+export function buildLedgerFor(root, files) {
+  const entries = []
+  const seen = new Set()
+  for (const file of files) {
+    if (!file) continue
+    const abs = isAbsolute(file) ? file : join(root, String(file).split('/').join(sep))
+    const rel = relative(root, abs).replace(/\\/g, '/')
+    if (seen.has(rel)) continue
+    seen.add(rel)
+    const layer = layerOfFile(root, abs)
+    if (!layer) continue
+    let text
+    try { text = readFileSync(abs, 'utf8') } catch { continue }
+    for (const e of parseEntries(text)) entries.push({ ...e, file: rel, layer })
+  }
+  return entries
+}

package/src/search.js

@@ -7,7 +7,7 @@
 import { existsSync } from 'node:fs'
 import { DatabaseSync } from 'node:sqlite'
 import { buildFtsQuery, buildIndex, buildLikeQuery, buildTrigramQuery, indexPath, needsSubstring, SUBSTRING_TABLE } from './fts.js'
-import { buildLedger } from './ledger.js'
+import { buildLedgerFor } from './ledger.js'
 import { rankBlend } from './rank.js'
 
 /** Rows pulled from the word index before ranking. */
@@ -67,7 +67,9 @@ export function search(store, query, { limit = 10, scope, key } = {}) {
     if (scope === 'global') filtered = rows.filter((r) => r.file.startsWith('global/'))
     else if (scope === 'workspace') filtered = rows.filter((r) => r.file.startsWith('workspaces/' + (key ?? '') + '/'))
     const hits = filtered.map((r) => ({ file: r.file, line: r.line, section: r.section, snippet: String(r.content).trim() }))
-    return rankBlend(hits, buildLedger(store.root)).slice(0, limit)
+    // Ranking needs importance and recency only for the files that matched. Building the whole
+    // ledger here read ~8,000 files for a handful of hits and made one query cost ~15 s.
+    return rankBlend(hits, buildLedgerFor(store.root, hits.map((h) => h.file))).slice(0, limit)
       .map((h) => ({ file: h.file, line: h.line, section: h.section ?? '', snippet: h.snippet, id: h.id ?? '', score: Math.round(h.score * 1000) / 1000 }))
   } finally {
     db.close()