@try-works/dsh-memory 0.1.30 → 0.1.31

package/STATUS.md

@@ -3,9 +3,26 @@
 A tracker and planner: what exists, what is verified working, what is written but not
 loaded, and what is left. Kept as a record of MEASURED state, not intention.
 
-**Last measured: 2026-10-17** (408 tests passing; plugin at 0.1.28, published and installed
-byte-identical and confirmed LOADED by the running harness at 0.2.1-rc.2; queue terminal-clean;
-the Memory tab reads the published map under the new event name).
+**Last measured: 2026-11-01** (409 tests passing; plugin at 0.1.30, published and installed
+byte-identical on harness 0.2.1-rc.2; queue terminal-clean; the Memory tab reads the published
+map under the new event name; the FTS index is 148.6 MB after a rebuild that used to be 436 MB,
+search measures 46-274 ms and an injection 3.3 s).
+
+## Store performance, 2026-11-01
+
+Three measurements that shaped 0.1.29/0.1.30, kept because each one is a design constraint
+rather than a one-off number:
+
+| what | before | after | why it mattered |
+|---|---|---|---|
+| full FTS rebuild | did not finish in 10 min | 43-50 s | every row was its own transaction (~292,000 commits). Batching made "the index is derived, rebuild it" safe |
+| `memory_search` | ~15,000 ms | 46-274 ms | `search()` built the WHOLE ledger (~8,000 files) just to rank the few files that matched; it now reads only the hit files |
+| injection | ~17,000 ms | 3,300 ms | the card quotes only learnings and decisions, but built the ledger for every layer including 425 history shards and 3,010 semantic entries |
+| index file | 436.2 MB | 148.6 MB | free-page bloat from incremental deletes; a rebuild reclaims it and the trigram table (345 MB of the old file) is mandatory, so this is the floor, not a saving to chase |
+
+Reading this store costs ~2.4 ms per file (8.5 MB takes ~19 s), so any code path that walks all
+~8,000 files is a multi-second operation by construction. That is the rule the two fixes above
+came from.
 
 ## How to read this
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@try-works/dsh-memory",
-  "version": "0.1.30",
+  "version": "0.1.31",
   "description": "Git-backed, workspace-scoped memory for the DeepSeek Harness (global layer + per-workspace layers)",
   "type": "module",
   "main": "src/index.js",

package/src/capture.js

@@ -10,6 +10,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync,
 import { join } from 'node:path'
 import { projectDigest, classifyJunk, extractCandidates, renderDigestHeader, totalToolCalls, dateOf } from './digest.js'
 import { summarizeSession, renderSummarySection } from './summarize.js'
+import { compactDigest } from './compact.js'
 import { advance, fail, isSettled, readQueue, REASONS, seed, STATES } from './queue.js'
 
 /** Digest file format. A stored state with a different value forces a full re-capture. */
@@ -127,6 +128,11 @@ export async function captureSession(store, services, session, options = {}) {
       : { bullets: [], source: "thin" })
   if (summary.bullets.length > 0) {
     appendFileSync(path, '\n' + renderSummarySection(summary.bullets, { stamp: meta.capturedAt, source: summary.source }), 'utf8')
+    // A long-lived session adds a section on every pass, and consolidation reads only the first
+    // and last 20 KB of a digest - so without this the middle grew forever and was never read.
+    // Folding is safe: entries already written are durable, and the recap keeps one line of every
+    // folded section. Bounded to the config budget so a raise is a deliberate choice.
+    try { compactDigest(path, { keepBytes: cfg.compactKeepBytes ?? 14000, maxRecapBytes: cfg.compactMaxRecapBytes ?? 4000 }) } catch {}
   }
 
   // SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID

package/src/compact.js

@@ -0,0 +1,126 @@
+// Fold the OLD summary sections of one digest into a bounded recap.
+//
+// A digest is append-only by nature: every capture pass adds one `## <stamp> - summary (llm)`
+// section, because each pass summarizes only the new events. A long-lived session therefore
+// accumulates hundreds of sections - one was measured at 381 sections / 572 KB - while
+// consolidation reads at most `maxDigestBytes` (20 KB) of it, kept as 60% head and 40% tail. So
+// the middle of a large digest was NEVER read, and every later pass re-read the same head.
+//
+// Compaction keeps the header, folds the older sections into one recap of sampled bullets, and
+// leaves the newest sections intact, so a digest fits in ONE consolidation pass. It never grows
+// the file and never drops a section without keeping a line of it in the recap.
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+
+/** A summary section heading. */
+const SUMMARY_HEADING = /^## .+ - summary \(.+\)$/
+/** The recap this module writes, which is part of the header, never a foldable section. */
+const RECAP_BLOCK = /^## compacted recap[^\n]*\n(?:- [^\n]*\n?)*/m
+
+/**
+ * Split a digest into its header and its summary sections.
+ *
+ * The recap is deliberately part of the header: a recap is already the compressed form of older
+ * sections, so folding it again would re-compress it on every pass and the file would never settle.
+ * @returns header (everything before the first summary section) and the summary sections.
+ */
+function splitSections(text) {
+  const lines = text.split('\n')
+  const starts = []
+  for (let i = 0; i < lines.length; i += 1) {
+    if (SUMMARY_HEADING.test(lines[i])) starts.push(i)
+  }
+  if (starts.length === 0) return { header: text, sections: [] }
+  const header = lines.slice(0, starts[0]).join('\n') + (starts[0] > 0 ? '\n' : '')
+  const sections = starts.map((start, index) => {
+    const end = index + 1 < starts.length ? starts[index + 1] : lines.length
+    return { text: lines.slice(start, end).join('\n') }
+  })
+  return { header, sections }
+}
+
+/** The timestamp a section is stamped with: `## <stamp> - summary`, or a recap's own range. */
+function stampOf(section) {
+  const line = section.text.split('\n', 1)[0]
+  const match = /^## (.+?) - summary \(/.exec(line)
+  if (match) return match[1].trim()
+  const range = /^## compacted recap[^(]*\(([^.]*)\.\./.exec(line)
+  return range ? range[1].trim() : ''
+}
+
+/** The first bullet of a section, which is the line most likely to carry its point. */
+function firstBullet(section) {
+  for (const line of section.text.split('\n')) {
+    if (line.startsWith('- ')) return line.slice(2).trim()
+  }
+  return ''
+}
+
+/**
+ * Render the recap that replaces the folded sections.
+ *
+ * Bullets are SAMPLED evenly rather than taken from the front, so the recap spans the whole
+ * folded range instead of over-representing its first few summaries. A previous recap, when one
+ * exists, contributes its bullets first: they describe older material than the sections being
+ * folded now, and keeping them is what makes repeated compactions accumulate context.
+ * @param folded - the sections being replaced, oldest first.
+ * @param options.maxRecapBytes - hard ceiling for the rendered recap.
+ * @param options.prior - the previous recap block, if the header had one.
+ * @returns the recap block.
+ */
+function renderRecap(folded, { maxRecapBytes, prior = '' }) {
+  const first = stampOf(folded[0])
+  const last = stampOf(folded[folded.length - 1])
+  let out = '## compacted recap - ' + folded.length + ' earlier summaries' + (first || last ? ' (' + first + '..' + last + ')' : '') + '\n'
+  if (prior) {
+    for (const row of prior.split('\n')) {
+      if (!row.startsWith('- ')) continue
+      if (out.length + row.length + 1 > maxRecapBytes) break
+      out += row + '\n'
+    }
+  }
+  const step = Math.max(1, Math.ceil(folded.length / 80))
+  for (let i = 0; i < folded.length; i += step) {
+    const bullet = firstBullet(folded[i])
+    if (!bullet) continue
+    const row = '- ' + bullet.slice(0, 180) + '\n'
+    if (out.length + row.length > maxRecapBytes) break
+    out += row
+  }
+  return out + '\n'
+}
+
+/**
+ * Compact one digest in place.
+ * @param path - the digest file.
+ * @param options.keepBytes - byte budget for the newest sections kept verbatim.
+ * @param options.maxRecapBytes - ceiling for the recap that replaces the folded ones.
+ * @returns what it did: `folded` is 0 when the digest already fits.
+ */
+export function compactDigest(path, { keepBytes = 14000, maxRecapBytes = 4000 } = {}) {
+  if (!existsSync(path)) return { folded: 0 }
+  const before = readFileSync(path, 'utf8')
+  const { header, sections } = splitSections(before)
+  if (sections.length === 0) return { folded: 0 }
+  // An earlier recap lives in the header. Its bullets are carried into the new recap rather than
+  // dropped, so repeated compactions accumulate context instead of erasing it one level at a time.
+  const priorMatch = RECAP_BLOCK.exec(header)
+  const prior = priorMatch ? priorMatch[0] : ''
+  const headerBase = prior ? header.replace(prior, '').replace(/\n{3,}/g, '\n\n') : header
+  // Keep the NEWEST sections that fit the budget, and always at least one.
+  const kept = []
+  let keptBytes = 0
+  for (let i = sections.length - 1; i >= 0; i -= 1) {
+    const size = sections[i].text.length
+    if (kept.length > 0 && keptBytes + size > keepBytes) break
+    kept.unshift(sections[i])
+    keptBytes += size
+  }
+  const folded = sections.slice(0, sections.length - kept.length)
+  if (folded.length === 0) return { folded: 0, kept: kept.length }
+  const recap = renderRecap(folded, { maxRecapBytes, prior })
+  const after = headerBase + recap + kept.map((s) => s.text).join('')
+  // A compaction that does not shrink the file is not worth the write; leave it alone.
+  if (after.length >= before.length) return { folded: 0, kept: kept.length }
+  writeFileSync(path, after, 'utf8')
+  return { folded: folded.length, kept: kept.length, bytesBefore: before.length, bytesAfter: after.length }
+}