dsh-memory-dba 0.0.0-stage → 0.1.0

package/src/index.mjs

@@ -0,0 +1,240 @@
+/**
+ * dsh-memory-dba — a file-based, git-portable memory plugin for DeepSeek Harness.
+ *
+ * Three layers, mirroring how the notes are actually maintained:
+ *   core.md      injected in full every turn, budget-capped
+ *   stream.md    a timeline, retrieved only
+ *   ws/*.md      detail notes, retrieved only
+ *
+ * The plugin owns injection and turn recording; retrieval is BM25 over all three
+ * layers, so the package has no runtime outside Node itself.
+ *
+ * Failure policy: memory is an enhancement. A broken layer, a missing directory,
+ * or a corrupt index degrades to an empty section and never fails the turn.
+ */
+
+import { buildCore, buildHits } from './assemble.mjs'
+import { fitHits, loadCorpus, search } from './search.mjs'
+import { bytes, ensureLayout, fingerprint, resolveHome } from './store.mjs'
+import { appendTurn } from './stream.mjs'
+import { queryTerms, tokenize } from './tokenize.mjs'
+import { embed } from './embed.mjs'
+import { docKey, semanticRecall } from './vector.mjs'
+
+export const name = 'memory-dba'
+
+/** The plugin reads and writes prompt sections through this service. */
+export const inject = ['systemPrompt']
+
+/** Config defaults; every field is overridable from cordis.yml. */
+export const defaults = {
+  home: '',
+  coreLimitKB: 15,
+  hitBudgetKB: 6,
+  topK: 5,
+  timeDecay: true,
+  halfLifeDays: 30,
+  writeStream: true,
+  tag: '',
+  exclude: [],
+  coreOrder: -50,
+  hitsOrder: 200,
+  vector: { enabled: true, url: 'http://127.0.0.1:11434', model: 'bge-m3', timeoutMs: 1500, minScore: 0.45, keepAlive: '30m' },
+}
+
+/** Text of one message's content blocks, ignoring tool calls and images. */
+function textOf(blocks) {
+  if (typeof blocks === 'string') return blocks.trim()
+  if (!Array.isArray(blocks)) return ''
+  return blocks
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('\n')
+    .trim()
+}
+
+/** Coerce a number, falling back when the config value is missing or invalid. */
+function num(value, fallback, { min = -Infinity, max = Infinity } = {}) {
+  const parsed = Number(value)
+  if (!Number.isFinite(parsed)) return fallback
+  return Math.min(max, Math.max(min, parsed))
+}
+
+/** Merge user config over defaults, rejecting unusable values instead of propagating NaN. */
+export function normalizeConfig(config = {}) {
+  const cfg = { ...defaults, ...config }
+  const vector = cfg.vector && typeof cfg.vector === 'object' ? cfg.vector : {}
+  const timeoutMs = vector.timeoutMs == null ? NaN : Number(vector.timeoutMs)
+  const minScore = vector.minScore == null ? NaN : Number(vector.minScore)
+  const keepAlive = typeof vector.keepAlive === 'string' && vector.keepAlive.trim() ? vector.keepAlive.trim() : defaults.vector.keepAlive
+  let url = defaults.vector.url
+  try {
+    const parsed = new URL(vector.url)
+    if (['http:', 'https:'].includes(parsed.protocol)) url = parsed.href.replace(/\/$/, '')
+  } catch {}
+  return {
+    ...cfg,
+    home: typeof cfg.home === 'string' ? cfg.home : '',
+    coreLimitKB: num(cfg.coreLimitKB, defaults.coreLimitKB, { min: 1, max: 512 }),
+    hitBudgetKB: num(cfg.hitBudgetKB, defaults.hitBudgetKB, { min: 1, max: 128 }),
+    topK: num(cfg.topK, defaults.topK, { min: 1, max: 50 }),
+    halfLifeDays: num(cfg.halfLifeDays, defaults.halfLifeDays, { min: 1, max: 3650 }),
+    coreOrder: num(cfg.coreOrder, defaults.coreOrder),
+    hitsOrder: num(cfg.hitsOrder, defaults.hitsOrder),
+    timeDecay: cfg.timeDecay !== false,
+    writeStream: cfg.writeStream !== false,
+    tag: typeof cfg.tag === 'string' ? cfg.tag : '',
+    exclude: Array.isArray(cfg.exclude) ? cfg.exclude.filter(name => typeof name === 'string') : [],
+    vector: {
+      enabled: vector.enabled !== false,
+      url,
+      model: typeof vector.model === 'string' && vector.model.trim() ? vector.model : defaults.vector.model,
+      timeoutMs: Number.isFinite(timeoutMs) && timeoutMs >= 1 ? Math.floor(timeoutMs) : defaults.vector.timeoutMs,
+      minScore: Number.isFinite(minScore) && minScore >= 0 && minScore <= 1 ? minScore : defaults.vector.minScore,
+      keepAlive,
+    },
+  }
+}
+
+export function recall(docs, query, options = {}) {
+  const first = search(docs, query, options)
+  const terms = queryTerms(query)
+  const shared = docs.some(doc => tokenize(doc.text).some(term => terms.has(term)))
+  if (!shared && (!first.hits.length || first.hits[0].score < 6)) {
+    const reduced = tokenize(query)
+      .filter(term => !/^[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]{2}$/.test(term))
+      .join(' ')
+      || [...query].filter(char => /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF]/.test(char)).join(' ')
+    const second = reduced ? search(docs, reduced, options) : { hits: [] }
+    if (second.hits.length) return { ...second, relaxed: true }
+  }
+  return { ...first, relaxed: false }
+}
+
+/**
+ * @param {import('@deepseek-ai/cordis').Context} ctx
+ * @param {Partial<typeof defaults>} config
+ */
+export function apply(ctx, config = {}) {
+  const cfg = normalizeConfig(config)
+  const home = resolveHome(cfg.home)
+
+  try {
+    ensureLayout(home)
+  } catch (error) {
+    // An unwritable home disables persistence but must not break startup.
+    console.error(`[memory-dba] cannot initialize ${home}: ${error.message}`)
+  }
+
+  // A cold Ollama takes longer than the synchronous recall budget, so warm it in the
+  // background; the first turn would otherwise lose semantic recall.
+  if (cfg.vector.enabled) {
+    embed('warmup', { url: cfg.vector.url, model: cfg.vector.model, timeoutMs: 8000, keepAlive: cfg.vector.keepAlive })
+      .catch(() => {})
+  }
+
+  let corpus = null
+  let corpusKey = ''
+  // The assembly context carries only a scope key, not a session id, so recall text
+  // is single-slot: the most recent user turn wins. Concurrent sessions (subagents)
+  // therefore render the last query, not their own.
+  let latestHits = ''
+  const pending = new Map()
+
+  const docs = () => {
+    const key = fingerprint(home)
+    if (corpus && key === corpusKey) return corpus
+    corpus = loadCorpus(home, { exclude: cfg.exclude })
+    corpusKey = key
+    return corpus
+  }
+
+  const recallQuery = query => {
+    if (!query) return { hits: [], applied: false, clipped: false }
+    const corpus = docs()
+    const semantic = cfg.vector.enabled
+      ? semanticRecall(home, query, corpus.map(doc => docKey(doc.text)), { ...cfg.vector, topK: cfg.topK })
+      : null
+    const result = recall(corpus, query, {
+      topK: cfg.topK,
+      timeDecay: cfg.timeDecay,
+      halfLifeDays: cfg.halfLifeDays,
+      budgetBytes: cfg.hitBudgetKB * 1024,
+      semantic,
+      semanticMinScore: cfg.vector.minScore,
+    })
+    const budget = Math.max(1, cfg.hitBudgetKB) * 1024
+    const keyword = result.hits.filter(hit => !hit.semantic)
+    const semanticHits = result.hits.filter(hit => hit.semantic)
+    // Keyword hits own the prompt; semantic excerpts get a reserved tail so a long
+    // workspace section cannot crowd them out entirely.
+    const keptKeyword = fitHits(keyword, semanticHits.length ? Math.floor(budget * 0.7) : budget)
+    const used = keptKeyword.reduce((total, hit) => total + bytes(hit.text) + 48, 0)
+    const hits = used < budget ? keptKeyword.concat(fitHits(semanticHits, budget - used)) : keptKeyword
+    return { ...result, hits }
+  }
+
+  const renderHits = query => {
+    const result = recallQuery(query)
+    return buildHits(result.hits, { budgetKB: cfg.hitBudgetKB })
+  }
+
+  ctx.on('session/disposed', session => {
+    pending.delete(session?.id)
+  })
+
+  ctx.on('session/event', (session, event) => {
+    try {
+      const sessionId = session?.id
+      if (!sessionId) return
+      if (event?.type === 'user/message') {
+        if (event.data?.source?.kind !== 'user') return
+        pending.delete(sessionId)
+        const user = textOf(event.data?.content)
+        if (!user) return
+        pending.set(sessionId, { user, assistant: '', at: new Date() })
+        latestHits = renderHits(user).text
+        return
+      }
+      if (event?.type === 'assistant/message') {
+        const turn = pending.get(sessionId)
+        if (!turn) return
+        const text = textOf(event.data?.message?.content)
+        if (!text) return
+        turn.assistant = turn.assistant ? `${turn.assistant}\n${text}` : text
+        return
+      }
+      if (event?.type === 'turn/end') {
+        const turn = pending.get(sessionId)
+        if (!turn) return
+        pending.delete(sessionId)
+        if (!cfg.writeStream) return
+        const outcome = appendTurn(home, { ...turn, tag: cfg.tag })
+        if (outcome.written) corpus = null
+      }
+    } catch (error) {
+      console.error(`[memory-dba] session event failed: ${error.message}`)
+    }
+  })
+
+  ctx.systemPrompt.section({
+    name: 'memory-dba:core',
+    order: cfg.coreOrder,
+    text: () => {
+      try {
+        return buildCore(home, { coreLimitKB: cfg.coreLimitKB }).text
+      } catch {
+        return ''
+      }
+    },
+  })
+
+  ctx.systemPrompt.section({
+    name: 'memory-dba:hits',
+    order: cfg.hitsOrder,
+    text: () => latestHits,
+  })
+}
+
+/** Exposed for tests: the retrieval path without a harness context. */
+export { buildCore, buildHits, fitHits, loadCorpus, search }

package/src/search.mjs

@@ -0,0 +1,258 @@
+/**
+ * BM25 over the three memory layers, with CJK bigrams and optional recency.
+ *
+ * Retrieval is a two-step ladder that mirrors how the memory is written:
+ *   1. rank by relevance only, so the same query yields the same candidates;
+ *   2. when the candidate set overflows the prompt budget, rerank with a time
+ *      weight so recent notes survive the cut.
+ * Relevance never gets replaced by recency; the weight only reorders candidates
+ * that already matched.
+ */
+
+import { readdirSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { bytes, readLayer } from './store.mjs'
+import { queryTerms, tokenize } from './tokenize.mjs'
+import { docKey } from './vector.mjs'
+
+const K1 = 1.5
+const B = 0.75
+export const STREAM_MAX_DOCS = 1000
+export const STREAM_MAX_BYTES = 128 * 1024
+const DATE_RE = /(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2}))?/
+
+/**
+ * @typedef {{ source: string, line: number, text: string, date: string|null }} MemoryDoc
+ */
+
+/**
+ * Split one layer into retrievable units. Core and stream split per non-empty
+ * line; a workspace file splits per `##` heading so a long note is one hit.
+ *
+ * @param {string} text Layer contents.
+ * @param {string} source Layer label used in results.
+ * @returns {MemoryDoc[]} Documents in source order.
+ */
+export function splitDocs(text, source) {
+  const docs = []
+  const lines = text.replace(/^\uFEFF/, '').split(/\r?\n/)
+  let block = []
+  let start = 1
+  const flush = () => {
+    const body = block.join('\n').trim()
+    if (body) docs.push({ source, line: start, text: body, date: dateOf(body) })
+    block = []
+  }
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i]
+    const heading = source.startsWith('ws/') && /^##\s+/.test(line)
+    if (!line.trim() || heading) {
+      if (block.length) flush()
+    }
+    if (line.trim()) {
+      if (!block.length) start = i + 1
+      block.push(line)
+      if (!source.startsWith('ws/')) flush()
+    }
+  }
+  if (block.length) flush()
+  return docs
+}
+
+/** First `YYYY-MM-DD[ HH:MM]` in the text, normalized to an ISO date string. */
+export function dateOf(text) {
+  const m = DATE_RE.exec(text || '')
+  if (!m) return null
+  const [, y, mo, d, h, mi] = m
+  if (!h) return `${y}-${mo}-${d}`
+  return `${y}-${mo}-${d}T${h}:${mi}`
+}
+
+/** Stream retrieval indexes a bounded recent window with original line numbers. */
+function streamDocs(text) {
+  const lines = text.replace(/^\uFEFF/, '').split(/\r?\n/)
+  const docs = []
+  let used = 0
+  for (let index = lines.length - 1; index >= 0 && docs.length < STREAM_MAX_DOCS; index -= 1) {
+    const body = lines[index].trim()
+    if (!body) continue
+    const cost = bytes(body)
+    if (cost > STREAM_MAX_BYTES) continue
+    if (used + cost > STREAM_MAX_BYTES) break
+    used += cost
+    docs.push({ source: 'stream', line: index + 1, text: body, date: dateOf(body) })
+  }
+  return docs.reverse()
+}
+
+/** Unavailable or invalid layers are skipped unless strict diagnostics are requested. */
+export function loadCorpus(home, { exclude = [], strict = false } = {}) {
+  const docs = []
+  for (const [file, label] of [['core.md', 'core'], ['stream.md', 'stream']]) {
+    try {
+      const text = readLayer(home, file)
+      docs.push(...(label === 'stream' ? streamDocs(text) : splitDocs(text, label)))
+    } catch (error) {
+      if (strict && error?.name !== 'MissingLayer') throw error
+    }
+  }
+  const wsDir = join(home, 'ws')
+  let names = []
+  try {
+    names = readdirSync(wsDir).filter(name => name.endsWith('.md')).sort()
+  } catch (error) {
+    if (strict && error?.code !== 'ENOENT') throw error
+  }
+  const skip = new Set(exclude)
+  for (const name of names) {
+    if (skip.has(name)) continue
+    try {
+      docs.push(...splitDocs(readLayer(home, `ws/${name}`), `ws/${name}`))
+    } catch (error) {
+      if (strict && error?.name !== 'MissingLayer') throw error
+    }
+  }
+  return docs
+}
+
+/**
+ * @typedef {object} SearchOptions
+ * @property {boolean} [timeDecay] Apply recency reranking.
+ * @property {number} [halfLifeDays] Weight halves every N days.
+ * @property {string} [now] Reference date, ISO; defaults to today.
+ * @property {number} [budgetBytes] Rerank only when matching candidates exceed this budget.
+ * @property {Map<string, number>|null} [semantic] Cosines keyed by document-text SHA-1.
+ * @property {number} [semanticMinScore] Cosine floor for appended semantic hits, default 0.45.
+ */
+
+/**
+ * Rank documents for a query.
+ *
+ * @param {MemoryDoc[]} docs Corpus.
+ * @param {string} query User query.
+ * @param {SearchOptions & { topK?: number }} [options]
+ * @returns {{ hits: Array<MemoryDoc & { score: number, base: number }>, applied: boolean, corpusDates: {dated: number, undated: number} }}
+ */
+export function search(docs, query, options = {}) {
+  const topK = Math.max(1, options.topK ?? 5)
+  const terms = queryTerms(query)
+  const empty = { hits: [], applied: false, corpusDates: { dated: 0, undated: 0 } }
+  if ((!terms.size && !options.semantic?.size) || !docs.length) return empty
+
+  const postings = new Map()
+  const lengths = new Array(docs.length)
+  for (let i = 0; i < docs.length; i += 1) {
+    const tokens = tokenize(docs[i].text)
+    lengths[i] = tokens.length || 1
+    const tf = new Map()
+    for (const token of tokens) tf.set(token, (tf.get(token) || 0) + 1)
+    for (const [term, count] of tf) {
+      let list = postings.get(term)
+      if (!list) postings.set(term, (list = new Map()))
+      list.set(i, count)
+    }
+  }
+
+  const total = docs.length
+  const avgLen = lengths.reduce((a, b) => a + b, 0) / total
+  const scores = new Map()
+  for (const [term, boost] of terms) {
+    const list = postings.get(term)
+    if (!list) continue
+    const idf = Math.log(1 + (total - list.size + 0.5) / (list.size + 0.5))
+    for (const [i, tf] of list) {
+      const denom = tf + K1 * (1 - B + (B * lengths[i]) / avgLen)
+      const gain = boost * idf * ((tf * (K1 + 1)) / denom)
+      scores.set(i, (scores.get(i) || 0) + gain)
+    }
+  }
+  const keys = options.semantic?.size ? docs.map(doc => docKey(doc.text)) : null
+  /** Semantic hits extend the keyword list; they never reorder it. */
+  const extras = picked => {
+    if (!keys) return picked
+    const minimum = Number.isFinite(options.semanticMinScore) ? options.semanticMinScore : 0.45
+    const seen = new Set(picked.map(hit => docKey(hit.text)))
+    const found = []
+    for (let index = 0; index < docs.length; index += 1) {
+      if (seen.has(keys[index])) continue
+      const score = options.semantic.get(keys[index])
+      if (Number.isFinite(score) && score >= minimum) found.push({ index, score })
+    }
+    found.sort((left, right) => right.score - left.score || left.index - right.index)
+    return picked.concat(found
+      .slice(0, topK)
+      .map(({ index, score }) => ({ ...docs[index], score: score * 10, base: score * 10, semantic: true })))
+  }
+  let ranked = [...scores.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0])
+  if (!ranked.length) return { ...empty, hits: extras([]) }
+
+  const base = ranked
+    .map(([i, score]) => ({ ...docs[i], score, base: score }))
+
+  const dated = base.filter(hit => hit.date)
+  const corpusDates = { dated: dated.length, undated: base.length - dated.length }
+
+  const withinBudget = Number.isFinite(options.budgetBytes)
+    && base.reduce((total, hit) => total + bytes(hit.text) + 48, 0) <= options.budgetBytes
+  if (!options.timeDecay || !dated.length || withinBudget) {
+    return { hits: extras(base.slice(0, topK)), applied: false, corpusDates }
+  }
+
+  const reference = options.now ? Date.parse(options.now) : Date.now()
+  const halfLife = options.halfLifeDays && options.halfLifeDays > 0 ? options.halfLifeDays : 30
+  const weights = new Map()
+  for (const hit of dated) {
+    const ageDays = Math.max(0, (reference - Date.parse(hit.date)) / 86_400_000) || 0
+    weights.set(hit, Math.max(0.5, 0.5 ** (ageDays / halfLife)))
+  }
+  const sortedWeights = [...weights.values()].sort((a, b) => a - b)
+  const neutral = sortedWeights[Math.floor(sortedWeights.length / 2)]
+  const reranked = base
+    .map((hit, index) => ({ ...hit, weight: weights.get(hit) ?? neutral, docIndex: ranked[index][0] }))
+    .sort((left, right) => right.base * right.weight - left.base * left.weight || left.docIndex - right.docIndex)
+    .map(({ weight, docIndex, ...hit }) => ({ ...hit, score: hit.base * weight }))
+
+  return { hits: extras(reranked.slice(0, topK)), applied: true, corpusDates, timeWeightNeutral: neutral }
+}
+
+/**
+ * Pick hits that fit a byte budget, keeping rank order. A single oversized hit
+ * is kept and reported as truncated by the caller, never silently dropped.
+ *
+ * @param {Array<MemoryDoc & { score: number }>} hits Ranked hits.
+ * @param {number} budgetBytes Prompt budget for the retrieval section.
+ */
+export function fitHits(hits, budgetBytes) {
+  const kept = []
+  let used = 0
+  for (const hit of hits) {
+    const cost = bytes(hit.text) + 48
+    if (used + cost > budgetBytes && kept.length) break
+    kept.push(hit)
+    used += cost
+  }
+  return kept
+}
+
+/** Memory-directory stats for the CLI and for doctor output. */
+export function corpusStats(home) {
+  const stats = { layers: {}, docs: 0, bytes: 0 }
+  for (const name of ['core.md', 'stream.md']) {
+    try {
+      const size = statSync(join(home, name)).size
+      stats.layers[name] = size
+      stats.bytes += size
+    } catch {
+      stats.layers[name] = 0
+    }
+  }
+  try {
+    for (const name of readdirSync(join(home, 'ws')).filter(f => f.endsWith('.md'))) {
+      const size = statSync(join(home, 'ws', name)).size
+      stats.layers[`ws/${name}`] = size
+      stats.bytes += size
+    }
+  } catch { /* no workspace layer yet */ }
+  stats.docs = loadCorpus(home).length
+  return stats
+}

package/src/store.mjs

@@ -0,0 +1,171 @@
+/**
+ * Filesystem layout for the three memory layers.
+ *
+ *   <home>/core.md          core memory, injected in full every turn
+ *   <home>/stream.md        one line per completed turn
+ *   <home>/ws/<area>.md     detailed notes, retrieved only
+ *   <home>/.index/          derived caches (safe to delete)
+ *
+ * The directory is meant to be versioned as a whole: copy it, commit it, or
+ * point two harness installs at the same path on different operating systems.
+ * Nothing here stores an absolute path.
+ */
+
+import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { homedir, platform } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+
+export const CORE_FILE = 'core.md'
+export const STREAM_FILE = 'stream.md'
+export const WS_DIR = 'ws'
+export const INDEX_DIR = '.index'
+
+const DEFAULT_HOME = platform() === 'win32'
+  ? join(homedir(), 'AppData', 'Local', 'dsh-memory-dba')
+  : join(homedir(), '.dsh-memory-dba')
+
+/** NUL and control bytes make a layer unreadable; keep tabs/newlines and all emoji. */
+const BAD_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g
+
+export function resolveHome(home) {
+  const chosen = home || process.env.DSH_MEMORY_HOME || DEFAULT_HOME
+  return resolve(chosen.replace(/^~(?=$|[\\/])/, homedir()))
+}
+
+/**
+ * Create the layer files on first run. Existing files are never touched, so
+ * initializing an installed package cannot overwrite a real memory directory.
+ *
+ * @param {string} home Absolute memory directory.
+ * @returns {{created: string[], existed: string[]}} What the call changed.
+ */
+export function ensureLayout(home) {
+  const created = []
+  const existed = []
+  mkdirSync(home, { recursive: true })
+  mkdirSync(join(home, WS_DIR), { recursive: true })
+  mkdirSync(join(home, INDEX_DIR), { recursive: true })
+
+  const seeds = [
+    [CORE_FILE, '# Core memory\n\n'],
+    [STREAM_FILE, '# Memory stream\n\n'],
+  ]
+  for (const [name, body] of seeds) {
+    const file = join(home, name)
+    if (existsSync(file)) {
+      existed.push(name)
+      continue
+    }
+    writeFileSync(file, body, 'utf8')
+    created.push(name)
+  }
+  return { created, existed }
+}
+
+/** UTF-8 byte length, the unit every budget in this package uses. */
+export function bytes(text) {
+  return Buffer.byteLength(text, 'utf8')
+}
+
+export class MissingLayer extends Error {
+  constructor(file) {
+    super(`memory layer missing: ${file}`)
+    this.name = 'MissingLayer'
+    this.file = file
+  }
+}
+
+/**
+ * Read one layer as text. Failure is reported as a distinct error type rather
+ * than an empty string, so callers can tell "no memory yet" from "read broke".
+ *
+ * @param {string} home Absolute memory directory.
+ * @param {string} name Layer file name relative to `home`.
+ * @returns {string} File contents without a BOM.
+ */
+export function readLayer(home, name) {
+  const file = join(home, name)
+  let raw
+  try {
+    raw = new TextDecoder('utf-8', { fatal: true }).decode(readFileSync(file))
+  } catch (error) {
+    if (error?.code === 'ENOENT') throw new MissingLayer(file)
+    throw error
+  }
+  return raw.replace(/^\uFEFF/, '')
+}
+
+/** Atomic-ish replacement: write a sibling temp file, then rename. */
+export function writeLayer(home, name, text) {
+  const file = join(home, name)
+  mkdirSync(dirname(file), { recursive: true })
+  const tmp = `${file}.tmp-${process.pid}`
+  writeFileSync(tmp, text.replace(BAD_CHARS, ''), 'utf8')
+  renameSync(tmp, file)
+}
+
+/** Append a single line; creates the file when absent. */
+export function appendLayer(home, name, line) {
+  const file = join(home, name)
+  mkdirSync(dirname(file), { recursive: true })
+  let needsNewline = false
+  if (existsSync(file)) {
+    const fd = openSync(file, 'r')
+    try {
+      const { size } = fstatSync(fd)
+      if (size > 0) {
+        const last = Buffer.alloc(1)
+        readSync(fd, last, 0, 1, size - 1)
+        needsNewline = last[0] !== 0x0a
+      }
+    } finally {
+      closeSync(fd)
+    }
+  }
+  const clean = line.replace(BAD_CHARS, '')
+  writeFileSync(file, `${needsNewline ? '\n' : ''}${clean}\n`, { encoding: 'utf8', flag: 'a' })
+}
+
+/**
+ * Cut text to a byte budget without splitting a UTF-8 sequence or a lone
+ * surrogate, and mark the truncation so a reader never mistakes a prefix for
+ * the whole layer.
+ *
+ * @param {string} text Source text.
+ * @param {number} limitBytes Maximum UTF-8 bytes.
+ * @param {string} [marker] Suffix appended when text was cut.
+ */
+export function clipBytes(text, limitBytes, marker = '\n\n[...clipped to fit the memory budget]') {
+  limitBytes = Number.isFinite(limitBytes) ? Math.max(0, Math.floor(limitBytes)) : 0
+  if (bytes(text) <= limitBytes) return { text, clipped: false }
+  if (bytes(marker) > limitBytes) marker = clipBytes(marker, limitBytes, '').text
+  const reserve = bytes(marker)
+  const budget = Math.max(0, limitBytes - reserve)
+  let used = 0
+  let out = ''
+  for (const ch of text) {
+    const size = bytes(ch)
+    if (used + size > budget) break
+    out += ch
+    used += size
+  }
+  return { text: out + marker, clipped: true }
+}
+
+/** Snapshot used to invalidate derived indexes without watching files. */
+export function fingerprint(home) {
+  const parts = []
+  let names = []
+  try {
+    names = readdirSync(join(home, WS_DIR)).filter(name => name.endsWith('.md')).sort()
+  } catch { /* An unavailable workspace has no cacheable entries. */ }
+  for (const name of [CORE_FILE, STREAM_FILE, ...names.map(name => `${WS_DIR}/${name}`)]) {
+    try {
+      const st = statSync(join(home, name))
+      parts.push(JSON.stringify([name, st.size, st.mtimeMs]))
+    } catch {
+      parts.push(`${name}:-`)
+    }
+  }
+  return parts.join('|')
+}