@heixui/dsh-session-suite 0.0.0-stage → 0.3.0

package/lib/memory.js

@@ -0,0 +1,224 @@
+/**
+ * Session memory ledger.
+ *
+ * DSH's durable log is the only source of truth for the conversation, so the
+ * suite keeps *derived* memory here: short, human-readable facts a session (or
+ * its agent) decided to remember. Because every entry records the session
+ * sequence it was observed at, a rollback can trim the ledger back to exactly
+ * the state the conversation had at that point 鈥?the piece the product design
+ * called out as the real differentiator.
+ */
+
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { ensureDir, readJson, sha256Text, shortId, writeJson } from './util.js'
+
+
+export class MemoryLedger {
+  /**
+   * @param {object} options ledger options
+   * @param {string} options.root suite data directory
+   */
+  constructor({ root }) {
+    this.root = path.join(root, 'memory')
+  }
+
+  /** @param {string} scope session id or `_global` */
+  #file(scope) {
+    return path.join(this.root, `${scope}.json`)
+  }
+
+  /**
+   * @param {string} scope session id or `_global`
+   * @returns {Promise<{ version: number, scope: string, entries: any[] }>} ledger
+   */
+  async read(scope) {
+    const data = await readJson(this.#file(scope))
+    if (!data || !Array.isArray(data.entries)) return { version: 1, scope, entries: [] }
+    return data
+  }
+
+  /** @param {string} scope session id or `_global` @param {any} data ledger */
+  async write(scope, data) {
+    await ensureDir(this.root)
+    await writeJson(this.#file(scope), data)
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @param {{ text: string, tags?: string[], seq?: number, turn?: number, source?: string }} input entry
+   * @returns {Promise<any>} stored entry
+   */
+  async add(sessionId, input) {
+    const text = String(input.text ?? '').trim()
+    if (!text) throw new Error('memory text is empty')
+    const data = await this.read(sessionId)
+    const entry = {
+      id: `mem_${shortId(text)}`,
+      text,
+      tags: Array.isArray(input.tags) ? input.tags.filter(tag => typeof tag === 'string') : [],
+      seq: typeof input.seq === 'number' ? input.seq : null,
+      turn: typeof input.turn === 'number' ? input.turn : null,
+      source: input.source ?? 'manual',
+      pinned: false,
+      createdAt: Date.now(),
+      digest: sha256Text(text).slice(0, 16),
+    }
+    data.entries.push(entry)
+    await this.write(sessionId, data)
+    return entry
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @param {string} id entry id
+   * @param {{ text?: string, tags?: string[], pinned?: boolean }} patch changes
+   * @returns {Promise<any | undefined>} updated entry
+   */
+  async update(sessionId, id, patch) {
+    const data = await this.read(sessionId)
+    const entry = data.entries.find(item => item.id === id)
+    if (!entry) return undefined
+    if (typeof patch.text === 'string' && patch.text.trim()) {
+      entry.text = patch.text.trim()
+      entry.digest = sha256Text(entry.text).slice(0, 16)
+      entry.editedAt = Date.now()
+    }
+    if (Array.isArray(patch.tags)) entry.tags = patch.tags.filter(tag => typeof tag === 'string')
+    if (typeof patch.pinned === 'boolean') entry.pinned = patch.pinned
+    await this.write(sessionId, data)
+    return entry
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @param {string} id entry id
+   * @returns {Promise<boolean>} removed
+   */
+  async remove(sessionId, id) {
+    const data = await this.read(sessionId)
+    const before = data.entries.length
+    data.entries = data.entries.filter(item => item.id !== id)
+    await this.write(sessionId, data)
+    return data.entries.length !== before
+  }
+
+  /**
+   * Trim entries observed after a session sequence 鈥?the memory half of a
+   * checkpoint rollback.
+   *
+   * @param {string} sessionId session
+   * @param {number} seq inclusive bound to keep
+   * @returns {Promise<{ removed: number, kept: number }>} outcome
+   */
+  async trimTo(sessionId, seq) {
+    const data = await this.read(sessionId)
+    const kept = []
+    let removed = 0
+    for (const entry of data.entries) {
+      if (typeof entry.seq === 'number' && entry.seq > seq && !entry.pinned) removed += 1
+      else kept.push(entry)
+    }
+    if (removed > 0) {
+      data.entries = kept
+      await this.write(sessionId, data)
+    }
+    return { removed, kept: kept.length }
+  }
+
+  /**
+   * Copy a ledger into another session, optionally only the entries observed up
+   * to a sequence. Used when a rewind or a segment delete forks a child: the
+   * conversation continues in a new session, so its memory has to follow.
+   *
+   * @param {string} fromId source session
+   * @param {string} toId destination session
+   * @param {number} [maxSeq] inclusive upper bound on `seq`
+   * @returns {Promise<{ copied: number }>} outcome
+   */
+  async clone(fromId, toId, maxSeq) {
+    if (fromId === toId) return { copied: 0 }
+    const data = await this.read(fromId)
+    const entries = data.entries
+      .filter(entry => typeof maxSeq !== 'number' || typeof entry.seq !== 'number' || entry.seq <= maxSeq)
+      .map(entry => ({ ...entry }))
+    await this.write(toId, { version: 1, scope: toId, entries })
+    return { copied: entries.length }
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @returns {Promise<any[]>} entries in insertion order
+   */
+  async list(sessionId) {
+    const data = await this.read(sessionId)
+    return data.entries
+  }
+
+  /**
+   * Everything this session contributed to the global long-term store.
+   *
+   * @param {string} sessionId session
+   * @param {{ text: string }} input entry
+   * @returns {Promise<any>} stored global entry
+   */
+  async addGlobal(sessionId, input) {
+    const text = String(input.text ?? '').trim()
+    if (!text) throw new Error('memory text is empty')
+    const data = await this.read('_global')
+    const entry = {
+      id: `gmem_${shortId(text)}`,
+      text,
+      sessionId,
+      tags: [],
+      pinned: false,
+      createdAt: Date.now(),
+    }
+    data.entries.push(entry)
+    await this.write('_global', data)
+    return entry
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @returns {Promise<any[]>} global entries that mention this session
+   */
+  async listGlobal(sessionId) {
+    const data = await this.read('_global')
+    return data.entries.filter(entry => entry.sessionId === sessionId)
+  }
+
+  /**
+   * @param {string} sessionId session
+   */
+  async purge(sessionId) {
+    await fs.rm(this.#file(sessionId), { force: true }).catch(() => {})
+    const data = await this.read('_global')
+    const kept = data.entries.filter(entry => entry.sessionId !== sessionId)
+    if (kept.length !== data.entries.length) {
+      data.entries = kept
+      await this.write('_global', data)
+    }
+  }
+
+  /**
+   * Whole-ledger snapshot used by the undo bundle.
+   *
+   * @param {string} sessionId session
+   * @returns {Promise<any>} ledger content
+   */
+  async snapshot(sessionId) {
+    return this.read(sessionId)
+  }
+
+  /**
+   * @param {string} sessionId session
+   * @param {any} data ledger content captured earlier
+   */
+  async restore(sessionId, data) {
+    if (!data || !Array.isArray(data.entries)) return
+    await this.write(sessionId, data)
+  }
+}
+
+export const LEDGER_GLOBAL = '_global'

package/lib/retrieval.js

@@ -0,0 +1,217 @@
+/**
+ * Retrieval primitives, deliberately free of I/O and of the memory engine.
+ *
+ * The engine used to rank with a single lexical signal: bigram coverage weighted
+ * by inverse document frequency. That works for precision but has one error
+ * profile, and a single route cannot outvote itself. Chapter 3 of *深入理解
+ * AI Agent* describes the standard remedy as hybrid retrieval: run several
+ * routes with *different* blind spots, then fuse them by rank rather than by
+ * score, because the scores live on incomparable scales (cosine 0..1 versus
+ * BM25 0..50).
+ *
+ * The book's example fuses a dense route with a sparse one. We have no embedding
+ * model on the recall path — recall runs offline and must stay offline — so this
+ * module fuses two *lexical* routes chosen to fail differently:
+ *
+ *   - bigrams (two-character sequences): precise, but blind to any paraphrase
+ *     that does not reproduce the exact pair, and blind to single characters.
+ *   - unigrams (single characters plus ASCII words): loose, noisy, but it still
+ *     finds a memory when only part of the query vocabulary overlaps.
+ *
+ * Fusing them is not a substitute for a dense route. It recovers ranking order
+ * among memories that are already reachable, and it cannot invent a match where
+ * no character is shared at all. That limit is documented rather than papered
+ * over.
+ */
+
+/**
+ * Tokenise into the bigram-plus-word vocabulary used for precise matching.
+ *
+ * Returns an array *with multiplicity*, because BM25 needs term frequency. The
+ * set form is the same vocabulary deduplicated, so `new Set(tokens(text))` is
+ * exactly the shingle set the rest of the engine already uses.
+ *
+ * @param {string} text input
+ * @returns {string[]} tokens, duplicates preserved
+ */
+export function tokens(text) {
+  const normalized = String(text ?? '').toLowerCase().replace(/[\s\p{P}\p{S}]+/gu, ' ')
+  const out = []
+  for (const word of normalized.split(' ')) {
+    if (!word) continue
+    if (/^[\x00-\x7f]+$/.test(word)) {
+      out.push(word)
+      continue
+    }
+    const chars = [...word]
+    if (chars.length === 1) out.push(chars[0])
+    for (let index = 0; index + 1 < chars.length; index += 1) out.push(`${chars[index]}${chars[index + 1]}`)
+  }
+  return out
+}
+
+/**
+ * Tokenise into the loose vocabulary: every single character, plus ASCII words.
+ *
+ * A CJK character carries far less meaning than a bigram, which is exactly the
+ * point — this route tolerates a query that shares only part of a word, which
+ * the bigram route cannot.
+ *
+ * @param {string} text input
+ * @returns {string[]} tokens, duplicates preserved
+ */
+export function unigrams(text) {
+  const normalized = String(text ?? '').toLowerCase().replace(/[\s\p{P}\p{S}]+/gu, ' ')
+  const out = []
+  for (const word of normalized.split(' ')) {
+    if (!word) continue
+    if (/^[\x00-\x7f]+$/.test(word)) {
+      out.push(word)
+      continue
+    }
+    for (const char of word) out.push(char)
+  }
+  return out
+}
+
+/** @param {string[]} list @returns {Map<string, number>} term frequencies */
+function termFrequency(list) {
+  const out = new Map()
+  for (const token of list) out.set(token, (out.get(token) ?? 0) + 1)
+  return out
+}
+
+/**
+ * Inverse document frequency, always positive.
+ *
+ * The textbook form `ln((N - df + 0.5) / (df + 0.5))` goes negative once a term
+ * appears in more than half the documents, which would make a common word
+ * *subtract* from relevance. Chapter 3 notes the same problem and recommends a
+ * floor; wrapping the ratio in `ln(1 + ...)` is the floor, and it is what
+ * production search engines actually ship.
+ *
+ * @param {number} total document count
+ * @param {number} seen document frequency
+ * @returns {number} idf, > 0
+ */
+function idf(total, seen) {
+  return Math.log(1 + (total - seen + 0.5) / (seen + 0.5))
+}
+
+/**
+ * BM25 relevance of every document to one query.
+ *
+ * BM25 is the correction the book describes for TF-IDF's two weaknesses: raw
+ * term frequency grows without bound, and long documents win merely by being
+ * long. `k1` makes repeated occurrences saturate; `b` normalises by length.
+ *
+ * @param {string[]} queryTokens query, with multiplicity
+ * @param {string[][]} documentTokens one token list per document
+ * @param {object} [options] tuning
+ * @param {number} [options.k1] term-frequency saturation, default 1.5
+ * @param {number} [options.b] length normalisation strength, default 0.75
+ * @returns {number[]} one score per document, in input order
+ */
+export function bm25(queryTokens, documentTokens, options = {}) {
+  const k1 = options.k1 ?? 1.5
+  const b = options.b ?? 0.75
+  const total = documentTokens.length
+  if (total === 0) return []
+
+  const frequencies = documentTokens.map(termFrequency)
+  const lengths = documentTokens.map(list => list.length)
+  const averageLength = lengths.reduce((sum, value) => sum + value, 0) / total || 1
+
+  const seen = new Map()
+  for (const frequency of frequencies) {
+    for (const token of frequency.keys()) seen.set(token, (seen.get(token) ?? 0) + 1)
+  }
+
+  const queryFrequency = termFrequency(queryTokens)
+  const scores = new Array(total).fill(0)
+
+  for (let index = 0; index < total; index += 1) {
+    const length = lengths[index]
+    let score = 0
+    for (const [token, queryCount] of queryFrequency) {
+      const frequency = frequencies[index].get(token) ?? 0
+      if (frequency === 0) continue
+      const weight = idf(total, seen.get(token) ?? 0)
+      const saturation = (frequency * (k1 + 1)) / (frequency + k1 * (1 - b + (b * length) / averageLength))
+      score += weight * saturation * Math.min(queryCount, 3)
+    }
+    scores[index] = score
+  }
+  return scores
+}
+
+/**
+ * IDF-weighted fraction of the query that a document accounts for.
+ *
+ * Coverage, not Jaccard: a memory may be far longer than the query, and being
+ * long must not be a penalty when the question is "did it contain what I asked".
+ * This is the loose route's scoring function; the precise route uses BM25.
+ *
+ * @param {string[]} queryTokens query, with multiplicity
+ * @param {string[]} documentTokens document, with multiplicity
+ * @param {string[][]} allDocumentTokens corpus, for document frequency
+ * @returns {number} coverage 0..1
+ */
+export function weightedCoverage(queryTokens, documentTokens, allDocumentTokens) {
+  const querySet = new Set(queryTokens)
+  if (querySet.size === 0) return 0
+  const documentSet = new Set(documentTokens)
+  const total = allDocumentTokens.length || 1
+  const seen = new Map()
+  for (const list of allDocumentTokens) {
+    for (const token of new Set(list)) seen.set(token, (seen.get(token) ?? 0) + 1)
+  }
+  let matched = 0
+  let available = 0
+  for (const token of querySet) {
+    const weight = idf(total, seen.get(token) ?? 0)
+    available += weight
+    if (documentSet.has(token)) matched += weight
+  }
+  return available > 0 ? matched / available : 0
+}
+
+/**
+ * Reciprocal Rank Fusion.
+ *
+ * Fusing by rank rather than by score is the whole point: BM25 returns values
+ * spanning tens, coverage returns 0..1, and adding them would let whichever
+ * route has the larger scale decide the outcome. RRF ignores the magnitudes and
+ * keeps only the ordering, which is the part both routes agree is meaningful.
+ *
+ * @param {Array<Array<string | number>>} rankings each route's ids, best first
+ * @param {number} [k] smoothing constant; 60 is the customary default
+ * @returns {Map<string | number, number>} fused score, normalised to 0..1
+ */
+export function reciprocalRankFusion(rankings, k = 60) {
+  const fused = new Map()
+  const routes = rankings.filter(ranking => ranking.length > 0)
+  for (const ranking of routes) {
+    ranking.forEach((id, index) => {
+      fused.set(id, (fused.get(id) ?? 0) + 1 / (k + index + 1))
+    })
+  }
+  // Normalise so a document ranked first by every route scores 1, keeping the
+  // fused signal comparable to the 0..1 importance score it is combined with.
+  if (routes.length > 0) {
+    const ceiling = routes.length / (k + 1)
+    for (const [id, score] of fused) fused.set(id, score / ceiling)
+  }
+  return fused
+}
+
+/**
+ * Order ids by a score map, best first, with a stable tiebreak.
+ *
+ * @param {Array<string | number>} ids candidates
+ * @param {Map<string | number, number>} scores score per id
+ * @returns {Array<string | number>} sorted ids
+ */
+export function rankIds(ids, scores) {
+  return [...ids].sort((a, b) => (scores.get(b) ?? 0) - (scores.get(a) ?? 0) || String(a).localeCompare(String(b)))
+}