@ilaomianyang/dsh-memory-graph 0.10.4 → 0.12.0

package/src/auto.js

@@ -32,14 +32,30 @@ export function inQuietHours(window, now) {
  * Should this tick run a distillation? Pure, so the rules are readable in one
  * place and testable without a clock.
  *
+ * Three rules, in the order they are asked:
+ *
+ * 1. **catch-up** — a pile bigger than one batch is a queue, not a schedule. It
+ *    runs every `catchUpEveryMinutes` until the pile is smaller than a batch, and
+ *    stops for the day at `maxBatchesPerDay`. This is the rule that lets an
+ *    imported history drain itself overnight without a person pressing anything,
+ *    and the daily cap is what keeps it from spending without a ceiling.
+ * 2. **schedule** — `everyMinutes` since the last run (or since the clock
+ *    started).
+ * 3. **threshold** — `minEpisodes` waiting.
+ *
+ * The gap (`minGapMinutes`) sits between 1 and 2/3: catch-up measures its own
+ * cadence, while the other two answers still respect the gap a person configured.
+ *
  * @param {object} input
  * @param {object} input.auto        the `auto` config group
  * @param {number} input.now         epoch ms
  * @param {number|null} input.lastAttemptAt  epoch ms of the previous attempt
  * @param {number} input.pending     undistilled episodes waiting
+ * @param {number} [input.batchSize] episodes one run distils
+ * @param {number} [input.batchesToday] automatic batches already run today
  * @returns {{due: boolean, rule: string|null, reason: string}}
  */
-export function autoDecision({ auto, now, lastAttemptAt = null, startedAt = null, pending = 0 }) {
+export function autoDecision({ auto, now, lastAttemptAt = null, startedAt = null, pending = 0, batchSize = 40, batchesToday = 0 }) {
   if (auto?.enabled !== true) return { due: false, rule: null, reason: 'auto is off' }
   if (pending <= 0) return { due: false, rule: null, reason: 'nothing to distill' }
   if (inQuietHours(auto.quietHours, now)) return { due: false, rule: null, reason: 'quiet hours' }
@@ -55,6 +71,24 @@ export function autoDecision({ auto, now, lastAttemptAt = null, startedAt = null
   const clock = lastAttemptAt ?? startedAt
   const sinceClock = clock === null ? null : now - clock
 
+  // Catch-up first: its cadence is its own, and it is the only rule that can be
+  // due while the gap configured for the others has not elapsed.
+  if (auto.catchUp === true && pending >= Math.max(1, batchSize)) {
+    const cap = auto.maxBatchesPerDay ?? 0
+    if (cap > 0 && batchesToday >= cap) {
+      return { due: false, rule: null, reason: `今日已跑 ${batchesToday} 批，达到上限 ${cap} · daily batch cap reached` }
+    }
+    const every = Math.max(1, auto.catchUpEveryMinutes ?? 10) * MINUTE
+    if (sinceRun === null || sinceRun >= every) {
+      return {
+        due: true,
+        rule: 'catch-up',
+        reason: `还有 ${pending} 条待蒸馏，今天第 ${batchesToday + 1} 批${cap > 0 ? ` / 上限 ${cap}` : ''}`,
+      }
+    }
+    return { due: false, rule: null, reason: `距上一批不到 ${auto.catchUpEveryMinutes ?? 10} 分钟 · catch-up gap` }
+  }
+
   const gap = (auto.minGapMinutes ?? 0) * MINUTE
   if (gap > 0 && sinceRun !== null && sinceRun < gap) {
     return { due: false, rule: null, reason: 'the gap between runs has not elapsed' }
@@ -85,6 +119,8 @@ export function createAutoDistiller({ cfg, vault, run, log = () => {}, now = ()
   let lastAttemptAt = null
   let startedAt = null
   let lastReport = null
+  let day = null
+  let batchesToday = 0
 
   async function loadState() {
     if (loaded) return
@@ -95,23 +131,37 @@ export function createAutoDistiller({ cfg, vault, run, log = () => {}, now = ()
       const start = Number(state?.startedAt)
       lastAttemptAt = Number.isFinite(attempt) && attempt > 0 ? attempt : null
       startedAt = Number.isFinite(start) && start > 0 ? start : null
+      day = typeof state?.batchDay === 'string' ? state.batchDay : null
+      const count = Number(state?.batchesToday)
+      batchesToday = Number.isFinite(count) && count > 0 ? count : 0
     } catch (error) {
-      log('warn', `auto: could not read .auto-state.json: ${error?.message ?? error}`)
+      log('warn', `自动蒸馏：读不了 .auto-state.json · auto: could not read .auto-state.json: ${error?.message ?? error}`)
     }
   }
 
+  /** Today's local date, the key the daily batch cap resets on. */
+  function todayKey(at) {
+    const date = new Date(at)
+    const pad = (value) => String(Math.abs(value)).padStart(2, '0')
+    return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`
+  }
+
   /** One decision, and at most one run. Returns what it decided, for the tests. */
   async function tick() {
     const auto = cfg.auto ?? {}
     await loadState()
     if (auto.enabled !== true) return { due: false, rule: null, reason: 'auto is off', ran: false }
 
+    const batchSize = Math.max(1, cfg.extractor?.maxEpisodesPerRun ?? 40)
     let pending = 0
     try {
-      const batch = await vault.readUndistilled(Math.max(1, auto.minEpisodes ?? 0, cfg.extractor?.maxEpisodesPerRun ?? 40))
+      // Ask for one batch more than a batch: the decision needs to know whether a
+      // *full* batch is waiting (catch-up) and whether the threshold is met, and
+      // one extra row answers both without a second read.
+      const batch = await vault.readUndistilled(Math.max(1, auto.minEpisodes ?? 0, batchSize))
       pending = Array.isArray(batch) ? batch.length : 0
     } catch (error) {
-      log('warn', `auto: could not count undistilled episodes: ${error?.message ?? error}`)
+      log('warn', `自动蒸馏：数不了还没蒸馏的情节 · auto: could not count undistilled episodes: ${error?.message ?? error}`)
       return { due: false, rule: null, reason: 'the vault could not be read', ran: false }
     }
 
@@ -126,41 +176,55 @@ export function createAutoDistiller({ cfg, vault, run, log = () => {}, now = ()
       try {
         await vault.writeAutoState?.({ startedAt: at })
       } catch (error) {
-        log('warn', `auto: could not start the schedule clock: ${error?.message ?? error}`)
+        log('warn', `自动蒸馏：没能启动计划时钟 · auto: could not start the schedule clock: ${error?.message ?? error}`)
       }
     }
 
-    const decision = autoDecision({ auto, now: at, lastAttemptAt, startedAt, pending })
-    if (!decision.due) return { ...decision, pending, ran: false }
+    // The daily cap is per calendar day, and a new day resets it — in memory now
+    // and on disk with the next write, so a restart cannot lose the count either.
+    const today = todayKey(at)
+    if (day !== today) {
+      day = today
+      batchesToday = 0
+    }
+
+    const decision = autoDecision({ auto, now: at, lastAttemptAt, startedAt, pending, batchSize, batchesToday })
+    if (!decision.due) return { ...decision, pending, batchesToday, ran: false }
 
     // Arm the gap before the call: a slow or failing run must not be retried on
     // the next tick.
     lastAttemptAt = at
+    // Counted before the call, like the gap: a run that fails or finds another one
+    // in flight still spent this batch's slot, so the daily cap stays a real cap.
+    if (decision.rule === 'catch-up') batchesToday += 1
     try {
-      await vault.writeAutoState?.({ lastAttemptAt: at })
+      await vault.writeAutoState?.(decision.rule === 'catch-up' ? { lastAttemptAt: at, batchDay: today, batchesToday } : { lastAttemptAt: at })
     } catch (error) {
-      log('warn', `auto: could not record the attempt: ${error?.message ?? error}`)
+      log('warn', `自动蒸馏：没能记下这次尝试 · auto: could not record the attempt: ${error?.message ?? error}`)
     }
 
     const report = await run({ project: auto.project ? String(auto.project) : undefined })
     lastReport = report
     if (report?.busy === true) {
-      log('info', 'auto: skipped, a distillation is already running')
+      log('info', '自动蒸馏：跳过，已经有一次蒸馏在跑 · auto: skipped, a distillation is already running')
     } else if (report?.ok === true) {
       log(
         'info',
-        `auto (${decision.rule}): read ${report.episodes} episode(s) → wrote ${report.written} card(s)` +
+        `自动蒸馏 auto (${decision.rule}): read ${report.episodes} episode(s) → wrote ${report.written} card(s)` +
           (report.model ? ` · ${report.model}` : ''),
       )
     } else {
-      log('warn', `auto (${decision.rule}): ${report?.message ?? report?.reason ?? 'did not run'}`)
+      log(
+        'warn',
+        `自动蒸馏 auto (${decision.rule}): ${report?.message ?? report?.reason ?? '没有运行 · did not run'}`,
+      )
     }
-    return { ...decision, pending, ran: true, report }
+    return { ...decision, pending, batchesToday, ran: true, report }
   }
 
   /** What the ticker knows, for a diagnostics line or a test. */
   function state() {
-    return { lastAttemptAt, lastReport }
+    return { lastAttemptAt, lastReport, batchesToday, day }
   }
 
   return { tick, state }

package/src/capture.js

@@ -4,117 +4,24 @@
  * Deliberately model-free. Every turn is appended as it closes, so the "never
  * loses anything" guarantee holds even while the extractor is unconfigured,
  * offline, or out of quota. Distillation happens later and separately.
+ *
+ * What counts as a real message, how it is truncated and how it is redacted
+ * lives in `turns.js`, shared with the history importer — see the note there for
+ * why there is exactly one copy of that logic.
  */
-import { nowIso } from './vault.js'
+import { createTurnAssembler, projectOf } from './turns.js'
 
 const FLUSH_AFTER_MS = 90_000
-const MAX_ASSISTANT_PARTS = 6
-
-function collectText(node, depth = 0, seen = new Set(), out = []) {
-  if (node === null || node === undefined || depth > 4) return out
-  if (typeof node === 'string') return out
-  if (Array.isArray(node)) {
-    for (const item of node) collectText(item, depth + 1, seen, out)
-    return out
-  }
-  if (typeof node !== 'object') return out
-  if (seen.has(node)) return out
-  seen.add(node)
-
-  if (node.type === 'text' && typeof node.text === 'string') {
-    out.push(node.text)
-    return out
-  }
-  for (const key of ['content', 'message', 'text', 'parts', 'blocks']) {
-    if (key in node) collectText(node[key], depth + 1, seen, out)
-  }
-  return out
-}
-
-function eventText(event) {
-  const parts = collectText(event?.data)
-  return parts.join('\n').replace(/\n{3,}/g, '\n\n').trim()
-}
-
-function projectOf(session, fallback) {
-  const cwd = session?.header?.cwd ?? session?.meta?.cwd ?? session?.cwd
-  if (typeof cwd === 'string' && cwd.trim() !== '') {
-    // Both separators, whatever platform this is running on: the path comes from
-    // the session header, which may well have been written on the other OS.
-    const parts = cwd.replace(/[\\/]+$/, '').split(/[\\/]/)
-    const base = parts[parts.length - 1] ?? ''
-    if (base !== '') return base
-  }
-  return fallback
-}
-
-function truncate(text, max) {
-  const value = String(text ?? '').trim()
-  if (value.length <= max) return value
-  return `${value.slice(0, max)}…`
-}
 
 export function registerCapture(ctx, { cfg, vault, redactor }) {
-  const buffers = new Map()
+  const assembler = createTurnAssembler({ cfg, redactor })
   const disposers = []
   const timers = new Map()
 
-  // Both the switch and the skip list are editable while the plugin runs (they
-  // are `.volatile()` in the schema), so neither is baked in here: the switch is
-  // read per event, and the compiled rules are rebuilt only when the list's text
-  // actually changes — a `new RegExp` on every token would be pure waste.
-  let skipKey
-  let skipRules = []
-
-  function compiledSkipRules() {
-    const list = cfg.capture.skipPatterns ?? []
-    const key = list.join('\u0000')
-    if (key !== skipKey) {
-      skipKey = key
-      skipRules = list.flatMap((pattern) => {
-        try {
-          return [new RegExp(pattern, 'i')]
-        } catch {
-          return []
-        }
-      })
-    }
-    return skipRules
-  }
-
-  /** True when the "user" side is really a harness injection, not a person typing. */
-  const injected = (text) => text !== '' && compiledSkipRules().some((re) => re.test(text))
-
-  /**
-   * Delimited harness blocks come off before the patterns are consulted.
-   *
-   * They are the one kind of injection whose extent is knowable, so removing them
-   * keeps the person's own words when a real message happens to carry one — the
-   * alternative (dropping the whole message) loses the part that was worth
-   * keeping. Undelimited blocks (a runtime snapshot) are handled by the patterns.
-   */
-  function stripInjectedBlocks(text) {
-    return String(text ?? '')
-      .replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, '')
-      .replace(/[ \t]+\n/g, '\n')
-      .replace(/\n{3,}/g, '\n\n')
-      .trim()
-  }
-
   const idOf = (session) => String(session?.id ?? session?.header?.id ?? 'unknown')
 
-  function bufferOf(session) {
-    const id = idOf(session)
-    let buffer = buffers.get(id)
-    if (!buffer) {
-      buffer = { project: projectOf(session, cfg.defaultProject), user: '', assistant: [], tools: 0, turn: null }
-      buffers.set(id, buffer)
-    }
-    return { id, buffer }
-  }
-
   function drop(id) {
-    buffers.delete(id)
+    assembler.forget(id)
     const timer = timers.get(id)
     if (timer) {
       clearTimeout(timer)
@@ -123,33 +30,9 @@ export function registerCapture(ctx, { cfg, vault, redactor }) {
   }
 
   async function flush(id) {
-    const buffer = buffers.get(id)
-    if (!buffer) return
-    const toolCount = buffer.tools
-    const user = truncate(buffer.user, cfg.capture.userChars)
-    const assistant = truncate(buffer.assistant.join('\n'), cfg.capture.assistantChars)
-
-    // Every episode is anchored to something the user actually said. A turn with
-    // no user side is context noise, and one too short to mean anything — such as
-    // "ok" — would only fill the vault with lines the extractor cannot use.
-    if (user === '' || user.length + assistant.length < cfg.capture.minChars) {
-      drop(id)
-      return
-    }
-
+    const episode = assembler.flush(id)
     drop(id)
-
-    const redactedUser = redactor.redact(user).text
-    const redactedAssistant = redactor.redact(assistant).text
-    await vault.appendEpisode({
-      t: nowIso(),
-      project: buffer.project,
-      session: id,
-      turn: buffer.turn,
-      user: redactedUser,
-      assistant: redactedAssistant,
-      ...(cfg.capture.includeToolOutput && toolCount > 0 ? { tools: toolCount } : {}),
-    })
+    if (episode) await vault.appendEpisode(episode)
   }
 
   function schedule(id) {
@@ -164,65 +47,42 @@ export function registerCapture(ctx, { cfg, vault, redactor }) {
 
   disposers.push(
     ctx.on('session/event', (session, event) => {
-      // The switch is read here, not at registration: turning capture off in
-      // 设置 must stop writes immediately without unregistering the listener.
+      // The switch is read inside the assembler, per event: turning capture off
+      // in 设置 must stop writes immediately without unregistering the listener.
       if (!cfg.capture.enabled) return
 
-      const type = event?.type
+      const id = idOf(session)
+      const episodes = assembler.push({
+        sessionId: id,
+        project: projectOf(session, cfg.defaultProject),
+        event,
+      })
 
-      // A turn is the natural write unit: flush it as soon as it closes, so an
-      // episode exists even if the next turn never comes.
-      if (type === 'turn/end') {
-        const id = idOf(session)
-        const buffer = buffers.get(id)
-        if (buffer) {
-          if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
-          flush(id).catch(() => {})
+      if (episodes.length > 0) {
+        for (const episode of episodes) {
+          vault.appendEpisode(episode).catch(() => {})
         }
-        return
       }
 
-      if (type !== 'user/message' && type !== 'assistant/message' && type !== 'tool/result') return
-
-      if (type === 'user/message') {
-        // The harness delivers runtime snapshots, skill catalogs, sampled time,
-        // approval notices and compaction checkpoints through the same channel as
-        // the person typing — and not always with a telling `source.kind`: a real
-        // runtime snapshot has arrived here tagged `user` (it was captured as if
-        // the person had typed it). So provenance can only *add* a skip, never
-        // veto the patterns; both are consulted, and delimited blocks come off
-        // first so a message that merely carried a reminder keeps its own words.
-        const kind = event?.data?.source?.kind
-        const text = stripInjectedBlocks(eventText(event))
-        const synthetic = (typeof kind === 'string' && kind !== 'user') || text === '' || injected(text)
-        if (synthetic) return
+      const type = event?.type
+      const relevant =
+        type === 'turn/end' || type === 'user/message' || type === 'assistant/message' || type === 'tool/result'
+      if (!relevant) return
 
-        const id = idOf(session)
-        const open = buffers.get(id)
-        if (open && (open.user !== '' || open.assistant.length > 0)) {
-          // A turn that never closed (interrupted, or an older harness): keep it.
-          flush(id).catch(() => {})
+      // A turn is the natural write unit: flush it as soon as it closes, so an
+      // episode exists even if the next turn never comes. The timer is the safety
+      // net for a turn that never closes at all.
+      if (type === 'turn/end') {
+        const timer = timers.get(id)
+        if (timer) {
+          clearTimeout(timer)
+          timers.delete(id)
         }
-
-        const { buffer } = bufferOf(session)
-        buffer.user = text
-        if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
-        schedule(id)
+        assembler.forget(id)
         return
       }
 
-      const { id, buffer } = bufferOf(session)
-
-      if (type === 'assistant/message') {
-        const text = eventText(event)
-        if (text !== '' && buffer.assistant.length < MAX_ASSISTANT_PARTS) buffer.assistant.push(text)
-        if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
-        schedule(id)
-        return
-      }
-
-      buffer.tools += 1
-      schedule(id)
+      if (assembler.isOpen(id)) schedule(id)
     }),
   )
 
@@ -231,7 +91,9 @@ export function registerCapture(ctx, { cfg, vault, redactor }) {
     () => {
       for (const timer of timers.values()) clearTimeout(timer)
       timers.clear()
-      for (const id of [...buffers.keys()]) flush(id).catch(() => {})
+      for (const episode of assembler.flushAll()) {
+        vault.appendEpisode(episode).catch(() => {})
+      }
     },
   ]
 }

package/src/config.js

@@ -114,6 +114,34 @@ const DEFAULTS = {
     quietHours: '',
     /** Project tag for automatic cards; empty uses the newest episode's project. */
     project: '',
+    /**
+     * Catch-up mode: instead of waiting for the schedule, keep running a batch
+     * every `catchUpEveryMinutes` while a full batch is still waiting — and stop
+     * at `maxBatchesPerDay`. This is what turns a backlog (an imported history,
+     * a week of sessions) into something that drains itself without a person
+     * pressing a button, while still capping what it can spend in one day.
+     */
+    catchUp: false,
+    /** Minutes between catch-up batches. */
+    catchUpEveryMinutes: 10,
+    /** Most automatic batches in one calendar day; 0 turns the daily cap off. */
+    maxBatchesPerDay: 6,
+  },
+
+  /**
+   * Update checking. The plugin asks the registry what the newest version is and
+   * compares it with the version it is running; it never installs unless asked
+   * (`autoInstall`) or a person presses the button. One HTTPS GET, no query
+   * string, nothing about the vault or the user in it.
+   */
+  update: {
+    enabled: true,
+    /** Hours a check result stays good for. */
+    checkEveryHours: 12,
+    /** Install a newer version at startup without being asked. */
+    autoInstall: false,
+    /** Registry to ask; empty uses the plugin manager's, then npm's own. */
+    registry: '',
   },
 }
 
@@ -241,6 +269,14 @@ export function resolveConfig(raw) {
   // A malformed window must not disable the rule or, worse, silence it forever:
   // parse what is parseable and fall back to "no quiet hours".
   cfg.auto.quietHours = parseQuietHours(cfg.auto.quietHours) === null ? '' : String(cfg.auto.quietHours).trim()
+  cfg.auto.catchUp = cfg.auto.catchUp === true
+  cfg.auto.catchUpEveryMinutes = int(cfg.auto.catchUpEveryMinutes, 10, 1, 1440)
+  cfg.auto.maxBatchesPerDay = int(cfg.auto.maxBatchesPerDay, 6, 0, 200)
+
+  cfg.update.enabled = cfg.update.enabled !== false
+  cfg.update.checkEveryHours = int(cfg.update.checkEveryHours, 12, 1, 24 * 30)
+  cfg.update.autoInstall = cfg.update.autoInstall === true
+  cfg.update.registry = String(cfg.update.registry || '').trim().replace(/\/+$/, '')
 
   return cfg
 }
@@ -276,5 +312,6 @@ export function paths(vaultDir) {
     audit: path.join(vaultDir, 'audit'),
     distillState: path.join(vaultDir, '.distill-state.json'),
     autoState: path.join(vaultDir, '.auto-state.json'),
+    updateState: path.join(vaultDir, '.update-state.json'),
   }
 }

package/src/distillapi.js

@@ -23,7 +23,10 @@ export function createDistillHandler({ getRunner }) {
   return async function handleDistill() {
     const runner = getRunner()
     if (typeof runner?.run !== 'function') {
-      return json({ ok: false, reason: 'unavailable', message: 'the distiller is not available' }, 500)
+      return json(
+        { ok: false, reason: 'unavailable', message: '蒸馏器不可用 · the distiller is not available' },
+        500,
+      )
     }
     return respond(await runner.run())
   }

package/src/distillrun.js

@@ -42,7 +42,11 @@ export function createDistillRunner(options) {
    */
   function run({ project, dryRun = false, signal } = {}) {
     if (inflight) {
-      return Promise.resolve({ ok: false, busy: true, reason: 'A distillation is already running.' })
+      return Promise.resolve({
+        ok: false,
+        busy: true,
+        reason: '已有一次蒸馏在跑，等它结束 · A distillation is already running.',
+      })
     }
 
     const task = (async () => {
@@ -66,7 +70,7 @@ export function createDistillRunner(options) {
           try {
             await refreshViews()
           } catch (error) {
-            log('warn', `distilled, but a vault view did not render: ${error?.message ?? error}`)
+            log('warn', `蒸馏完成了，但有一个记忆库视图没能渲染出来 · distilled, but a vault view did not render: ${error?.message ?? error}`)
           }
         }
         return {
@@ -85,7 +89,7 @@ export function createDistillRunner(options) {
           reason: report.reason ?? null,
         }
       } catch (error) {
-        log('warn', `distillation failed: ${error?.message ?? error}`)
+        log('warn', `蒸馏失败 · distillation failed: ${error?.message ?? error}`)
         return {
           ok: false,
           busy: false,