@yeaft/webchat-agent 0.1.628 → 0.1.630

package/unify/dream-v2/schedule.js

@@ -0,0 +1,73 @@
+/**
+ * dream-v2/schedule.js — DESIGN-v2 §10.2.
+ *
+ * Two trigger paths:
+ *
+ *   1. 12-hour interval timer.
+ *   2. Manual trigger (UI button or `/dream` command), routed in via
+ *      `triggerNow()` — sets `manual: true` so the per-group threshold
+ *      is bypassed.
+ *
+ * The scheduler is a thin wrapper around `runDream()` that prevents
+ * concurrent passes (a second tick while the previous is still running
+ * is dropped, not queued — DESIGN-v2 §10.1: "slow is OK, doesn't
+ * compete with user latency").
+ */
+
+import { DREAM_INTERVAL_HOURS } from './limits.js';
+
+export const DEFAULT_INTERVAL_MS = DREAM_INTERVAL_HOURS * 60 * 60 * 1000;
+
+/**
+ * Build a scheduler around `runDream()`. The runner closure captures
+ * everything `runDream` needs (memory root, llm, message-store hooks,
+ * onProgress sink); the scheduler only knows how to call it.
+ *
+ * @param {{
+ *   run: (opts: { manual: boolean, scopeFilter?: string[] }) => Promise<object>,
+ *   intervalMs?: number,
+ *   logger?: { info?: (...a:any) => void, warn?: (...a:any) => void, error?: (...a:any) => void },
+ * }} args
+ */
+export function createDreamScheduler({ run, intervalMs = DEFAULT_INTERVAL_MS, logger }) {
+  if (typeof run !== 'function') throw new Error('createDreamScheduler: run callable required');
+  const log = logger || {};
+  let timer = null;
+  let inflight = null;
+
+  async function fire(opts) {
+    if (inflight) {
+      log.warn?.('[dream] tick dropped — previous run still in progress');
+      return inflight;
+    }
+    inflight = (async () => {
+      try {
+        return await run(opts);
+      } catch (err) {
+        log.error?.('[dream] run failed:', err && err.message ? err.message : err);
+        return { error: err && err.message ? err.message : String(err) };
+      } finally {
+        inflight = null;
+      }
+    })();
+    return inflight;
+  }
+
+  return {
+    start() {
+      if (timer) return;
+      timer = setInterval(() => { fire({ manual: false }).catch(() => {}); }, intervalMs);
+      // Don't keep the event loop alive solely for the dream ticker.
+      if (typeof timer.unref === 'function') timer.unref();
+    },
+    stop() {
+      if (timer) { clearInterval(timer); timer = null; }
+    },
+    triggerNow(scopeFilter) {
+      return fire({ manual: true, scopeFilter });
+    },
+    isRunning() { return !!inflight; },
+    /** Test hook: fires once without scheduling a timer. */
+    _fire: fire,
+  };
+}

package/unify/dream-v2/segment.js

@@ -0,0 +1,191 @@
+/**
+ * dream-v2/segment.js — DESIGN-v2 §17.
+ *
+ * Three independent length-control concerns, kept pure so they can be
+ * unit-tested without touching disk or any LLM:
+ *
+ *   1. truncateMessage — clamp a single message body to
+ *      MAX_SINGLE_MESSAGE_CHARS, appending a clear notice. The full
+ *      body is still preserved in the conversation log; this only
+ *      affects what dream sees. (§17.3)
+ *
+ *   2. estimateTokens — rough chars-to-tokens approximation (we use 4
+ *      chars/token, a stable industry approximation that doesn't drag
+ *      a tokenizer into this layer; precise counts aren't required for
+ *      "should we segment?" decisions and a small over-count is the
+ *      safe direction).
+ *
+ *   3. segmentDiff — split a long per-group diff into K consecutive
+ *      slices, each ≤ MAX_DIFF_TOKENS_PER_TRIAGE, with a 3-message
+ *      overlap between adjacent slices for context continuity. (§17.1)
+ *
+ *   4. needsBatchedApply / batchSourcesForApply — when an Apply target's
+ *      memory + summary + sources cumulatively exceed MAX_APPLY_TOKENS,
+ *      split the sources (one source = one group's contribution) into
+ *      batches; the LLM is then called once per batch, threading the
+ *      written-back memory.md as input to the next batch. (§17.2)
+ *
+ * No side-effects. All functions are deterministic given their inputs.
+ */
+
+import {
+  MAX_SINGLE_MESSAGE_CHARS,
+  MAX_DIFF_TOKENS_PER_TRIAGE,
+  MAX_APPLY_TOKENS,
+  DREAM_OVERLAP,
+} from './limits.js';
+
+const TRUNCATION_NOTICE = '\n\n[message truncated for dream, original preserved in conversation log]';
+
+/**
+ * Truncate a single message body if it exceeds the per-message char cap.
+ * Idempotent: passing in an already-truncated body returns it unchanged.
+ *
+ * @param {string} body
+ * @returns {string}
+ */
+export function truncateMessage(body) {
+  const s = String(body || '');
+  if (s.length <= MAX_SINGLE_MESSAGE_CHARS) return s;
+  if (s.endsWith(TRUNCATION_NOTICE)) return s;
+  // Reserve room for the notice without overflowing the cap.
+  const room = Math.max(0, MAX_SINGLE_MESSAGE_CHARS - TRUNCATION_NOTICE.length);
+  return s.slice(0, room) + TRUNCATION_NOTICE;
+}
+
+/**
+ * Conservative chars-to-tokens approximation. We over-count slightly
+ * (1 token ≈ 4 chars) to make MAX_*_TOKENS act as a true upper bound.
+ *
+ * @param {string} text
+ * @returns {number}
+ */
+export function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(String(text).length / 4);
+}
+
+/**
+ * Estimate the token cost of an array of messages (header + body for each).
+ * @param {Array<{id?: string, role?: string, body?: string}>} msgs
+ */
+export function estimateMessagesTokens(msgs) {
+  if (!Array.isArray(msgs)) return 0;
+  let n = 0;
+  for (const m of msgs) {
+    n += estimateTokens(m.role || '');
+    n += estimateTokens(m.body || '');
+    n += 2; // separator overhead
+  }
+  return n;
+}
+
+/**
+ * Split a contiguous group diff into ≤MAX-token segments, with a
+ * DREAM_OVERLAP-message tail/head overlap between consecutive segments.
+ *
+ * Returns segments in temporal order. Each segment is `{ messages, kind }`
+ * where `kind` is 'overlap' for messages that exist only as continuity
+ * preamble (because they appeared in a prior segment), and 'new' for
+ * the rest. The first segment has no overlap header.
+ *
+ * Properties:
+ *   - The union of `kind: 'new'` messages across all segments equals
+ *     the input diff exactly, in order, with no duplicates.
+ *   - Each segment's total token estimate ≤ MAX_DIFF_TOKENS_PER_TRIAGE
+ *     unless a single message alone exceeds the cap, in which case
+ *     that message gets its own segment (we never split a message).
+ *
+ * @param {Array<{id?: string, role?: string, body?: string}>} diff
+ * @param {number} [maxTokens=MAX_DIFF_TOKENS_PER_TRIAGE]
+ * @param {number} [overlap=DREAM_OVERLAP]
+ * @returns {Array<{ messages: Array<object>, overlapCount: number, newCount: number }>}
+ */
+export function segmentDiff(diff, maxTokens = MAX_DIFF_TOKENS_PER_TRIAGE, overlap = DREAM_OVERLAP) {
+  const msgs = Array.isArray(diff) ? diff : [];
+  if (msgs.length === 0) return [];
+
+  // Fast path: whole diff fits in one segment.
+  if (estimateMessagesTokens(msgs) <= maxTokens) {
+    return [{ messages: msgs, overlapCount: 0, newCount: msgs.length }];
+  }
+
+  const segments = [];
+  let cursor = 0;
+  while (cursor < msgs.length) {
+    const overlapHead = segments.length > 0
+      ? msgs.slice(Math.max(0, cursor - overlap), cursor)
+      : [];
+    let used = estimateMessagesTokens(overlapHead);
+    let end = cursor;
+    while (end < msgs.length) {
+      const cost = estimateTokens(msgs[end].body || '') + estimateTokens(msgs[end].role || '') + 2;
+      if (used + cost > maxTokens && end > cursor) break;
+      used += cost;
+      end += 1;
+    }
+    // If we made no progress (single oversized message), advance by 1.
+    if (end === cursor) end = cursor + 1;
+    segments.push({
+      messages: [...overlapHead, ...msgs.slice(cursor, end)],
+      overlapCount: overlapHead.length,
+      newCount: end - cursor,
+    });
+    cursor = end;
+  }
+  return segments;
+}
+
+// ─── apply batching ───────────────────────────────────────────
+
+/**
+ * Decide whether a merged apply target needs to be split into batches.
+ *
+ * @param {{ memoryMd?: string, summaryMd?: string, sources: Array<{ groupId: string, diff: any }> }} merged
+ * @param {number} [maxTokens=MAX_APPLY_TOKENS]
+ */
+export function needsBatchedApply(merged, maxTokens = MAX_APPLY_TOKENS) {
+  return totalApplyTokens(merged) > maxTokens;
+}
+
+function totalApplyTokens(merged) {
+  let n = estimateTokens(merged.memoryMd || '') + estimateTokens(merged.summaryMd || '');
+  for (const src of merged.sources || []) n += estimateMessagesTokens(src.diff || []);
+  return n;
+}
+
+/**
+ * Pack `merged.sources` into ordered batches such that each batch's
+ * (memoryMd + summaryMd + that batch's sources) ≤ maxTokens. The first
+ * batch uses the original memoryMd; subsequent batches assume the LLM's
+ * previous-batch output replaces memoryMd, so we account for the same
+ * baseline cost in each batch.
+ *
+ * If a single source (one group's diff) alone would overflow, it still
+ * goes into its own batch — we never split a source diff here (segment
+ * happens earlier, in triage).
+ *
+ * @param {{ memoryMd?: string, summaryMd?: string, sources: Array<{ groupId: string, diff: any }> }} merged
+ * @param {number} [maxTokens=MAX_APPLY_TOKENS]
+ * @returns {Array<{ groupId: string, diff: any }[]>}
+ */
+export function batchSourcesForApply(merged, maxTokens = MAX_APPLY_TOKENS) {
+  const sources = Array.isArray(merged.sources) ? merged.sources : [];
+  if (sources.length === 0) return [];
+  const baseline = estimateTokens(merged.memoryMd || '') + estimateTokens(merged.summaryMd || '');
+  const batches = [];
+  let cur = [];
+  let used = baseline;
+  for (const src of sources) {
+    const cost = estimateMessagesTokens(src.diff || []);
+    if (cur.length > 0 && used + cost > maxTokens) {
+      batches.push(cur);
+      cur = [];
+      used = baseline;
+    }
+    cur.push(src);
+    used += cost;
+  }
+  if (cur.length > 0) batches.push(cur);
+  return batches;
+}

package/unify/dream-v2/snapshot.js

@@ -0,0 +1,80 @@
+/**
+ * dream-v2/snapshot.js — DESIGN-v2 §16.3 + §10.
+ *
+ * Pre-Apply backup of memory.md + summary.md to
+ * `~/.yeaft/memory/.dream-bak/<ts>/<scope-path>/`. The runner takes a
+ * snapshot once per merged target before Apply mutates it; that snapshot
+ * is the unit of rollback in case of LLM error or write failure.
+ *
+ * `pruneOldSnapshots()` keeps the most recent DREAM_BACKUP_KEEP
+ * timestamp directories under `.dream-bak/` and rm-rf's the rest.
+ *
+ * Pure I/O. No LLM. No control-flow.
+ */
+
+import { promises as fsp, existsSync } from 'fs';
+import { join, dirname } from 'path';
+
+import { DREAM_BACKUP_KEEP } from './limits.js';
+
+/** Folder name where snapshots live, relative to memory root. */
+export const BACKUP_DIRNAME = '.dream-bak';
+
+/**
+ * Build a stable filesystem-safe ISO timestamp string. Same shape that
+ * the migration script uses (`migrate-r6-to-v2.js`).
+ */
+export function tsForBackup(d = new Date()) {
+  return d.toISOString().replace(/[:.]/g, '-');
+}
+
+/**
+ * Snapshot a single scope's memory.md + summary.md into
+ * `<root>/.dream-bak/<ts>/<scopeRelDir>/`. Missing source files are
+ * skipped silently; the destination dir is always created so that an
+ * absent snapshot is still distinguishable from "didn't run".
+ *
+ * @param {string} root — memory root
+ * @param {string} ts   — timestamp folder name (re-use across all
+ *                        scopes in one dream pass)
+ * @param {string} scopeRelDir — e.g. 'user', 'group/g-eng', 'topic/sci/phys'
+ * @returns {Promise<{ backupDir: string, copied: string[] }>}
+ */
+export async function snapshotScope(root, ts, scopeRelDir) {
+  const srcDir = join(root, scopeRelDir);
+  const dstDir = join(root, BACKUP_DIRNAME, ts, scopeRelDir);
+  await fsp.mkdir(dstDir, { recursive: true });
+  const copied = [];
+  for (const name of ['memory.md', 'summary.md']) {
+    const s = join(srcDir, name);
+    if (!existsSync(s)) continue;
+    const d = join(dstDir, name);
+    await fsp.copyFile(s, d);
+    copied.push(name);
+  }
+  return { backupDir: dstDir, copied };
+}
+
+/**
+ * Keep the `keep` newest snapshot timestamps, rm-rf the rest.
+ *
+ * @param {string} root
+ * @param {number} [keep=DREAM_BACKUP_KEEP]
+ * @returns {Promise<{ kept: string[], removed: string[] }>}
+ */
+export async function pruneOldSnapshots(root, keep = DREAM_BACKUP_KEEP) {
+  const baseDir = join(root, BACKUP_DIRNAME);
+  if (!existsSync(baseDir)) return { kept: [], removed: [] };
+  let entries;
+  try { entries = await fsp.readdir(baseDir, { withFileTypes: true }); }
+  catch (err) { if (err && err.code === 'ENOENT') return { kept: [], removed: [] }; throw err; }
+  const dirs = entries.filter(e => e.isDirectory()).map(e => e.name).sort();
+  // sort() of ISO-with-dashes timestamps is chronological.
+  const cutoff = Math.max(0, dirs.length - keep);
+  const removed = dirs.slice(0, cutoff);
+  const kept = dirs.slice(cutoff);
+  for (const name of removed) {
+    await fsp.rm(join(baseDir, name), { recursive: true, force: true }).catch(() => {});
+  }
+  return { kept, removed };
+}

package/unify/dream-v2/state.js

@@ -0,0 +1,177 @@
+/**
+ * dream-v2/state.js — DESIGN-v2 §11.
+ *
+ * Two pieces of state, tracked separately:
+ *
+ *   1. Per-group control state (used to decide whether a group enters
+ *      triage and how far to advance the cursor):
+ *
+ *        ~/.yeaft/memory/group/<id>/.dream-state
+ *
+ *      A 3-line text file:
+ *
+ *        lastDreamMessageId: m-1024
+ *        lastDreamAt: 2026-04-28T03:07:00Z
+ *        messageCount: 491
+ *
+ *      Fields are independent of each other; missing fields default to
+ *      empty / null / 0. The file is rewritten atomically every dream.
+ *
+ *      The virtual `_no-group/` group lives at the same path layout
+ *      (`group/_no-group/.dream-state`) and uses the same accessor.
+ *
+ *   2. Per-scope observability marker, embedded inside the scope's
+ *      `memory.md` between two HTML comments at the file's tail:
+ *
+ *        <!-- dream-state -->
+ *        lastDreamAt: 2026-04-28T03:07:00Z
+ *        <!-- /dream-state -->
+ *
+ *      Read for the debug panel only; it does NOT participate in any
+ *      control-flow decision. We update it by replacing the existing
+ *      block (if any) or appending a new one to the end of the file.
+ *
+ * Both helpers are pure I/O; no LLM, no logic beyond parsing.
+ */
+
+import { promises as fsp, existsSync } from 'fs';
+import { join, dirname } from 'path';
+
+const STATE_FILE = '.dream-state';
+const DREAM_BLOCK_OPEN = '<!-- dream-state -->';
+const DREAM_BLOCK_CLOSE = '<!-- /dream-state -->';
+
+// ─── per-group ────────────────────────────────────────────────
+
+/**
+ * Read a group's .dream-state. Missing file → defaults.
+ *
+ * @param {string} root — memory root, e.g. ~/.yeaft/memory
+ * @param {string} groupId
+ * @returns {Promise<{ lastDreamMessageId: string|null, lastDreamAt: string|null, messageCount: number }>}
+ */
+export async function readGroupState(root, groupId) {
+  const abs = join(root, 'group', groupId, STATE_FILE);
+  const empty = { lastDreamMessageId: null, lastDreamAt: null, messageCount: 0 };
+  let raw;
+  try { raw = await fsp.readFile(abs, 'utf8'); }
+  catch (err) { if (err && err.code === 'ENOENT') return empty; throw err; }
+  return parseGroupState(raw);
+}
+
+/**
+ * Atomically rewrite a group's .dream-state. Creates the group dir if
+ * absent. Unknown fields are ignored.
+ *
+ * @param {string} root
+ * @param {string} groupId
+ * @param {{ lastDreamMessageId?: string|null, lastDreamAt?: string|null, messageCount?: number }} state
+ */
+export async function writeGroupState(root, groupId, state) {
+  const dir = join(root, 'group', groupId);
+  await fsp.mkdir(dir, { recursive: true });
+  const abs = join(dir, STATE_FILE);
+  const body =
+    `lastDreamMessageId: ${state.lastDreamMessageId == null ? '' : state.lastDreamMessageId}\n` +
+    `lastDreamAt: ${state.lastDreamAt == null ? '' : state.lastDreamAt}\n` +
+    `messageCount: ${Number.isFinite(state.messageCount) ? state.messageCount : 0}\n`;
+  await atomicWrite(abs, body);
+}
+
+/**
+ * Parse the 3-line key:value format. Tolerant of stray whitespace and
+ * empty values.
+ * @param {string} raw
+ */
+function parseGroupState(raw) {
+  const out = { lastDreamMessageId: null, lastDreamAt: null, messageCount: 0 };
+  const lines = String(raw || '').split(/\r?\n/);
+  for (const ln of lines) {
+    const m = /^(\w[\w-]*)\s*:\s*(.*)$/.exec(ln);
+    if (!m) continue;
+    const k = m[1];
+    const v = m[2].trim();
+    if (k === 'lastDreamMessageId') out.lastDreamMessageId = v || null;
+    else if (k === 'lastDreamAt') out.lastDreamAt = v || null;
+    else if (k === 'messageCount') {
+      const n = Number(v);
+      out.messageCount = Number.isFinite(n) ? n : 0;
+    }
+  }
+  return out;
+}
+
+// ─── per-scope marker (memory.md tail block) ───────────────────
+
+/**
+ * Read the lastDreamAt timestamp from a scope's memory.md, or null if
+ * the file or the dream-state block is absent.
+ *
+ * @param {string} memoryMdAbsPath
+ * @returns {Promise<string|null>}
+ */
+export async function readScopeDreamMarker(memoryMdAbsPath) {
+  let raw;
+  try { raw = await fsp.readFile(memoryMdAbsPath, 'utf8'); }
+  catch (err) { if (err && err.code === 'ENOENT') return null; throw err; }
+  const block = extractDreamBlock(raw);
+  if (!block) return null;
+  const m = /^lastDreamAt:\s*(.*)$/m.exec(block);
+  return m ? (m[1].trim() || null) : null;
+}
+
+/**
+ * Replace or append the per-scope dream-state block in memory.md.
+ * Returns the new file body (caller decides how to persist).
+ *
+ * @param {string} memoryMd — current full file content
+ * @param {{ lastDreamAt: string }} fields
+ * @returns {string}
+ */
+export function withDreamMarker(memoryMd, fields) {
+  const block = renderDreamBlock(fields);
+  const body = String(memoryMd || '');
+  if (body.includes(DREAM_BLOCK_OPEN) && body.includes(DREAM_BLOCK_CLOSE)) {
+    // Replace existing block.
+    return body.replace(
+      new RegExp(`${escapeRe(DREAM_BLOCK_OPEN)}[\\s\\S]*?${escapeRe(DREAM_BLOCK_CLOSE)}`),
+      block,
+    );
+  }
+  // Append. Ensure exactly one newline before the block.
+  const trimmed = body.replace(/\s+$/, '');
+  const sep = trimmed.length === 0 ? '' : '\n\n';
+  return `${trimmed}${sep}${block}\n`;
+}
+
+/**
+ * Extract the contents of the dream-state block (between the two HTML
+ * comments). Returns null if the block isn't present.
+ * @param {string} body
+ */
+function extractDreamBlock(body) {
+  const re = new RegExp(`${escapeRe(DREAM_BLOCK_OPEN)}([\\s\\S]*?)${escapeRe(DREAM_BLOCK_CLOSE)}`);
+  const m = re.exec(String(body || ''));
+  return m ? m[1].trim() : null;
+}
+
+function renderDreamBlock(fields) {
+  const lines = [DREAM_BLOCK_OPEN];
+  if (fields.lastDreamAt) lines.push(`lastDreamAt: ${fields.lastDreamAt}`);
+  lines.push(DREAM_BLOCK_CLOSE);
+  return lines.join('\n');
+}
+
+function escapeRe(s) { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }
+
+// ─── shared atomic writer ─────────────────────────────────────
+
+async function atomicWrite(absPath, content) {
+  await fsp.mkdir(dirname(absPath), { recursive: true });
+  const tmp = `${absPath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2, 8)}`;
+  await fsp.writeFile(tmp, content, 'utf8');
+  await fsp.rename(tmp, absPath);
+}
+
+// re-exported for tests
+export const _internals = { parseGroupState, extractDreamBlock };