@yeaft/webchat-agent 0.1.629 → 0.1.631

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.629",
+  "version": "0.1.631",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/config.js

@@ -217,6 +217,8 @@ function loadLegacyConfig(dir, overrides) {
     maxContinueTurns: overrides.maxContinueTurns ?? fileConfig.maxContinueTurns ?? DEFAULTS.maxContinueTurns,
     // task-318: legacy path never had the `unify` section — defaults.
     unify: normaliseUnifySection(null),
+    // DESIGN-v2 feature flag — opt-in via override only on legacy path.
+    memoryV2: !!overrides.memoryV2,
     providers: null,
     primaryModel: null,
     fastModel: null,
@@ -313,6 +315,13 @@ export function loadConfig(overrides = {}) {
     // don't pollute the flat config namespace used by chat/crew code.
     unify: normaliseUnifySection(jsonConfig.unify),
 
+    // DESIGN-v2 feature flag. When true the engine routes recall through
+    // memory/recall-v2.js (per-scope memory.md + summary.md) and reads from
+    // the v2 store layout. Defaults to false during the rollout; PR-E flips
+    // the default and deletes the legacy paths.
+    memoryV2: overrides.memoryV2 !== undefined ? !!overrides.memoryV2
+      : (jsonConfig.memoryV2 !== undefined ? !!jsonConfig.memoryV2 : false),
+
     // Legacy fields (null when using config.json)
     apiKey: overrides.apiKey || null,
     openaiApiKey: null,

package/unify/dream-v2/apply.js

@@ -0,0 +1,271 @@
+/**
+ * dream-v2/apply.js — DESIGN-v2 §16 + §17.2.
+ *
+ * Execute one merged-target action: either UPDATE an existing scope's
+ * memory.md/summary.md, or CREATE a new one (currently topic-only).
+ *
+ * The flow per target:
+ *   1. snapshot existing files into .dream-bak/<ts>/<scope>/
+ *   2. if total content fits, run UPDATE (or CREATE) once
+ *   3. if it doesn't, batch the sources by group (segment.batchSourcesForApply)
+ *      and chain the LLM calls — each batch's output becomes the next
+ *      batch's `current memory.md`. The prompt threads "this is batch K
+ *      of N" so the LLM doesn't think previous content was lost.
+ *   4. write tmp + rename for memory.md and summary.md
+ *   5. update the per-scope dream-state marker inside memory.md
+ *
+ * The LLM call is injected (`opts.llm`). Snapshots are injectable too
+ * (`opts.snapshot`) to allow tests to skip the .dream-bak side-effect.
+ */
+
+import { promises as fsp } from 'fs';
+import { join, dirname } from 'path';
+
+import { writeMemory, writeSummary, readMemory, readSummary } from '../memory/store-v2.js';
+import { withDreamMarker } from './state.js';
+import { batchSourcesForApply, needsBatchedApply, truncateMessage } from './segment.js';
+import { snapshotScope } from './snapshot.js';
+import { parseJsonSafe } from './triage.js';
+
+const SYSTEM = `You are the Apply stage of a dream pipeline. You rewrite a single scope's memory.md and summary.md based on recent group conversations. Reply with strict JSON only — no prose, no fences.`;
+
+/**
+ * Build the UPDATE prompt body. Accepts the current scope state +
+ * one or more `(groupId, diff)` source blocks.
+ *
+ * @param {{
+ *   target: string,
+ *   memoryMd: string,
+ *   summaryMd: string,
+ *   sources: Array<{ groupId: string, diff: Array<object> }>,
+ *   batchInfo?: { index: number, total: number },
+ * }} ctx
+ */
+export function buildUpdatePrompt(ctx) {
+  const lines = [];
+  lines.push('You are updating an existing memory scope.');
+  lines.push('');
+  lines.push(`Scope: ${ctx.target}`);
+  if (ctx.batchInfo && ctx.batchInfo.total > 1) {
+    lines.push(`This is batch ${ctx.batchInfo.index} of ${ctx.batchInfo.total}.`);
+    lines.push('Earlier batches have already been folded into the current memory.md below.');
+  }
+  lines.push('');
+  lines.push('Current memory.md:');
+  lines.push('"""');
+  lines.push(ctx.memoryMd || '');
+  lines.push('"""');
+  lines.push('');
+  lines.push('Current summary.md:');
+  lines.push('"""');
+  lines.push(ctx.summaryMd || '');
+  lines.push('"""');
+  lines.push('');
+  lines.push('Recent conversations:');
+  for (const src of (ctx.sources || [])) {
+    lines.push('');
+    lines.push(`[group/${src.groupId}]`);
+    for (const m of (src.diff || [])) {
+      const head = `[${m.role || 'message'}${m.kind === 'overlap' ? ' (already processed)' : ''}]`;
+      lines.push(head);
+      lines.push(truncateMessage(m.body || ''));
+    }
+  }
+  lines.push('');
+  lines.push('Task:');
+  lines.push('- Extract from these conversations what is relevant to THIS scope.');
+  lines.push('- Integrate it into memory.md (reorganize sections if needed).');
+  lines.push('- Drop stale or contradicted entries.');
+  lines.push('- Rewrite summary.md (1–3 sentences).');
+  lines.push('- The same conversations are being processed for OTHER scopes too.');
+  lines.push('  Only handle what is relevant here. Ignore the rest.');
+  lines.push('');
+  lines.push('Hard rules:');
+  lines.push('- Never read or reference any other scope\'s files.');
+  lines.push('- Never modify VP system-prompt, group charter, or user preferences.');
+  lines.push('- If something contradicts a charter, annotate with');
+  lines.push('  "⚠️ contradicts charter — verify which is current" and continue.');
+  lines.push('');
+  lines.push('Reply with strict JSON of the shape:');
+  lines.push('{ "memory_md": "...", "summary_md": "..." }');
+  return lines.join('\n');
+}
+
+/**
+ * Build the CREATE prompt body. Used when a target scope does not yet
+ * exist on disk (currently only topic/<...>).
+ *
+ * @param {{
+ *   target: string,
+ *   sources: Array<{ groupId: string, diff: Array<object> }>,
+ *   siblingTopics?: Array<{ path: string, summary: string }>,
+ * }} ctx
+ */
+export function buildCreatePrompt(ctx) {
+  const lines = [];
+  lines.push('You are creating a new memory scope from scratch.');
+  lines.push('');
+  lines.push(`Scope path: ${ctx.target}   (must be ≤2 levels)`);
+  lines.push('');
+  lines.push('Source conversations:');
+  for (const src of (ctx.sources || [])) {
+    lines.push('');
+    lines.push(`[group/${src.groupId}]`);
+    for (const m of (src.diff || [])) {
+      const head = `[${m.role || 'message'}${m.kind === 'overlap' ? ' (already processed)' : ''}]`;
+      lines.push(head);
+      lines.push(truncateMessage(m.body || ''));
+    }
+  }
+  lines.push('');
+  if (ctx.siblingTopics && ctx.siblingTopics.length > 0) {
+    lines.push('For tone reference, sibling/parent topic summaries:');
+    for (const t of ctx.siblingTopics) {
+      lines.push(`  - ${t.path}: ${oneLine(t.summary)}`);
+    }
+    lines.push('');
+  }
+  lines.push('Task:');
+  lines.push('1. Write memory.md from scratch with reasonable section structure.');
+  lines.push('2. Write summary.md (1–3 sentences).');
+  lines.push('');
+  lines.push('Reply with strict JSON of the shape:');
+  lines.push('{ "memory_md": "...", "summary_md": "..." }');
+  return lines.join('\n');
+}
+
+/**
+ * Translate `target` like 'group/g-eng' or 'topic/sci/phys' to a Scope
+ * understood by store-v2. Throws if the path is malformed.
+ *
+ * @param {string} target
+ * @returns {{ kind: string, id?: string, path?: string[] }}
+ */
+export function targetToScope(target) {
+  if (!target || typeof target !== 'string') throw new Error('apply.targetToScope: target required');
+  if (target === 'user') return { kind: 'user' };
+  const segs = target.split('/').filter(Boolean);
+  const head = segs[0];
+  if (head === 'vp' && segs.length === 2) return { kind: 'vp', id: segs[1] };
+  if (head === 'group' && segs.length === 2) return { kind: 'group', id: segs[1] };
+  if (head === 'feature' && segs.length === 2) return { kind: 'feature', id: segs[1] };
+  if (head === 'topic' && (segs.length === 2 || segs.length === 3)) {
+    return { kind: 'topic', path: segs.slice(1) };
+  }
+  throw new Error(`apply.targetToScope: malformed target ${JSON.stringify(target)}`);
+}
+
+/**
+ * Run a single merged-target apply. Returns a record of what happened
+ * for the runner's debug-panel feed.
+ *
+ * @param {{
+ *   target: string,
+ *   kind: 'update'|'create',
+ *   sources: Array<{ groupId: string, diff: any }>,
+ * }} merged
+ * @param {{
+ *   root: string,
+ *   ts: string,                    // shared timestamp folder
+ *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ *   limits?: { MAX_APPLY_TOKENS?: number },
+ *   snapshot?: typeof snapshotScope,
+ *   nowIso?: () => string,
+ *   onProgress?: (event: object) => void,
+ *   siblingTopicsFor?: (target: string) => Promise<Array<{path:string, summary:string}>>,
+ * }} opts
+ */
+export async function applyMergedTarget(merged, opts) {
+  if (!opts || !opts.root) throw new Error('apply.applyMergedTarget: opts.root required');
+  if (!opts.llm) throw new Error('apply.applyMergedTarget: opts.llm required');
+  const ts = opts.ts || new Date().toISOString().replace(/[:.]/g, '-');
+  const snapFn = opts.snapshot || snapshotScope;
+  const nowIso = opts.nowIso ? opts.nowIso() : new Date().toISOString();
+  const scope = targetToScope(merged.target);
+  const scopeDirRel = scopeRelDir(scope);
+
+  if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'snapshot' });
+  await snapFn(opts.root, ts, scopeDirRel);
+
+  let memoryMd = await readMemory(scope, { root: opts.root });
+  let summaryMd = await readSummary(scope, { root: opts.root });
+
+  if (merged.kind === 'create' && (memoryMd || summaryMd)) {
+    // Race / partial state: the scope already exists. Treat as update —
+    // safer than overwriting arbitrary bytes.
+    merged = { ...merged, kind: 'update' };
+  }
+
+  let batchesUsed = 0;
+  const maxApply = (opts.limits && opts.limits.MAX_APPLY_TOKENS) || undefined;
+
+  if (merged.kind === 'create') {
+    const siblings = opts.siblingTopicsFor ? await opts.siblingTopicsFor(merged.target) : [];
+    const prompt = buildCreatePrompt({
+      target: merged.target,
+      sources: merged.sources,
+      siblingTopics: siblings,
+    });
+    if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'llm', batch: 1, of: 1 });
+    const raw = await opts.llm({ pass: 'create', prompt, system: SYSTEM });
+    const parsed = parseJsonSafe(raw);
+    if (!parsed || typeof parsed.memory_md !== 'string') {
+      throw new Error(`apply: CREATE returned malformed JSON for ${merged.target}`);
+    }
+    memoryMd = parsed.memory_md;
+    summaryMd = typeof parsed.summary_md === 'string' ? parsed.summary_md : '';
+    batchesUsed = 1;
+  } else {
+    // UPDATE — possibly batched.
+    const batches = needsBatchedApply(
+      { memoryMd, summaryMd, sources: merged.sources },
+      maxApply,
+    )
+      ? batchSourcesForApply({ memoryMd, summaryMd, sources: merged.sources }, maxApply)
+      : [merged.sources];
+
+    let i = 0;
+    for (const batch of batches) {
+      i += 1;
+      const prompt = buildUpdatePrompt({
+        target: merged.target,
+        memoryMd,
+        summaryMd,
+        sources: batch,
+        batchInfo: { index: i, total: batches.length },
+      });
+      if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'llm', batch: i, of: batches.length });
+      const raw = await opts.llm({ pass: 'update', prompt, system: SYSTEM });
+      const parsed = parseJsonSafe(raw);
+      if (!parsed || typeof parsed.memory_md !== 'string') {
+        throw new Error(`apply: UPDATE batch ${i} returned malformed JSON for ${merged.target}`);
+      }
+      memoryMd = parsed.memory_md;
+      if (typeof parsed.summary_md === 'string') summaryMd = parsed.summary_md;
+    }
+    batchesUsed = batches.length;
+  }
+
+  // Stamp the per-scope dream marker, then atomically write both files.
+  const stamped = withDreamMarker(memoryMd, { lastDreamAt: nowIso });
+  await writeMemory(scope, stamped, { root: opts.root });
+  await writeSummary(scope, summaryMd || '', { root: opts.root });
+
+  if (opts.onProgress) opts.onProgress({ phase: 'apply', target: merged.target, status: 'done', batches: batchesUsed });
+  return { target: merged.target, kind: merged.kind, batches: batchesUsed };
+}
+
+// ─── helpers ──────────────────────────────────────────────────
+
+function scopeRelDir(scope) {
+  switch (scope.kind) {
+    case 'user':    return 'user';
+    case 'vp':      return `vp/${scope.id}`;
+    case 'group':   return `group/${scope.id}`;
+    case 'feature': return `feature/${scope.id}`;
+    case 'topic':   return `topic/${scope.path.join('/')}`;
+    default: throw new Error(`apply.scopeRelDir: unknown kind ${scope.kind}`);
+  }
+}
+
+function oneLine(s) { return String(s || '').replace(/\s+/g, ' ').trim().slice(0, 200); }

package/unify/dream-v2/limits.js

@@ -0,0 +1,46 @@
+/**
+ * dream-v2/limits.js — DESIGN-v2 §18 constants.
+ *
+ * Centralised so tests and runtime share one source of truth. Exposed
+ * as both named exports and a `DEFAULT_LIMITS` object for `runDream()`
+ * callers that want to override one knob without restating the rest.
+ *
+ * `loadLimitsFromConfig(config)` merges a `~/.yeaft/config.json`
+ * `unify.dream` block on top of defaults; unknown keys are ignored,
+ * malformed values fall back to default.
+ */
+
+export const DREAM_INTERVAL_HOURS = 12;
+export const DREAM_OVERLAP = 3;
+export const MIN_NEW_PER_GROUP = 20;
+export const MAX_SINGLE_MESSAGE_CHARS = 8000;
+export const MAX_DIFF_TOKENS_PER_TRIAGE = 60000;
+export const MAX_APPLY_TOKENS = 80000;
+export const DREAM_BACKUP_KEEP = 7;
+
+export const DEFAULT_LIMITS = Object.freeze({
+  DREAM_INTERVAL_HOURS,
+  DREAM_OVERLAP,
+  MIN_NEW_PER_GROUP,
+  MAX_SINGLE_MESSAGE_CHARS,
+  MAX_DIFF_TOKENS_PER_TRIAGE,
+  MAX_APPLY_TOKENS,
+  DREAM_BACKUP_KEEP,
+});
+
+/**
+ * Merge a config object's `unify.dream` block on top of defaults.
+ * @param {object} [config]
+ */
+export function loadLimitsFromConfig(config) {
+  const out = { ...DEFAULT_LIMITS };
+  const ud = config && config.unify && config.unify.dream;
+  if (!ud || typeof ud !== 'object') return out;
+  for (const k of Object.keys(out)) {
+    if (Object.prototype.hasOwnProperty.call(ud, k)) {
+      const v = ud[k];
+      if (Number.isFinite(v) && v > 0) out[k] = v;
+    }
+  }
+  return out;
+}

package/unify/dream-v2/merge.js

@@ -0,0 +1,87 @@
+/**
+ * dream-v2/merge.js — DESIGN-v2 §15.
+ *
+ * Pure-code stage between Triage and Apply.
+ *
+ * Triage emits per-group action lists; the runner needs to flip that
+ * into a per-target source list so Apply runs once per target rather
+ * than once per (group × target) pair. (User scope in particular would
+ * be rewritten dozens of times if we didn't merge.)
+ *
+ * Input shape (one entry per group that crossed the newCount threshold):
+ *
+ *   [
+ *     {
+ *       groupId: 'g-eng',
+ *       diff: [<message>, ...],            // the per-group source diff
+ *                                          // (already truncated/segmented if needed)
+ *       actions: [
+ *         { kind: 'update', scope: 'group/g-eng' },
+ *         { kind: 'update', scope: 'vp/zhang-san' },
+ *         { kind: 'update', scope: 'user' },
+ *         { kind: 'create', scope: 'topic/life/parenting' },
+ *         ...
+ *       ],
+ *     },
+ *     ...
+ *   ]
+ *
+ * Output shape:
+ *
+ *   [
+ *     { target: 'user',
+ *       kind: 'update',                    // 'update' wins over 'create' if any group says update
+ *       sources: [
+ *         { groupId: 'g-eng',  diff: [...] },
+ *         { groupId: 'g-life', diff: [...] },
+ *       ],
+ *     },
+ *     { target: 'topic/life/parenting',
+ *       kind: 'create',                    // create only if every contributing group said create
+ *       sources: [{ groupId: 'g-life', diff: [...] }],
+ *     },
+ *     ...
+ *   ]
+ *
+ * Determinism contract:
+ *   - Targets are returned sorted alphabetically by target path.
+ *   - Within a target, sources are sorted by groupId.
+ *   This makes the debug-panel output predictable across runs.
+ */
+
+/**
+ * Merge per-group triage outputs into per-target apply units.
+ *
+ * @param {Array<{ groupId: string, diff: any, actions: Array<{ kind: 'update'|'create', scope: string }> }>} groupTriages
+ * @returns {Array<{ target: string, kind: 'update'|'create', sources: Array<{ groupId: string, diff: any }> }>}
+ */
+export function mergeByTarget(groupTriages) {
+  const byTarget = new Map();
+  for (const g of (groupTriages || [])) {
+    const groupId = g && g.groupId;
+    const diff = g && g.diff;
+    const actions = Array.isArray(g && g.actions) ? g.actions : [];
+    if (!groupId) continue;
+    for (const a of actions) {
+      if (!a || !a.scope) continue;
+      const k = a.kind === 'create' ? 'create' : 'update';
+      let entry = byTarget.get(a.scope);
+      if (!entry) {
+        entry = { target: a.scope, kind: k, sources: [] };
+        byTarget.set(a.scope, entry);
+      }
+      // 'update' wins: any contributing group that already considers
+      // the scope existing means we treat the apply as an update.
+      if (k === 'update') entry.kind = 'update';
+      // Avoid duplicate (target, group) pairs — should never happen
+      // in normal triage but be defensive.
+      if (!entry.sources.some(s => s.groupId === groupId)) {
+        entry.sources.push({ groupId, diff });
+      }
+    }
+  }
+  const out = Array.from(byTarget.values());
+  out.sort((a, b) => a.target.localeCompare(b.target));
+  for (const e of out) e.sources.sort((a, b) => a.groupId.localeCompare(b.groupId));
+  return out;
+}