@yeaft/webchat-agent 0.1.629 → 0.1.631

package/unify/dream-v2/state.js

@@ -0,0 +1,177 @@
+/**
+ * dream-v2/state.js — DESIGN-v2 §11.
+ *
+ * Two pieces of state, tracked separately:
+ *
+ *   1. Per-group control state (used to decide whether a group enters
+ *      triage and how far to advance the cursor):
+ *
+ *        ~/.yeaft/memory/group/<id>/.dream-state
+ *
+ *      A 3-line text file:
+ *
+ *        lastDreamMessageId: m-1024
+ *        lastDreamAt: 2026-04-28T03:07:00Z
+ *        messageCount: 491
+ *
+ *      Fields are independent of each other; missing fields default to
+ *      empty / null / 0. The file is rewritten atomically every dream.
+ *
+ *      The virtual `_no-group/` group lives at the same path layout
+ *      (`group/_no-group/.dream-state`) and uses the same accessor.
+ *
+ *   2. Per-scope observability marker, embedded inside the scope's
+ *      `memory.md` between two HTML comments at the file's tail:
+ *
+ *        <!-- dream-state -->
+ *        lastDreamAt: 2026-04-28T03:07:00Z
+ *        <!-- /dream-state -->
+ *
+ *      Read for the debug panel only; it does NOT participate in any
+ *      control-flow decision. We update it by replacing the existing
+ *      block (if any) or appending a new one to the end of the file.
+ *
+ * Both helpers are pure I/O; no LLM, no logic beyond parsing.
+ */
+
+import { promises as fsp, existsSync } from 'fs';
+import { join, dirname } from 'path';
+
+const STATE_FILE = '.dream-state';
+const DREAM_BLOCK_OPEN = '<!-- dream-state -->';
+const DREAM_BLOCK_CLOSE = '<!-- /dream-state -->';
+
+// ─── per-group ────────────────────────────────────────────────
+
+/**
+ * Read a group's .dream-state. Missing file → defaults.
+ *
+ * @param {string} root — memory root, e.g. ~/.yeaft/memory
+ * @param {string} groupId
+ * @returns {Promise<{ lastDreamMessageId: string|null, lastDreamAt: string|null, messageCount: number }>}
+ */
+export async function readGroupState(root, groupId) {
+  const abs = join(root, 'group', groupId, STATE_FILE);
+  const empty = { lastDreamMessageId: null, lastDreamAt: null, messageCount: 0 };
+  let raw;
+  try { raw = await fsp.readFile(abs, 'utf8'); }
+  catch (err) { if (err && err.code === 'ENOENT') return empty; throw err; }
+  return parseGroupState(raw);
+}
+
+/**
+ * Atomically rewrite a group's .dream-state. Creates the group dir if
+ * absent. Unknown fields are ignored.
+ *
+ * @param {string} root
+ * @param {string} groupId
+ * @param {{ lastDreamMessageId?: string|null, lastDreamAt?: string|null, messageCount?: number }} state
+ */
+export async function writeGroupState(root, groupId, state) {
+  const dir = join(root, 'group', groupId);
+  await fsp.mkdir(dir, { recursive: true });
+  const abs = join(dir, STATE_FILE);
+  const body =
+    `lastDreamMessageId: ${state.lastDreamMessageId == null ? '' : state.lastDreamMessageId}\n` +
+    `lastDreamAt: ${state.lastDreamAt == null ? '' : state.lastDreamAt}\n` +
+    `messageCount: ${Number.isFinite(state.messageCount) ? state.messageCount : 0}\n`;
+  await atomicWrite(abs, body);
+}
+
+/**
+ * Parse the 3-line key:value format. Tolerant of stray whitespace and
+ * empty values.
+ * @param {string} raw
+ */
+function parseGroupState(raw) {
+  const out = { lastDreamMessageId: null, lastDreamAt: null, messageCount: 0 };
+  const lines = String(raw || '').split(/\r?\n/);
+  for (const ln of lines) {
+    const m = /^(\w[\w-]*)\s*:\s*(.*)$/.exec(ln);
+    if (!m) continue;
+    const k = m[1];
+    const v = m[2].trim();
+    if (k === 'lastDreamMessageId') out.lastDreamMessageId = v || null;
+    else if (k === 'lastDreamAt') out.lastDreamAt = v || null;
+    else if (k === 'messageCount') {
+      const n = Number(v);
+      out.messageCount = Number.isFinite(n) ? n : 0;
+    }
+  }
+  return out;
+}
+
+// ─── per-scope marker (memory.md tail block) ───────────────────
+
+/**
+ * Read the lastDreamAt timestamp from a scope's memory.md, or null if
+ * the file or the dream-state block is absent.
+ *
+ * @param {string} memoryMdAbsPath
+ * @returns {Promise<string|null>}
+ */
+export async function readScopeDreamMarker(memoryMdAbsPath) {
+  let raw;
+  try { raw = await fsp.readFile(memoryMdAbsPath, 'utf8'); }
+  catch (err) { if (err && err.code === 'ENOENT') return null; throw err; }
+  const block = extractDreamBlock(raw);
+  if (!block) return null;
+  const m = /^lastDreamAt:\s*(.*)$/m.exec(block);
+  return m ? (m[1].trim() || null) : null;
+}
+
+/**
+ * Replace or append the per-scope dream-state block in memory.md.
+ * Returns the new file body (caller decides how to persist).
+ *
+ * @param {string} memoryMd — current full file content
+ * @param {{ lastDreamAt: string }} fields
+ * @returns {string}
+ */
+export function withDreamMarker(memoryMd, fields) {
+  const block = renderDreamBlock(fields);
+  const body = String(memoryMd || '');
+  if (body.includes(DREAM_BLOCK_OPEN) && body.includes(DREAM_BLOCK_CLOSE)) {
+    // Replace existing block.
+    return body.replace(
+      new RegExp(`${escapeRe(DREAM_BLOCK_OPEN)}[\\s\\S]*?${escapeRe(DREAM_BLOCK_CLOSE)}`),
+      block,
+    );
+  }
+  // Append. Ensure exactly one newline before the block.
+  const trimmed = body.replace(/\s+$/, '');
+  const sep = trimmed.length === 0 ? '' : '\n\n';
+  return `${trimmed}${sep}${block}\n`;
+}
+
+/**
+ * Extract the contents of the dream-state block (between the two HTML
+ * comments). Returns null if the block isn't present.
+ * @param {string} body
+ */
+function extractDreamBlock(body) {
+  const re = new RegExp(`${escapeRe(DREAM_BLOCK_OPEN)}([\\s\\S]*?)${escapeRe(DREAM_BLOCK_CLOSE)}`);
+  const m = re.exec(String(body || ''));
+  return m ? m[1].trim() : null;
+}
+
+function renderDreamBlock(fields) {
+  const lines = [DREAM_BLOCK_OPEN];
+  if (fields.lastDreamAt) lines.push(`lastDreamAt: ${fields.lastDreamAt}`);
+  lines.push(DREAM_BLOCK_CLOSE);
+  return lines.join('\n');
+}
+
+function escapeRe(s) { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }
+
+// ─── shared atomic writer ─────────────────────────────────────
+
+async function atomicWrite(absPath, content) {
+  await fsp.mkdir(dirname(absPath), { recursive: true });
+  const tmp = `${absPath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2, 8)}`;
+  await fsp.writeFile(tmp, content, 'utf8');
+  await fsp.rename(tmp, absPath);
+}
+
+// re-exported for tests
+export const _internals = { parseGroupState, extractDreamBlock };

package/unify/dream-v2/triage.js

@@ -0,0 +1,287 @@
+/**
+ * dream-v2/triage.js — DESIGN-v2 §14.
+ *
+ * Decide, for one group's diff, which scopes should be touched by Apply.
+ * The decision is two-staged on purpose:
+ *
+ *   1. **Hard rules** (this module, no LLM): everything we can determine
+ *      from message metadata. Always include the active group, every VP
+ *      that spoke as an assistant in the diff, every feature referenced
+ *      via `featureId`, and `user` (so painted-over user-profile signals
+ *      can't be missed).
+ *
+ *   2. **Soft classification** (LLM, two passes):
+ *        Pass-1: high-recall — does the diff carry user-profile signal?
+ *                              what topics (category-level) does it touch?
+ *        Pass-2: high-precision — for each topic Pass-1 surfaced, bind
+ *                              it to an exact existing path or propose
+ *                              a new ≤2-level path.
+ *
+ *      VP / group / feature are deliberately NOT asked of the LLM —
+ *      Hard Rules already cover them, and giving the LLM a chance to
+ *      drop a structurally-required scope would weaken the contract.
+ *
+ *      `user_profile_signals === true` does not need Pass-2 either: the
+ *      Hard Rule already added `user` and Apply itself decides whether
+ *      to actually rewrite anything (an UPDATE with no relevant content
+ *      reads as a no-op rewrite of the existing memory).
+ *
+ * The LLM is injected as a callable: `llm({ pass, prompt, system })` →
+ * Promise<string>. Tests pass a stub; runner injects the real adapter.
+ *
+ * The triage actions are unioned across segments when one group's diff
+ * is split in segment.js — segment-level triage runs N times and the
+ * caller dedupes. (Implemented as a thin wrapper `triageGroupSegments`
+ * below.)
+ */
+
+import { isValidTopic } from '../memory/store-v2.js';
+import { truncateMessage } from './segment.js';
+
+const SYSTEM = `You are the Triage stage of a dream pipeline that decides which scopes a recent group conversation should affect. Reply with strict JSON only — no prose, no markdown fences.`;
+
+/**
+ * Hard rules: deterministically derive must-include scopes from the
+ * structure of the diff.
+ *
+ * Inputs:
+ *   - groupId: the active group ('_no-group' is allowed and skips the
+ *     `group/<id>` entry — by convention the virtual group has no scope
+ *     of its own).
+ *   - messages: the diff (already overlap-prefixed if applicable).
+ *
+ * @param {{ groupId: string, messages: Array<object> }} args
+ * @returns {Array<{ kind: 'update', scope: string }>}
+ */
+export function applyHardRules({ groupId, messages }) {
+  const out = new Map();
+  const add = (scope) => { if (!out.has(scope)) out.set(scope, { kind: 'update', scope }); };
+
+  // user is always in.
+  add('user');
+
+  // active group, except the virtual _no-group bucket.
+  if (groupId && groupId !== '_no-group') add(`group/${groupId}`);
+
+  for (const m of (messages || [])) {
+    if (!m || typeof m !== 'object') continue;
+    // Active VP: any assistant message's vpId.
+    if (m.role === 'assistant') {
+      const vp = m.vpId || (m.author && /^vp:(.+)$/.exec(m.author)?.[1]);
+      if (vp && /^[A-Za-z0-9_\-.一-鿿]+$/.test(vp)) add(`vp/${vp}`);
+    }
+    // Active feature: explicit metadata.
+    if (m.featureId && /^[A-Za-z0-9_\-.一-鿿]+$/.test(m.featureId)) {
+      add(`feature/${m.featureId}`);
+    }
+  }
+
+  return Array.from(out.values());
+}
+
+// ─── soft classification ──────────────────────────────────────
+
+/**
+ * Build the prompt used for Pass-1.
+ *
+ * @param {{ groupId: string, messages: Array<object>, topicSummaries: Array<{ path: string, summary: string }> }} ctx
+ */
+export function buildPass1Prompt(ctx) {
+  const lines = [
+    'You are deciding whether a recent group conversation carries:',
+    '  - signals that should update the USER profile, and/or',
+    '  - signals that should update one or more TOPIC scopes.',
+    '',
+    'Do NOT mention vp/, group/, or feature/ scopes — those are handled by hard rules.',
+    '',
+    `Group: ${ctx.groupId}`,
+    '',
+    'Existing topic scopes (path — summary):',
+  ];
+  if (!ctx.topicSummaries || ctx.topicSummaries.length === 0) {
+    lines.push('  (none)');
+  } else {
+    for (const t of ctx.topicSummaries) {
+      lines.push(`  - ${t.path} — ${oneLine(t.summary)}`);
+    }
+  }
+  lines.push('', 'Conversation:');
+  for (const m of (ctx.messages || [])) {
+    const head = `[${m.role || 'message'}${m.kind === 'overlap' ? ' (already processed)' : ''}]`;
+    lines.push(head);
+    lines.push(truncateMessage(m.body || ''));
+    lines.push('');
+  }
+  lines.push('Respond with strict JSON of the shape:');
+  lines.push('{');
+  lines.push('  "user_profile_signals": boolean,');
+  lines.push('  "topics": [ "<short category description>", ... ],');
+  lines.push('  "trivial_only": boolean');
+  lines.push('}');
+  return lines.join('\n');
+}
+
+/**
+ * Build the Pass-2 prompt for a single topic description.
+ *
+ * @param {{ description: string, existingTopics: Array<{ path: string, summary: string }> }} ctx
+ */
+export function buildPass2Prompt(ctx) {
+  const lines = [
+    'Bind a free-form topic description to an exact path under topic/.',
+    'Rules:',
+    '  - At most TWO path segments. Reject any third level.',
+    '  - Segments may contain letters, digits, dashes, underscores, dots, CJK.',
+    '  - Prefer matching an existing path if the description fits.',
+    '',
+    `Description: ${ctx.description}`,
+    '',
+    'Existing topics:',
+  ];
+  if (!ctx.existingTopics || ctx.existingTopics.length === 0) {
+    lines.push('  (none)');
+  } else {
+    for (const t of ctx.existingTopics) {
+      lines.push(`  - ${t.path} — ${oneLine(t.summary)}`);
+    }
+  }
+  lines.push('');
+  lines.push('Reply with strict JSON, exactly one of:');
+  lines.push('  { "decision": "match", "path": "<existing path>" }');
+  lines.push('  { "decision": "new",   "path": "<new ≤2-segment path>" }');
+  lines.push('  { "decision": "none" }');
+  return lines.join('\n');
+}
+
+/**
+ * Run soft classification for one segment of one group's diff.
+ *
+ * @param {{
+ *   groupId: string,
+ *   messages: Array<object>,
+ *   topicSummaries: Array<{ path: string, summary: string }>,
+ *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ * }} args
+ * @returns {Promise<Array<{ kind: 'update'|'create', scope: string }>>}
+ */
+export async function classifySoft({ groupId, messages, topicSummaries, llm }) {
+  if (!llm) throw new Error('triage.classifySoft: llm callable required');
+  const pass1Prompt = buildPass1Prompt({ groupId, messages, topicSummaries });
+  const pass1Raw = await llm({ pass: 'triage-pass1', prompt: pass1Prompt, system: SYSTEM });
+  const pass1 = parseJsonSafe(pass1Raw);
+  const out = [];
+
+  // user_profile_signals: covered by hard rules; we only emit explicit
+  // user action here when Pass-1 says yes (idempotent if hard rules
+  // already added it).
+  if (pass1 && pass1.user_profile_signals === true) {
+    out.push({ kind: 'update', scope: 'user' });
+  }
+
+  const topicDescriptions = (pass1 && Array.isArray(pass1.topics)) ? pass1.topics : [];
+  for (const description of topicDescriptions) {
+    if (typeof description !== 'string' || !description.trim()) continue;
+    const pass2Prompt = buildPass2Prompt({
+      description: description.trim(),
+      existingTopics: topicSummaries || [],
+    });
+    const pass2Raw = await llm({ pass: 'triage-pass2', prompt: pass2Prompt, system: SYSTEM });
+    const pass2 = parseJsonSafe(pass2Raw);
+    if (!pass2 || !pass2.decision) continue;
+    if (pass2.decision === 'none') continue;
+    const path = String(pass2.path || '').trim();
+    if (!path) continue;
+    const segs = path.split('/').filter(Boolean);
+    if (!isValidTopic({ kind: 'topic', path: segs })) continue;
+    const scope = `topic/${segs.join('/')}`;
+    if (pass2.decision === 'match') {
+      out.push({ kind: 'update', scope });
+    } else if (pass2.decision === 'new') {
+      out.push({ kind: 'create', scope });
+    }
+  }
+  return out;
+}
+
+/**
+ * Combine hard-rule and soft-classification results for one segment.
+ * Dedupes by scope — `update` wins if any source said update.
+ *
+ * @param {{
+ *   groupId: string,
+ *   messages: Array<object>,
+ *   topicSummaries: Array<{ path: string, summary: string }>,
+ *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ * }} args
+ * @returns {Promise<Array<{ kind: 'update'|'create', scope: string }>>}
+ */
+export async function triageOneSegment(args) {
+  const hard = applyHardRules({ groupId: args.groupId, messages: args.messages });
+  const soft = await classifySoft(args);
+  return dedupeActions([...hard, ...soft]);
+}
+
+/**
+ * Triage a group's diff that has already been split into N segments.
+ * Runs each segment serially, accumulates and dedupes actions.
+ *
+ * @param {{
+ *   groupId: string,
+ *   segments: Array<{ messages: Array<object> }>,
+ *   topicSummaries: Array<{ path: string, summary: string }>,
+ *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ *   onProgress?: (event: object) => void,
+ * }} args
+ * @returns {Promise<Array<{ kind: 'update'|'create', scope: string }>>}
+ */
+export async function triageGroupSegments({ groupId, segments, topicSummaries, llm, onProgress }) {
+  let acc = [];
+  let i = 0;
+  for (const seg of (segments || [])) {
+    i += 1;
+    if (onProgress) onProgress({ phase: 'triage', groupId, segment: i, total: segments.length });
+    const segActions = await triageOneSegment({
+      groupId,
+      messages: seg.messages,
+      topicSummaries,
+      llm,
+    });
+    acc = dedupeActions([...acc, ...segActions]);
+  }
+  return acc;
+}
+
+// ─── helpers ──────────────────────────────────────────────────
+
+function dedupeActions(list) {
+  const map = new Map();
+  for (const a of list) {
+    if (!a || !a.scope) continue;
+    const cur = map.get(a.scope);
+    if (!cur) { map.set(a.scope, { ...a }); continue; }
+    if (a.kind === 'update') cur.kind = 'update';
+  }
+  return Array.from(map.values());
+}
+
+function oneLine(s) {
+  return String(s || '').replace(/\s+/g, ' ').trim().slice(0, 200);
+}
+
+/** Lenient JSON parse: tolerate fenced ```json blocks. Returns null on failure. */
+export function parseJsonSafe(raw) {
+  if (typeof raw !== 'string') return null;
+  let s = raw.trim();
+  // Strip markdown fences if present.
+  const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/.exec(s);
+  if (fenced) s = fenced[1].trim();
+  try { return JSON.parse(s); }
+  catch { /* try to recover the first {...} block */ }
+  const start = s.indexOf('{');
+  const end = s.lastIndexOf('}');
+  if (start >= 0 && end > start) {
+    try { return JSON.parse(s.slice(start, end + 1)); }
+    catch { return null; }
+  }
+  return null;
+}

package/unify/engine.js

@@ -21,6 +21,7 @@ import { randomUUID } from 'crypto';
 import { buildSystemPrompt, buildWorkerPrompt } from './prompts.js';
 import { LLMContextError, LLMAbortError } from './llm/adapter.js';
 import { recallR6, formatForInjection } from './memory/recall-r6.js';
+import { recallV2 } from './memory/recall-v2.js';
 import { shouldConsolidate, consolidate, partitionMessages } from './memory/consolidate.js';
 import { extractMemories } from './memory/extract.js';
 import { runCompact as runCompactOrchestrator } from './compact/orchestrator.js';
@@ -489,15 +490,40 @@ export class Engine {
 
   /**
    * Perform memory recall for a given prompt.
-   * Uses recallR6 (R6 shard-based recall) when memoryShardStore is available,
-   * falling back to empty results if not.
+   *
+   * Routes:
+   *   - config.memoryV2 === true → recall-v2 (per-scope memory.md + summary.md)
+   *   - else → R6 shard-based recall (legacy)
    *
    * @param {string} prompt
+   * @param {{ groupId?: string, vpId?: string, featureId?: string }} [ctx]
    * @returns {Promise<{ profile: string, entries: object[], formatted: string }|null>}
    */
-  async #recallMemory(prompt) {
+  async #recallMemory(prompt, ctx = {}) {
     const memory = { profile: '', entries: [], formatted: '' };
 
+    // ─── v2 path (DESIGN-v2) ───────────────────────────────────
+    if (this.#config && this.#config.memoryV2 && this.#yeaftDir) {
+      try {
+        const result = await recallV2({
+          prompt,
+          root: `${this.#yeaftDir}/memory`,
+          groupId: ctx.groupId,
+          vpId: ctx.vpId,
+          featureId: ctx.featureId,
+        });
+        memory.entries = result.sections || [];
+        memory.formatted = result.formatted || '';
+        // Profile concept: in v2 the user/memory.md IS the profile.
+        const userSec = (result.sections || []).find(s => s.kind === 'user');
+        memory.profile = userSec ? (userSec.summary || '') : '';
+      } catch {
+        // Fail soft — empty injection.
+      }
+      return memory;
+    }
+
+    // ─── R6 legacy path ────────────────────────────────────────
     // Build user profile from user-memory shard store (R6 path),
     // falling back to legacy readProfile if shard store unavailable.
     try {
@@ -885,7 +911,15 @@ export class Engine {
     }
 
     // R6 recall: append shard-based recall results to memory injection
-    const recallResult = await this.#recallMemory(prompt);
+    const recallResult = await this.#recallMemory(prompt, {
+      groupId,
+      vpId: vpPersona && typeof vpPersona === 'object' && typeof vpPersona.vpId === 'string'
+        ? vpPersona.vpId
+        : (typeof senderVpId === 'string' ? senderVpId : undefined),
+      featureId: typeof inboundEnvelope === 'object' && inboundEnvelope
+        ? inboundEnvelope.featureId
+        : undefined,
+    });
     if (recallResult && recallResult.formatted) {
       memoryInjection = memoryInjection
         ? memoryInjection + '\n\n' + recallResult.formatted