@yeaft/webchat-agent 0.1.741 → 0.1.746

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.741",
+  "version": "0.1.746",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/dream-v2/schedule.js

@@ -26,13 +26,22 @@ export const DEFAULT_INTERVAL_MS = DREAM_INTERVAL_HOURS * 60 * 60 * 1000;
  * everything `runDream` needs (memory root, llm, message-store hooks,
  * onProgress sink); the scheduler only knows how to call it.
  *
+ * `keepAlive` controls whether the interval timer holds the event loop
+ * open. Default `false` preserves the original behaviour (CLI / one-shot
+ * invocations should not be kept alive solely by the dream ticker). The
+ * web server passes `true` because (a) the HTTP listener already pins
+ * the loop, so unref'ing buys nothing, and (b) on Node 22 some platforms
+ * never schedule unref'd timers when nothing else wakes them — meaning
+ * the server saw zero ticks in production. See PR fix/dream-cadence.
+ *
  * @param {{
  *   run: (opts: { manual: boolean, scopeFilter?: string[] }) => Promise<object>,
  *   intervalMs?: number,
+ *   keepAlive?: boolean,
  *   logger?: { info?: (...a:any) => void, warn?: (...a:any) => void, error?: (...a:any) => void },
  * }} args
  */
-export function createDreamScheduler({ run, intervalMs = DEFAULT_INTERVAL_MS, logger }) {
+export function createDreamScheduler({ run, intervalMs = DEFAULT_INTERVAL_MS, keepAlive = false, logger }) {
   if (typeof run !== 'function') throw new Error('createDreamScheduler: run callable required');
   const log = logger || {};
   let timer = null;
@@ -60,8 +69,11 @@ export function createDreamScheduler({ run, intervalMs = DEFAULT_INTERVAL_MS, lo
     start() {
       if (timer) return;
       timer = setInterval(() => { fire({ manual: false }).catch(() => {}); }, intervalMs);
-      // Don't keep the event loop alive solely for the dream ticker.
-      if (typeof timer.unref === 'function') timer.unref();
+      // Server / long-lived hosts pass keepAlive=true so the interval
+      // participates in the loop normally. Short-lived hosts (CLI, tests)
+      // leave keepAlive=false to avoid pinning the loop open with the
+      // ticker alone.
+      if (!keepAlive && typeof timer.unref === 'function') timer.unref();
     },
     stop() {
       if (timer) { clearInterval(timer); timer = null; }

package/unify/dream-v2/session-wiring.js

@@ -13,7 +13,8 @@ import { join } from 'path';
 import { runDream } from './runner.js';
 import { createDreamScheduler } from './schedule.js';
 import { listGroups, openGroup } from '../groups/group-store.js';
-import { DREAM_NUDGE_AFTER_MESSAGES } from './limits.js';
+import { readGroupState } from './state.js';
+import { DREAM_NUDGE_AFTER_MESSAGES, DREAM_INTERVAL_HOURS } from './limits.js';
 
 /**
  * Build the per-call options for runDream. Pure: takes a session and returns
@@ -154,6 +155,12 @@ export function createV2DreamScheduler(session) {
 
   const v2 = createDreamScheduler({
     run,
+    // Server mode — the HTTP listener already pins the event loop, so the
+    // ticker should be allowed to participate normally. Without this, on
+    // some Node 22 builds the unref'd interval stops being scheduled and
+    // dream effectively never fires (observed in production: 12 days
+    // between ticks, see fix/dream-cadence-and-ui-trigger).
+    keepAlive: !!session?.config?.serverMode,
     logger: session.config?.debug ? console : undefined,
   });
   // Auto-start the timer.
@@ -192,6 +199,12 @@ export function createV2DreamScheduler(session) {
     noteUserMessage: nudgeOnUserMessage,
     triggerDreamNow() { return v2.triggerNow(); },
     triggerDreamForScopes(scopeFilter) { return v2.triggerNow(scopeFilter); },
+    /**
+     * Non-manual catch-up fire. MIN_NEW_PER_GROUP still applies, so groups
+     * below threshold are skipped — same shape as the interval timer
+     * tick. Used by `bootCatchUpStaleDream`.
+     */
+    catchUpNudge() { return v2.nudge(); },
     shutdown() { v2.stop(); },
     get isRunning() { return v2.isRunning(); },
     // Preserve direct access for tests.
@@ -246,3 +259,105 @@ export async function bootInitEmptyGroups(args) {
   out.triggered = empty;
   return out;
 }
+
+/**
+ * fix/dream-cadence-and-ui-trigger: stale-cadence catch-up.
+ *
+ * Walks every group on disk and reads the per-group `.dream-state`
+ * `lastDreamAt`. The newest of those is the effective "session was last
+ * cleanly Dream-processed at" timestamp. If that's older than
+ * `DREAM_INTERVAL_HOURS` (or there's no record at all and at least one
+ * group has user traffic), schedule a non-manual catch-up tick.
+ *
+ * Why this matters: the only thing previously triggering dream on a
+ * long-lived server was `setInterval(...).unref()` plus the user-traffic
+ * nudge. In practice (production: 12 days idle), neither fired reliably.
+ * This boot-time catch-up gives us a deterministic "if we were stale at
+ * boot, run once" guarantee that's independent of timer behaviour.
+ *
+ * MIN_NEW_PER_GROUP still gates per-group writes inside `runDream` —
+ * non-manual catch-up will not produce empty segment churn for groups
+ * that haven't crossed threshold.
+ *
+ * Pure side-effect, fire-and-forget.
+ *
+ * @param {{
+ *   yeaftDir: string,
+ *   dreamScheduler: { catchUpNudge?: () => Promise<any> },
+ *   intervalHours?: number,
+ *   now?: number,
+ *   config?: { debug?: boolean },
+ * }} args
+ * @returns {Promise<{ stale: boolean, lastDreamAt: string|null, ageMs: number|null, fired: boolean }>}
+ */
+export async function bootCatchUpStaleDream(args) {
+  const out = { stale: false, lastDreamAt: null, ageMs: null, fired: false };
+  if (!args || !args.dreamScheduler) return out;
+
+  const memoryRoot = join(args.yeaftDir, 'memory');
+  const groupsRoot = join(args.yeaftDir, 'groups');
+  const intervalMs = (args.intervalHours ?? DREAM_INTERVAL_HOURS) * 60 * 60 * 1000;
+  const now = args.now ?? Date.now();
+
+  let groupIds;
+  try { groupIds = listGroups(groupsRoot).map(g => g.id); }
+  catch { return out; }
+
+  // Find the newest lastDreamAt across all groups.
+  let newestAt = null;
+  let anyTraffic = false;
+  for (const gid of groupIds) {
+    let st;
+    try { st = await readGroupState(memoryRoot, gid); }
+    catch { continue; }
+    if (st.lastDreamAt) {
+      const t = Date.parse(st.lastDreamAt);
+      if (Number.isFinite(t) && (newestAt === null || t > newestAt)) newestAt = t;
+    }
+    if (!anyTraffic) {
+      try {
+        const h = openGroup(groupsRoot, gid);
+        const first = h.streamMessages().next();
+        if (!first.done) anyTraffic = true;
+      } catch { /* keep going */ }
+    }
+  }
+
+  // No groups, no traffic, nothing to catch up.
+  if (!anyTraffic) return out;
+
+  out.lastDreamAt = newestAt === null ? null : new Date(newestAt).toISOString();
+  out.ageMs = newestAt === null ? null : (now - newestAt);
+  out.stale = (newestAt === null) || (now - newestAt > intervalMs);
+  if (!out.stale) return out;
+
+  if (args.config?.debug) {
+    // eslint-disable-next-line no-console
+    console.log(`[dream-v2] boot catch-up: stale (lastDreamAt=${out.lastDreamAt}, ageMs=${out.ageMs}); firing one non-manual tick.`);
+  }
+
+  // Non-manual: MIN_NEW_PER_GROUP still applies. Use the public
+  // `catchUpNudge()` adapter — same dedupe path as the interval timer
+  // (both go through `fire({ manual: false })`), so a concurrent timer
+  // tick will be coalesced by the existing in-flight guard.
+  //
+  // Fail closed if the scheduler shim doesn't expose `catchUpNudge`:
+  // the previous `triggerDreamForScopes()` fallback routed through
+  // `v2.triggerNow()` which sets `manual: true` and bypasses
+  // MIN_NEW_PER_GROUP — directly contradicting the contract documented
+  // above. Better to skip than to silently fire a different semantic.
+  // PR #743 review feedback (Martin).
+  try {
+    if (typeof args.dreamScheduler.catchUpNudge === 'function') {
+      Promise.resolve(args.dreamScheduler.catchUpNudge()).catch(() => {});
+      out.fired = true;
+    } else {
+      // Shim does not expose the non-manual catch-up path. Refuse to
+      // substitute a manual fire — the caller should upgrade the shim.
+      out.fired = false;
+    }
+  } catch {
+    out.fired = false;
+  }
+  return out;
+}

package/unify/engine.js

@@ -1278,10 +1278,30 @@ export class Engine {
 
     // PR-L: track this query()'s tool-arc for reflection.
     // `turnStartIdx` is where the current user message lives; the arc
-    // we may collapse spans (turnStartIdx + 1 .. last assistant/tool).
+    // we may collapse spans (arcStartIdx .. last assistant/tool).
+    //
+    // Periodic-T1 fix: T1 must fire EVERY TOOL_BATCH_SIZE (13) tool
+    // calls, not just the first 13. So instead of a one-shot boolean,
+    // track:
+    //   • `lastT1AtToolCount` — toolCount snapshot at the last T1
+    //     ATTEMPT (success OR error). Trigger when
+    //     `queryToolCount - lastT1AtToolCount >= TOOL_BATCH_SIZE`.
+    //   • `arcStartIdx` — first index of the current (uncollapsed)
+    //     tool arc. Initialised to turnStartIdx + 1; reset after each
+    //     successful T1 collapse to `conversationMessages.length`
+    //     (i.e. the slot the next assistant message will land in).
+    //   • `t1CollapsesDone` — count of T1 firings that ACTUALLY
+    //     rewrote history. Distinct from `lastT1AtToolCount` because
+    //     the catch block bumps the latter to back off after a
+    //     transient reflector error WITHOUT having collapsed
+    //     anything. The T2 schedule check below is gated on this
+    //     counter (==0 means "no T1 ever rewrote the arc, T2 may
+    //     fall back at end_turn").
     const turnStartIdx = conversationMessages.length - 1;
     let queryToolCount = 0;
-    let t1Fired = false;
+    let lastT1AtToolCount = 0;
+    let arcStartIdx = turnStartIdx + 1;
+    let t1CollapsesDone = 0;
     const queryNumber = (this.#__queryCounter = (this.#__queryCounter || 0) + 1);
 
     // feat-6af5f9f1 PR B: a Turn = one user prompt + all AI responses.
@@ -1784,10 +1804,17 @@ export class Engine {
 
         // PR-L: T2 end-of-turn (asynchronous) reflection. Fires when the
         // total tool count for this query() exceeds TURN_SUMMARY_THRESHOLD
-        // (5) AND T1 didn't already collapse the arc. Kicks off the
+        // (5) AND no T1 has actually rewritten the arc yet. Kicks off the
         // primary-model call without await; the next query()'s
         // `#applyPendingT2Reflections` carries the result forward.
-        if (queryToolCount > TURN_SUMMARY_THRESHOLD && !t1Fired) {
+        //
+        // Periodic-T1 fix: gate on `t1CollapsesDone === 0`, NOT
+        // `lastT1AtToolCount === 0`. The catch block of T1 bumps
+        // `lastT1AtToolCount` after a reflector error to avoid
+        // tight-loop retries — but no collapse happened, so T2 should
+        // still be allowed to fall back at end_turn. Fowler-review
+        // critical finding.
+        if (queryToolCount > TURN_SUMMARY_THRESHOLD && t1CollapsesDone === 0) {
           const arcStart = turnStartIdx + 1;
           const arcEnd = conversationMessages.length - 1;
           if (arcEnd > arcStart) {
@@ -2029,23 +2056,41 @@ export class Engine {
         break;
       }
 
-      // PR-L: T1 in-turn (synchronous) reflection. Fires exactly once per
-      // query() lifetime, the moment queryToolCount crosses
-      // TOOL_BATCH_SIZE (13). Generates a markdown reflection over the
-      // assistant+tool arc since the user prompt and rewrites the history
-      // in place — collapsing it to a SINGLE assistant message — before
-      // the next adapter.stream() runs.
-      if (!t1Fired
-          && queryToolCount >= TOOL_BATCH_SIZE
-          && !this.#reflectedTurns.has(`${queryNumber}:t1`)
-          && !abortedDuringTools && !signal?.aborted) {
-        t1Fired = true;
-        this.#reflectedTurns.add(`${queryNumber}:t1`);
+      // PR-L: T1 in-turn (synchronous) reflection. Fires once per
+      // adapter loop iteration where ≥ TOOL_BATCH_SIZE (13) tool
+      // calls have accumulated since the last T1 firing — not just
+      // the first 13 of the query(). Generates a markdown reflection
+      // over the assistant+tool arc since the last T1 firing (or
+      // since the user prompt for the first batch) and rewrites that
+      // range to a SINGLE synthetic user message before the next
+      // adapter.stream() runs.
+      //
+      // Loop semantics:
+      //   - First batch: arcStartIdx = turnStartIdx + 1, fires when
+      //     queryToolCount reaches 13.
+      //   - Each subsequent batch: arcStartIdx is updated to the slot
+      //     right after the just-inserted reflection message; fires
+      //     again whenever 13 more tools have run since
+      //     lastT1AtToolCount.
+      //   - The dedup Set key includes `lastT1AtToolCount` so each
+      //     batch within the same query gets a distinct entry — without
+      //     this the second batch would be silently skipped.
+      const t1BatchDue = queryToolCount - lastT1AtToolCount >= TOOL_BATCH_SIZE;
+      if (t1BatchDue && !abortedDuringTools && !signal?.aborted) {
+        const t1DedupKey = `${queryNumber}:t1:${queryToolCount}`;
+        if (this.#reflectedTurns.has(t1DedupKey)) {
+          // Defensive: should never hit since t1BatchDue gates re-entry
+          // and queryNumber namespaces queries. Kept as belt-and-
+          // suspenders against any future external mutation of the
+          // cursor (or a re-entrant query() that this code doesn't
+          // anticipate).
+        } else {
+          this.#reflectedTurns.add(t1DedupKey);
+        const batchStart = arcStartIdx;
+        const batchEnd = conversationMessages.length - 1;
         try {
-          const arcStart = turnStartIdx + 1;
-          const arcEnd = conversationMessages.length - 1;
           const { pairs, assistantText } = extractToolPairsFromRange(
-            conversationMessages, arcStart, arcEnd,
+            conversationMessages, batchStart, batchEnd,
           );
           yield {
             type: 'reflection',
@@ -2053,7 +2098,7 @@ export class Engine {
             loopNumber: turnNumber,
             trigger: 't1',
             status: 'pending',
-            loopRange: [arcStart, arcEnd],
+            loopRange: [batchStart, batchEnd],
             toolCount: pairs.length,
           };
           const { content, durationMs } = await runT1Reflection({
@@ -2065,10 +2110,21 @@ export class Engine {
             signal,
           });
           const next = collapseRangeToReflection(
-            conversationMessages, arcStart, arcEnd, content,
+            conversationMessages, batchStart, batchEnd, content,
           );
           conversationMessages.length = 0;
           for (const m of next) conversationMessages.push(m);
+          // After collapse: the just-inserted reflection lives at
+          // index `batchStart`. The next tool arc therefore starts
+          // immediately after it, i.e. at conversationMessages.length
+          // (the next assistant message will land here).
+          arcStartIdx = conversationMessages.length;
+          lastT1AtToolCount = queryToolCount;
+          // Bump the success counter — used by the T2 schedule check
+          // to decide whether T2 still has work to do at end_turn.
+          // Distinct from lastT1AtToolCount which the catch block
+          // also bumps (but without rewriting history).
+          t1CollapsesDone += 1;
           yield {
             type: 'reflection',
             turnId: queryTurnId,
@@ -2078,7 +2134,7 @@ export class Engine {
             // so the frontend key stays stable across pending → ready and
             // the spinner card is replaced in place (no orphan).
             status: 'ready',
-            loopRange: [arcStart, arcEnd],
+            loopRange: [batchStart, batchEnd],
             toolCount: pairs.length,
             content,
             durationMs,
@@ -2094,6 +2150,19 @@ export class Engine {
             status: 'error',
             error: err && err.message || String(err),
           };
+          // Advance lastT1AtToolCount past this batch so we don't
+          // tight-loop on a hiccuping reflector. The next attempt is
+          // 13 tools from now, not immediately. arcStartIdx is left
+          // alone because history wasn't rewritten — the tail still
+          // begins where it did. The trade-off: the next batch's
+          // reflection will cover the tools that just failed too,
+          // which is fine (they're still in conversationMessages).
+          //
+          // We do NOT bump t1CollapsesDone — see the variable's
+          // declaration comment. This keeps the T2 fallback path live
+          // when every T1 attempt has errored.
+          lastT1AtToolCount = queryToolCount;
+        }
         }
       }
 

package/unify/session.js

@@ -43,7 +43,7 @@ import { Engine } from './engine.js';
 import { ensureDefaultGroupIfEmpty } from './groups/group-crud.js';
 import { seedDefaultVps } from './vp/seed-defaults.js';
 import { runSummaryBackfill } from './memory/seed-backfill.js';
-import { createV2DreamScheduler, bootInitEmptyGroups } from './dream-v2/session-wiring.js';
+import { createV2DreamScheduler, bootInitEmptyGroups, bootCatchUpStaleDream } from './dream-v2/session-wiring.js';
 import { openSegmentIndex } from './memory/index-db.js';
 import { syncAll as syncSegmentIndex } from './memory/segment-sync.js';
 import { openAmsRegistry } from './memory/ams-registry.js';
@@ -97,6 +97,7 @@ export async function loadSession(options = {}) {
     skipSkills = false,
     extraTools = [],
     configOverrides = {},
+    serverMode = false,
   } = options;
 
   // ─── 1. Determine yeaftDir + ensure directory structure ──
@@ -119,6 +120,10 @@ export async function loadSession(options = {}) {
 
   // ─── 2. Load config ───────────────────────────────────
   const config = loadConfig(overrides);
+  // fix/dream-cadence-and-ui-trigger: tag config so the dream scheduler
+  // can decide whether to keep its interval timer alive (server) or
+  // unref it (CLI / tests). Non-persisted — set per-session by caller.
+  if (serverMode) config.serverMode = true;
 
   // ─── 2.1 Migration state check (task-334i) ────────────
   //        If the group-chat feature flag is on but migration has not
@@ -328,6 +333,21 @@ export async function loadSession(options = {}) {
     }).catch(() => { /* best-effort boot init */ });
   }
 
+  // fix/dream-cadence-and-ui-trigger: stale-cadence catch-up. If the
+  // newest per-group lastDreamAt across all groups is older than
+  // DREAM_INTERVAL_HOURS (or absent and there's user traffic), fire a
+  // single non-manual tick now. Independent of the interval timer —
+  // necessary because production observed 12 days between scheduled
+  // ticks (the unref'd interval did not fire reliably on long-lived
+  // server processes).
+  if (!config._readOnly) {
+    bootCatchUpStaleDream({
+      yeaftDir,
+      dreamScheduler,
+      config,
+    }).catch(() => { /* best-effort catch-up */ });
+  }
+
   // H2.f.5: thread engine registry, input queue, and dispatcher retired.
   // The session exposes a single `engine`; web-bridge calls engine.query()
   // directly. Memory recall happens via memory/preflow.js (pre-turn) and

package/unify/web-bridge.js

@@ -1665,6 +1665,7 @@ async function ensureSessionLoaded() {
     ...(yeaftDir && { dir: yeaftDir }),
     skipMCP: false,
     skipSkills: false,
+    serverMode: true,
   });
 
   installUnifyRuntimeBridge(session);
@@ -2367,11 +2368,27 @@ export async function handleUnifyDreamTrigger(msg = {}) {
 
     const result = await session.dreamScheduler.triggerDreamNow();
 
+    // fix/dream-cadence-and-ui-trigger: derive a single "entries
+    // created" count for the UI bubble. The runner returns a richer
+    // shape (groups[], targets[]); the front-end button only needs a
+    // scalar plus the start timestamp.
+    const targets = Array.isArray(result?.targets) ? result.targets : [];
+    const entriesCreated = targets.filter(t => t && t.status === 'done').length;
+    const lastDreamAt = result?.startedAt || new Date().toISOString();
+
+    // Spread `result` FIRST so derived fields (success, entriesCreated,
+    // lastDreamAt) authoritatively shadow anything the runner might grow
+    // with the same name. Today there is no collision (runner.js returns
+    // { groups, targets, startedAt, error?, skipped? }) but the failure
+    // mode of the alternative ordering is silent — review feedback from
+    // PR #743.
     sendToServer({
       type: 'unify_dream_result',
       vpId,
-      success: !result.error && !result.skipped,
       ...result,
+      success: !result.error && !result.skipped,
+      entriesCreated,
+      lastDreamAt,
     });
   } catch (err) {
     sendToServer({
@@ -2570,6 +2587,7 @@ export async function handleUnifyLoadHistory(msg) {
       ...(yeaftDir && { dir: yeaftDir }),
       skipMCP: false,
       skipSkills: false,
+      serverMode: true,
     });
     installUnifyRuntimeBridge(session);
 
@@ -2762,6 +2780,7 @@ export async function resetUnifySession() {
       ...(yeaftDir && { dir: yeaftDir }),
       skipMCP: false,
       skipSkills: false,
+      serverMode: true,
     });
     installUnifyRuntimeBridge(session);