@yeaft/webchat-agent 0.1.660 → 0.1.662

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.660",
+  "version": "0.1.662",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/config.js

@@ -315,12 +315,13 @@ export function loadConfig(overrides = {}) {
     // don't pollute the flat config namespace used by chat/crew code.
     unify: normaliseUnifySection(jsonConfig.unify),
 
-    // DESIGN-v2 feature flag. When true the session wires the v2 dream
-    // pipeline (dream-v2/runner.js) and opens the FTS5 SegmentIndex used
-    // by the engine's pre-turn recall (groups/pre-flow.js →
-    // memory/preflow.js). PR-E flipped the default to true; users who
-    // need the legacy R6 paths can opt out via `"memoryV2": false` in
-    // ~/.yeaft/config.json.
+    // DESIGN-v2 feature flag. When true the session opens the FTS5
+    // SegmentIndex (used by groups/pre-flow.js → memory/preflow.js
+    // for pre-turn recall) and wires the v2 dream pipeline
+    // (dream-v2/runner.js). When false both are skipped — no recall,
+    // no dream — turns still work but without memory injection. The
+    // legacy R6 recall + dream-scheduler paths have been deleted, so
+    // `false` is now a "memory off" kill switch rather than a fallback.
     memoryV2: overrides.memoryV2 !== undefined ? !!overrides.memoryV2
       : (jsonConfig.memoryV2 !== undefined ? !!jsonConfig.memoryV2 : true),
 

package/unify/engine.js

@@ -20,7 +20,7 @@
 import { randomUUID } from 'crypto';
 import { buildSystemPrompt, buildWorkerPrompt } from './prompts.js';
 import { LLMContextError, LLMAbortError } from './llm/adapter.js';
-import { runMemoryPreflow } from './groups/pre-flow.js';
+import { runMemoryPreflow, buildRelevantScopes } from './groups/pre-flow.js';
 import { shouldConsolidate, consolidate, partitionMessages } from './memory/consolidate.js';
 import { extractMemories } from './memory/extract.js';
 import { runCompact as runCompactOrchestrator } from './compact/orchestrator.js';
@@ -29,6 +29,7 @@ import { archiveTurn } from './archive/turn-archive.js';
 import { archiveToolResults } from './archive/tool-results.js';
 import { buildMemoryInjection } from './memory/layout.js';
 import { readSummary as readScopeSummary } from './memory/store-v2.js';
+import { runAdjust } from './memory/adjust.js';
 import { runStopHooks } from './stop-hooks.js';
 // H2.f.5: threads/ retired. Persisted messages still carry a `threadId`
 // field for back-compat with old conversation files; new writes always use
@@ -153,6 +154,9 @@ export class Engine {
   /** @type {import('./memory/index-db.js').SegmentIndex|null} — GC.1: SQLite FTS5 segment index */
   #memoryIndex;
 
+  /** @type {import('./memory/ams-registry.js').AmsRegistry|null} — group-keyed AMS cache */
+  #amsRegistry;
+
   /** @type {import('./tools/registry.js').ToolRegistry|null} */
   #toolRegistry;
 
@@ -214,6 +218,14 @@ export class Engine {
   #reflectedTurns = new Set();
   #__queryCounter = 0;
 
+  /**
+   * Per-group "adjust has run at least once this engine lifetime" flag.
+   * Keyed by groupId (or 'default'). The first turn always runs adjust;
+   * subsequent turns only run on budget pressure or new memory.
+   * @type {Map<string, boolean>}
+   */
+  #adjustRanByGroup = new Map();
+
   /** @type {string|null} */
   #abortReason = null;
 
@@ -231,7 +243,7 @@ export class Engine {
    *   yeaftDir?: string,
    * }} params
    */
-  constructor({ adapter, trace, config, conversationStore, memoryStore, memoryShardStore, memoryIndex, toolRegistry, skillManager, mcpManager, yeaftDir }) {
+  constructor({ adapter, trace, config, conversationStore, memoryStore, memoryShardStore, memoryIndex, amsRegistry, toolRegistry, skillManager, mcpManager, yeaftDir }) {
     this.#adapter = adapter;
     this.#trace = trace;
     this.#config = config;
@@ -241,6 +253,7 @@ export class Engine {
     this.#memoryStore = memoryStore || null;
     this.#memoryShardStore = memoryShardStore || null;
     this.#memoryIndex = memoryIndex || null;
+    this.#amsRegistry = amsRegistry || null;
     this.#toolRegistry = toolRegistry || null;
     this.#skillManager = skillManager || null;
     this.#mcpManager = mcpManager || null;
@@ -378,6 +391,161 @@ export class Engine {
     return { user: user || '', group: group || '', vp: vp || '' };
   }
 
+  /**
+   * Prepare the per-turn AMS for the active group. Idempotent and safe
+   * to call when the AMS registry isn't wired (returns null).
+   *
+   * @param {{
+   *   groupId?: string,
+   *   ownVpId?: string|null,
+   *   featureId?: string,
+   *   summaries: { user?: string, group?: string, vp?: string },
+   *   recallEntries: object[],
+   * }} args
+   * @returns {{
+   *   ams: import('./memory/ams.js').ActiveMemorySet,
+   *   groupKey: string,
+   *   ownVpId: string|null,
+   *   scopes: string[],
+   *   snapshotBlock: string,
+   * } | null}
+   */
+  #prepareAms(args) {
+    if (!this.#amsRegistry) return null;
+    const groupKey = args.groupId || 'default';
+    const ownVpId = args.ownVpId || null;
+    const ams = this.#amsRegistry.getOrCreate(groupKey, { ownVpId });
+
+    // (a) Resident: rebuild from the same scope summaries the worker
+    // prompt is already going to see.
+    const residentEntries = [];
+    if (args.summaries?.user)  residentEntries.push({ scope: 'user', summary: args.summaries.user });
+    if (args.groupId && args.summaries?.group) {
+      residentEntries.push({ scope: `group/${args.groupId}`, summary: args.summaries.group });
+    }
+    if (ownVpId && args.summaries?.vp) {
+      residentEntries.push({ scope: `vp/${ownVpId}`, summary: args.summaries.vp });
+    }
+    ams.setResident(residentEntries);
+
+    // (b) onDemand: replace with this turn's FTS hits.
+    const segs = Array.isArray(args.recallEntries) ? args.recallEntries : [];
+    ams.setOnDemand(segs);
+
+    // (c) Snapshot — render the AMS layers as a single prompt block.
+    const snapshotBlock = this.#renderAmsSnapshot(ams);
+
+    const scopes = buildRelevantScopes({
+      groupId: args.groupId,
+      vpId: ownVpId,
+      featureId: args.featureId,
+    });
+
+    return { ams, groupKey, ownVpId, scopes, snapshotBlock };
+  }
+
+  /**
+   * Render an AMS snapshot as a markdown block suitable for prompt
+   * injection. Mirrors the heading style of the existing memory blocks
+   * so the LLM sees a consistent layout.
+   *
+   * @param {import('./memory/ams.js').ActiveMemorySet} ams
+   * @returns {string}
+   */
+  #renderAmsSnapshot(ams) {
+    const snap = ams.snapshot();
+    if (!snap) return '';
+    const parts = [];
+    if (snap.resident.length === 0 && snap.recent.length === 0 && snap.onDemand.length === 0) {
+      return '';
+    }
+    parts.push('## Active Memory Set');
+    if (snap.resident.length > 0) {
+      parts.push('### Resident');
+      for (const r of snap.resident) {
+        parts.push(`- **${r.scope}**: ${r.summary}`);
+      }
+    }
+    if (snap.recent.length > 0) {
+      parts.push('### Recent');
+      for (const s of snap.recent) {
+        parts.push(`- (${s.scope}) ${(s.body || '').trim()}`);
+      }
+    }
+    if (snap.onDemand.length > 0) {
+      parts.push('### OnDemand');
+      for (const s of snap.onDemand) {
+        parts.push(`- (${s.scope}) ${(s.body || '').trim()}`);
+      }
+    }
+    return parts.join('\n');
+  }
+
+  /**
+   * Post-turn AMS correction. Decides whether to run via
+   * `shouldRunAdjust`, then drives the LLM round-trip through
+   * `runAdjust`. Persists the AMS to disk if membership changed.
+   *
+   * Failure here is intentionally swallowed — adjust is a best-effort
+   * memory-quality step; a parse failure or LLM blip should never
+   * surface as a turn failure.
+   *
+   * @param {{
+   *   amsContext: { ams: import('./memory/ams.js').ActiveMemorySet, groupKey: string, ownVpId: string|null, scopes: string[] }|null,
+   *   userMsg: string,
+   *   assistantReply: string,
+   *   turnTokenUsage: number,
+   * }} args
+   * @returns {Promise<{ ran: boolean, added: number, evicted: number, reason: string } | null>}
+   */
+  async #runAdjustHook(args) {
+    const ctx = args.amsContext;
+    if (!ctx || !this.#amsRegistry || !this.#memoryIndex) return null;
+    const totalBudget = ctx.ams.budget?.total || 0;
+    if (!totalBudget) return null;
+
+    const adjustRanThisSession = this.#adjustRanByGroup.get(ctx.groupKey) === true;
+    try {
+      const result = await runAdjust({
+        trigger: {
+          newMemoryWritten: false,        // dream writes happen async; treat as false here
+          onDemandSize: ctx.ams.onDemandIds().length,
+          turnTokenUsage: args.turnTokenUsage,
+          totalBudget,
+          adjustRanThisSession,
+        },
+        ams: ctx.ams,
+        index: this.#memoryIndex,
+        scopes: ctx.scopes,
+        ownVpId: ctx.ownVpId,
+        userMsg: args.userMsg,
+        assistantReply: args.assistantReply,
+        runLLM: async (prompt) => {
+          const out = await this.#adapter.call({
+            model: this.#fastConfig.model,
+            system: 'You are a memory-management subroutine. Reply with a single JSON object as instructed.',
+            messages: [{ role: 'user', content: prompt }],
+            maxTokens: 1024,
+          });
+          return out?.text || '';
+        },
+      });
+      if (result?.ran) {
+        this.#adjustRanByGroup.set(ctx.groupKey, true);
+        // Always persist when we ran — even with no membership change,
+        // the adjustRanThisSession bit is part of the on-disk state we
+        // want to preserve.
+        this.#amsRegistry.markDirty(ctx.groupKey);
+        this.#amsRegistry.persist(ctx.groupKey, {
+          adjustRanThisSession: true,
+        });
+      }
+      return result;
+    } catch {
+      return null;
+    }
+  }
+
   /**
    * Build the system prompt with memory, compact summary, skill content,
    * and (Phase 8 wire-up) Layer-A scope summaries.
@@ -895,6 +1063,34 @@ export class Engine {
         : (typeof senderVpId === 'string' ? senderVpId : undefined),
     });
 
+    // ─── AMS: populate + snapshot ───────────────────────────────
+    // Group-keyed and persisted across session deactivation. Each turn:
+    //   (a) resident layer is rebuilt from <scope>/summary.md (the
+    //       summaries already loaded above are the same scopes, so
+    //       reuse them);
+    //   (b) onDemand is replaced with this turn's FTS hits;
+    //   (c) we render a budget-aware snapshot block and append it to
+    //       memoryInjection. Adjust runs post-turn (see end_turn below).
+    const ownVpIdForAms = vpPersona && typeof vpPersona === 'object'
+      && typeof vpPersona.vpId === 'string'
+      ? vpPersona.vpId
+      : (typeof senderVpId === 'string' ? senderVpId : null);
+    const featureIdForAms = typeof inboundEnvelope === 'object' && inboundEnvelope
+      ? inboundEnvelope.featureId
+      : undefined;
+    const amsContext = this.#prepareAms({
+      groupId,
+      ownVpId: ownVpIdForAms,
+      featureId: featureIdForAms,
+      summaries,
+      recallEntries: recallResult ? (recallResult.entries || []) : [],
+    });
+    if (amsContext && amsContext.snapshotBlock) {
+      memoryInjection = memoryInjection
+        ? memoryInjection + '\n\n' + amsContext.snapshotBlock
+        : amsContext.snapshotBlock;
+    }
+
     const systemPrompt = this.#buildSystemPrompt(undefined, compactSummary, prompt, memoryInjection, userProfile, vpPersona, summaries);
 
     // Build conversation: existing messages + new user message
@@ -925,6 +1121,8 @@ export class Engine {
     let toolLoopTurns = 0; // task-327b: tool-use turns for long-loop auto-bump
     let fullResponseText = '';
     let currentModel = this.#config.model;
+    let cumulativeInputTokens = 0;
+    let cumulativeOutputTokens = 0;
 
     while (true) {
       turnNumber++;
@@ -1061,6 +1259,8 @@ export class Engine {
             case 'usage':
               totalUsage.inputTokens += event.inputTokens;
               totalUsage.outputTokens += event.outputTokens;
+              cumulativeInputTokens += event.inputTokens || 0;
+              cumulativeOutputTokens += event.outputTokens || 0;
               yield event;
               break;
             case 'stop':
@@ -1269,6 +1469,27 @@ export class Engine {
           }
         }
 
+        // ─── Post-turn AMS adjust ────────────────────────────────
+        // shouldRunAdjust gates the LLM round-trip so most turns are
+        // free; first turn always runs, plus on budget pressure.
+        if (amsContext) {
+          const adjustResult = await this.#runAdjustHook({
+            amsContext,
+            userMsg: prompt,
+            assistantReply: fullResponseText,
+            turnTokenUsage: cumulativeInputTokens + cumulativeOutputTokens,
+          });
+          if (adjustResult && adjustResult.ran) {
+            yield {
+              type: 'ams_adjust',
+              groupKey: amsContext.groupKey,
+              added: adjustResult.added,
+              evicted: adjustResult.evicted,
+              reason: adjustResult.reason,
+            };
+          }
+        }
+
         // PR-L: T2 end-of-turn (asynchronous) reflection. Fires when the
         // total tool count for this query() exceeds TURN_SUMMARY_THRESHOLD
         // (5) AND T1 didn't already collapse the arc. Kicks off the

package/unify/groups/pre-flow.js

@@ -158,9 +158,8 @@ export function selectRespondingVps(input) {
 /**
  * Build the heading for a single scope's formatted memory block.
  *
- * Mirrors recall-v2's formatRecallV2 heading style so the system
- * prompt looks the same to the LLM whether recall came from FTS
- * (here) or from per-scope file reads (recall-v2).
+ * Heading style is the original recall-v2 format, kept so the system
+ * prompt the LLM sees stays stable across the FTS migration.
  *
  * @param {string} scope
  * @returns {string}

package/unify/memory/ams-registry.js

@@ -0,0 +1,233 @@
+/**
+ * memory/ams-registry.js — group-keyed AMS lifecycle.
+ *
+ * The Active Memory Set is conceptually session-scoped, but Yeaft's
+ * unit of "session" is a group: a deactivated group can be reactivated
+ * later and should resume with the AMS state it had on disconnect (the
+ * onDemand segments it had pulled in, the recent LRU touches, whether
+ * `adjust` already ran). Sessions come and go; the group's AMS persists.
+ *
+ * Persistence is identity-only:
+ *
+ *   ~/.yeaft/memory/groups/<groupId>/ams.json
+ *   {
+ *     "version": 1,
+ *     "ownVpId": "alice"|null,
+ *     "onDemandIds": ["seg_..."],
+ *     "recentIds":   ["seg_..."],
+ *     "adjustRanThisSession": true|false,
+ *     "savedAt": "2026-04-29T..."
+ *   }
+ *
+ * Bodies are NOT serialised — they're re-hydrated from the SegmentIndex
+ * on load, so a body edited by Dream after save still surfaces correctly
+ * the next time the group is opened. Resident layer is derived state
+ * (rebuilt every turn from `<scope>/summary.md`) — never persisted.
+ *
+ * For the single-VP Unify path (no group), the registry uses the literal
+ * key `"default"` so there's still a stable home for AMS state.
+ */
+
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+
+import { ActiveMemorySet } from './ams.js';
+import { computeBudget } from './budget.js';
+
+export const AMS_FILE_VERSION = 1;
+export const DEFAULT_GROUP_KEY = 'default';
+
+/**
+ * @typedef {object} AmsRegistryDeps
+ * @property {string}                                    yeaftDir
+ * @property {import('./index-db.js').SegmentIndex|null} memoryIndex
+ * @property {object}                                    config
+ */
+
+/**
+ * @typedef {object} AmsCacheEntry
+ * @property {ActiveMemorySet} ams
+ * @property {string|null}     ownVpId
+ */
+
+/**
+ * Group-keyed in-memory cache + disk persistence for AMS instances.
+ *
+ * Lifecycle:
+ *   - getOrCreate(groupId, {ownVpId}) — returns the cached AMS or loads
+ *     from disk; falls through to a fresh empty AMS on cold start.
+ *   - persist(groupId) — writes the current cached AMS to disk.
+ *   - persistAll() — convenience for shutdown.
+ *
+ * The registry is intentionally narrow: it does not mutate the AMS
+ * itself (that's the engine's job). It only caches, loads, and saves.
+ */
+export class AmsRegistry {
+  /** @param {AmsRegistryDeps} deps */
+  constructor(deps) {
+    this.yeaftDir = deps.yeaftDir;
+    this.memoryIndex = deps.memoryIndex || null;
+    this.config = deps.config || {};
+    /** @type {Map<string, AmsCacheEntry>} */
+    this._cache = new Map();
+    /** @type {Set<string>} */
+    this._dirty = new Set();
+  }
+
+  /**
+   * Resolve the on-disk path for a group's ams.json.
+   *
+   * @param {string} groupId
+   * @returns {string}
+   */
+  amsPath(groupId) {
+    const safe = String(groupId || DEFAULT_GROUP_KEY).replace(/[^A-Za-z0-9._-]/g, '_');
+    return join(this.yeaftDir, 'memory', 'groups', safe, 'ams.json');
+  }
+
+  /**
+   * Compute the BudgetSplit for this session/model.
+   *
+   * @returns {import('./budget.js').BudgetSplit}
+   */
+  _budget() {
+    const ctx = Number.isFinite(this.config?.maxContextTokens)
+      ? this.config.maxContextTokens
+      : 200_000;
+    return computeBudget(ctx);
+  }
+
+  /**
+   * Get the AMS for a group, creating it on first access.
+   * Loads persisted state from disk if any; on cold start returns an
+   * empty AMS keyed to the supplied ownVpId.
+   *
+   * @param {string|null|undefined} groupId
+   * @param {{ ownVpId?: string|null }} [opts]
+   * @returns {ActiveMemorySet}
+   */
+  getOrCreate(groupId, opts = {}) {
+    const key = groupId || DEFAULT_GROUP_KEY;
+    const cached = this._cache.get(key);
+    if (cached) return cached.ams;
+
+    const ownVpId = opts.ownVpId || null;
+    const budget = this._budget();
+    const ams = new ActiveMemorySet({ ownVpId, budget });
+    // Best-effort hydrate from disk.
+    this._hydrate(key, ams);
+    this._cache.set(key, { ams, ownVpId });
+    return ams;
+  }
+
+  /**
+   * Mark a group's AMS as dirty so the next persist() actually writes.
+   * The engine calls this after `runAdjust` mutates membership.
+   *
+   * @param {string|null|undefined} groupId
+   */
+  markDirty(groupId) {
+    this._dirty.add(groupId || DEFAULT_GROUP_KEY);
+  }
+
+  /**
+   * Persist a single group's AMS to disk. No-op when the cached entry
+   * is missing or hasn't been marked dirty.
+   *
+   * @param {string|null|undefined} groupId
+   * @param {{ force?: boolean, adjustRanThisSession?: boolean }} [opts]
+   * @returns {boolean} true if the file was written
+   */
+  persist(groupId, opts = {}) {
+    const key = groupId || DEFAULT_GROUP_KEY;
+    const entry = this._cache.get(key);
+    if (!entry) return false;
+    if (!opts.force && !this._dirty.has(key)) return false;
+
+    const path = this.amsPath(key);
+    const payload = {
+      version: AMS_FILE_VERSION,
+      ownVpId: entry.ownVpId,
+      onDemandIds: entry.ams.onDemandIds(),
+      recentIds: entry.ams.recentIds(),
+      adjustRanThisSession: Boolean(opts.adjustRanThisSession),
+      savedAt: new Date().toISOString(),
+    };
+
+    try {
+      mkdirSync(dirname(path), { recursive: true });
+      const tmp = `${path}.tmp`;
+      writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8');
+      renameSync(tmp, path);
+      this._dirty.delete(key);
+      return true;
+    } catch {
+      // Persistence failure is non-fatal — AMS continues to live in memory.
+      return false;
+    }
+  }
+
+  /**
+   * Persist every cached, dirty AMS. Called on session shutdown.
+   *
+   * @returns {number} number of files written
+   */
+  persistAll() {
+    let n = 0;
+    for (const key of this._dirty) {
+      if (this.persist(key, { force: true })) n += 1;
+    }
+    return n;
+  }
+
+  /**
+   * Best-effort hydrate: read ams.json, re-resolve segment ids via the
+   * SegmentIndex (skipping ids that no longer exist), populate AMS.
+   * Silent on every error — a corrupt or missing file is the cold-start
+   * case, indistinguishable from "first use of this group".
+   *
+   * @private
+   * @param {string} key
+   * @param {ActiveMemorySet} ams
+   */
+  _hydrate(key, ams) {
+    const path = this.amsPath(key);
+    if (!existsSync(path)) return;
+    let payload;
+    try { payload = JSON.parse(readFileSync(path, 'utf8') || '{}'); }
+    catch { return; }
+    if (!payload || typeof payload !== 'object') return;
+
+    if (!this.memoryIndex) return;
+
+    const onDemandIds = Array.isArray(payload.onDemandIds) ? payload.onDemandIds : [];
+    const recentIds   = Array.isArray(payload.recentIds)   ? payload.recentIds   : [];
+
+    const onDemandSegs = [];
+    for (const id of onDemandIds) {
+      try {
+        const seg = this.memoryIndex.get(id);
+        if (seg) onDemandSegs.push(seg);
+      } catch { /* skip unresolvable */ }
+    }
+    if (onDemandSegs.length > 0) ams.setOnDemand(onDemandSegs);
+
+    for (const id of recentIds) {
+      try {
+        const seg = this.memoryIndex.get(id);
+        if (seg) ams.touchRecent(seg);
+      } catch { /* skip */ }
+    }
+  }
+}
+
+/**
+ * Factory for the registry. Kept as a thin function so call sites can
+ * stay symmetrical with the other store openers in session.js.
+ *
+ * @param {AmsRegistryDeps} deps
+ * @returns {AmsRegistry}
+ */
+export function openAmsRegistry(deps) {
+  return new AmsRegistry(deps);
+}

package/unify/memory/keywords.js

@@ -1,9 +1,8 @@
 /**
  * keywords.js — pure-rule keyword extraction shared by memory recall paths.
  *
- * Extracted from the legacy R5 recall.js so recall-v2.js (and any future
- * recall path) can use it without dragging in the rest of the R5 module.
- * Pure CPU, no LLM, <1ms.
+ * Pure CPU, no LLM, <1ms. Used by `groups/pre-flow.js` to derive FTS
+ * query terms from the user message before hitting `memory/preflow.js`.
  */
 
 /** Common stop words filtered out before frequency counting. */