@yeaft/webchat-agent 1.0.440 → 1.0.442

package/yeaft/dream/segment.js

@@ -1,7 +1,7 @@
 /**
  * dream/segment.js.
  *
- * Three independent length-control concerns, kept pure so they can be
+ * Four independent length-control concerns, kept pure so they can be
  * unit-tested without touching disk or any LLM:
  *
  *   1. truncateMessage — clamp a single message body to
@@ -21,15 +21,21 @@
  *
  *   4. needsBatchedApply / batchSourcesForApply — when an Apply target's
  *      memory + summary + sources cumulatively exceed MAX_APPLY_TOKENS,
- *      split the sources (one source = one group's contribution) into
- *      batches; the LLM is then called once per batch, threading the
- *      written-back memory.md as input to the next batch. (§17.2)
+ *      split source messages into bounded batches; the LLM is then called
+ *      once per batch, threading the written-back memory.md as input to the
+ *      next batch. A single Session source may therefore produce multiple
+ *      ordered batches.
+ *
+ * Dream only needs durable user/assistant prose. Tool results are execution
+ * history and can be enormous; `compressDreamMessages()` drops them before
+ * triage/apply while preserving the original conversation on disk.
  *
  * No side-effects. All functions are deterministic given their inputs.
  */
 
 import {
   MAX_SINGLE_MESSAGE_CHARS,
+  MAX_DREAM_PROMPT_CHARS,
   MAX_DIFF_TOKENS_PER_TRIAGE,
   MAX_APPLY_TOKENS,
   DREAM_OVERLAP,
@@ -80,6 +86,67 @@ export function estimateMessagesTokens(msgs) {
   return n;
 }
 
+/**
+ * Remove execution-only messages from the Dream input. The transcript remains
+ * the source of truth; this is only a prompt projection. Overlap messages are
+ * retained for triage context but never become new durable evidence.
+ *
+ * @param {Array<object>} messages
+ * @returns {Array<object>}
+ */
+export function compressDreamMessages(messages) {
+  if (!Array.isArray(messages)) return [];
+  return messages
+    .filter(message => message && typeof message === 'object')
+    .filter(message => ['user', 'assistant'].includes(String(message.role || '').toLowerCase()))
+    .map(message => ({
+      ...message,
+      body: truncateMessage(message.body || message.content || ''),
+    }))
+    .filter(message => String(message.body || '').trim());
+}
+
+/**
+ * Keep only newly observed messages when applying/extracting. Triage may use
+ * overlap context, but re-feeding it to Apply causes old Dreamed content to be
+ * processed again on every pass.
+ *
+ * @param {Array<object>} messages
+ * @returns {Array<object>}
+ */
+export function selectDreamNewMessages(messages) {
+  return (Array.isArray(messages) ? messages : [])
+    .filter(message => message && message.kind !== 'overlap');
+}
+
+/**
+ * Hard cap the final Dream prompt at the provider boundary. Keep both the
+ * prompt contract (head) and the JSON/output instruction (tail); discard only
+ * the middle source transcript. The durable transcript and canonical memory
+ * remain on disk.
+ *
+ * @param {string} prompt
+ * @param {number} [maxChars=MAX_DREAM_PROMPT_CHARS]
+ * @returns {string}
+ */
+export function boundDreamPrompt(prompt, maxChars = MAX_DREAM_PROMPT_CHARS) {
+  const text = String(prompt || '');
+  const cap = Number.isFinite(maxChars) && maxChars > 0
+    ? Math.floor(maxChars)
+    : MAX_DREAM_PROMPT_CHARS;
+  if (text.length <= cap) return text;
+  const marker = '\n\n[Dream prompt compressed: middle transcript omitted; durable source remains on disk]\n\n';
+  if (cap <= marker.length) {
+    const head = Math.ceil(cap / 2);
+    const tail = cap - head;
+    return text.slice(0, head) + (tail > 0 ? text.slice(-tail) : '');
+  }
+  const room = cap - marker.length;
+  const head = Math.ceil(room * 0.62);
+  const tail = room - head;
+  return text.slice(0, head) + marker + (tail > 0 ? text.slice(-tail) : '');
+}
+
 /**
  * Split a contiguous group diff into ≤MAX-token segments, with a
  * DREAM_OVERLAP-message tail/head overlap between consecutive segments.
@@ -99,14 +166,19 @@ export function estimateMessagesTokens(msgs) {
  * @param {Array<{id?: string, role?: string, body?: string}>} diff
  * @param {number} [maxTokens=MAX_DIFF_TOKENS_PER_TRIAGE]
  * @param {number} [overlap=DREAM_OVERLAP]
+ * @param {number} [maxPromptChars=MAX_DREAM_PROMPT_CHARS]
  * @returns {Array<{ messages: Array<object>, overlapCount: number, newCount: number }>}
  */
-export function segmentDiff(diff, maxTokens = MAX_DIFF_TOKENS_PER_TRIAGE, overlap = DREAM_OVERLAP) {
+export function segmentDiff(diff, maxTokens = MAX_DIFF_TOKENS_PER_TRIAGE, overlap = DREAM_OVERLAP, maxPromptChars = MAX_DREAM_PROMPT_CHARS) {
   const msgs = Array.isArray(diff) ? diff : [];
+  const boundedPromptChars = Number.isFinite(maxPromptChars) && maxPromptChars > 0
+    ? maxPromptChars
+    : MAX_DREAM_PROMPT_CHARS;
+  const boundedMaxTokens = Math.min(maxTokens, Math.max(1, Math.floor(boundedPromptChars / 4) - 2048));
   if (msgs.length === 0) return [];
 
   // Fast path: whole diff fits in one segment.
-  if (estimateMessagesTokens(msgs) <= maxTokens) {
+  if (estimateMessagesTokens(msgs) <= boundedMaxTokens) {
     return [{ messages: msgs, overlapCount: 0, newCount: msgs.length }];
   }
 
@@ -120,7 +192,7 @@ export function segmentDiff(diff, maxTokens = MAX_DIFF_TOKENS_PER_TRIAGE, overla
     let end = cursor;
     while (end < msgs.length) {
       const cost = estimateTokens(msgs[end].body || '') + estimateTokens(msgs[end].role || '') + 2;
-      if (used + cost > maxTokens && end > cursor) break;
+      if (used + cost > boundedMaxTokens && end > cursor) break;
       used += cost;
       end += 1;
     }
@@ -161,9 +233,9 @@ function totalApplyTokens(merged) {
  * previous-batch output replaces memoryMd, so we account for the same
  * baseline cost in each batch.
  *
- * If a single source (one group's diff) alone would overflow, it still
- * goes into its own batch — we never split a source diff here (segment
- * happens earlier, in triage).
+ * Sources are split into ordered message chunks when one Session's diff is
+ * larger than the apply budget. This is required because a single Session
+ * can contribute thousands of messages after a long gap between Dream runs.
  *
  * @param {{ memoryMd?: string, summaryMd?: string, sources: Array<{ sessionId: string, diff: any }> }} merged
  * @param {number} [maxTokens=MAX_APPLY_TOKENS]
@@ -173,10 +245,11 @@ export function batchSourcesForApply(merged, maxTokens = MAX_APPLY_TOKENS) {
   const sources = Array.isArray(merged.sources) ? merged.sources : [];
   if (sources.length === 0) return [];
   const baseline = estimateTokens(merged.memoryMd || '') + estimateTokens(merged.summaryMd || '');
+  const sourceChunks = sources.flatMap(source => splitApplySource(source, Math.max(1, maxTokens - baseline)));
   const batches = [];
   let cur = [];
   let used = baseline;
-  for (const src of sources) {
+  for (const src of sourceChunks) {
     const cost = estimateMessagesTokens(src.diff || []);
     if (cur.length > 0 && used + cost > maxTokens) {
       batches.push(cur);
@@ -189,3 +262,23 @@ export function batchSourcesForApply(merged, maxTokens = MAX_APPLY_TOKENS) {
   if (cur.length > 0) batches.push(cur);
   return batches;
 }
+
+function splitApplySource(source, budget) {
+  const messages = Array.isArray(source?.diff) ? source.diff : [];
+  if (messages.length === 0) return [{ ...source, diff: [] }];
+  const chunks = [];
+  let current = [];
+  let used = 0;
+  for (const message of messages) {
+    const cost = estimateMessagesTokens([message]);
+    if (current.length > 0 && used + cost > budget) {
+      chunks.push({ ...source, diff: current });
+      current = [];
+      used = 0;
+    }
+    current.push(message);
+    used += cost;
+  }
+  if (current.length > 0) chunks.push({ ...source, diff: current });
+  return chunks;
+}

package/yeaft/dream/session-wiring.js

@@ -46,7 +46,8 @@ import { parseMessage, parseSeqFromId } from '../conversation/persist.js';
 import { loadSessionConfig, resolveSessionConfig } from '../sessions/session-config.js';
 import { listSessions as listSessionMetas } from '../sessions/session-store.js';
 import { readSessionState } from './state.js';
-import { DREAM_NUDGE_AFTER_MESSAGES, DREAM_INTERVAL_HOURS } from './limits.js';
+import { boundDreamPrompt } from './segment.js';
+import { DREAM_NUDGE_AFTER_MESSAGES, DREAM_INTERVAL_HOURS, loadLimitsFromConfig } from './limits.js';
 
 /**
  * Build the per-call options for runDream. Pure: takes a session and returns
@@ -83,6 +84,7 @@ export function buildRunDreamOpts(session, onProgress) {
     root: memoryRoot,
     language: session.config?.language || 'en',
     segmentIndex: session.memoryIndex || null,
+    limits: loadLimitsFromConfig(session.config),
     llm: makeLlm(session),
     listSessions: async () => {
       try { return listRegisteredSessions([sessionConversationsRoot, legacySessionConversationsRoot]); }
@@ -355,7 +357,11 @@ function makeLlm(session) {
   return async ({ pass, prompt, system, sessionId }) => {
     const adapter = session.adapter;
     const effectiveConfig = resolveDreamSessionConfig(session, sessionId);
-    const model = effectiveConfig?.model || effectiveConfig?.primaryModel;
+    // Session model overrides may be stored provider-qualified while the
+    // resolved config also exposes a provider-local `model` id. Dream must
+    // route through the exact Session selection first; otherwise a short id
+    // can resolve to another provider or fail as unsupported.
+    const model = effectiveConfig?.primaryModel || effectiveConfig?.model;
     if (!model) {
       throw new Error(`dream: no session model configured (pass=${pass}, sessionId=${sessionId || 'unknown'})`);
     }
@@ -371,11 +377,13 @@ function makeLlm(session) {
     const effectiveSystem = system || (String(effectiveConfig?.language || '').toLowerCase().startsWith('zh')
       ? `你是梦境流水线 — pass: ${pass}。请用中文生成自然语言内容；JSON key 保持英文。`
       : `You are the dream pipeline — pass: ${pass}.`);
+    const dreamLimits = loadLimitsFromConfig(effectiveConfig);
+    const boundedPrompt = boundDreamPrompt(prompt, dreamLimits.MAX_DREAM_PROMPT_CHARS);
 
     const r = await adapter.call({
       model,
       system: effectiveSystem,
-      messages: [{ role: 'user', content: prompt }],
+      messages: [{ role: 'user', content: boundedPrompt }],
       maxTokens: 2048,
       modelEffort: effectiveConfig?.modelEffort || undefined,
     });
@@ -402,7 +410,8 @@ function makeLlm(session) {
       model: model || 'unknown',
       sessionId: sessionId || null,
       systemPrompt: effectiveSystem,
-      messages: [{ role: 'user', content: prompt }],
+      messages: [{ role: 'user', content: boundedPrompt }],
+      promptChars: boundedPrompt.length,
       response: typeof r?.text === 'string' ? r.text : '',
       toolCalls: [],
       usage,

package/yeaft/dream/topic-consolidation.js

@@ -18,6 +18,7 @@ import {
   TOPIC_REDIRECT_FILE,
 } from '../memory/topic-redirect.js';
 import { render } from './prompts/index.js';
+import { boundDreamPrompt } from './segment.js';
 import { parseJsonSafe } from './triage.js';
 import { snapshotScope } from './snapshot.js';
 
@@ -54,9 +55,9 @@ export async function consolidateSessionTopics(opts) {
     const parsed = parseJsonSafe(await opts.llm({
       pass: 'topic-consolidation',
       system: topicConsolidationSystem(opts.language),
-      prompt: render('consolidateTopics', {
+      prompt: boundDreamPrompt(render('consolidateTopics', {
         topics: batch.map(topic => `- ${topic.path}: ${topic.summary}`).join('\n'),
-      }, { language: opts.language }),
+      }, { language: opts.language }), opts.maxPromptChars),
     }));
     if (!Array.isArray(parsed?.groups)) continue;
     groups.push(...validateGroups(parsed.groups, topics));
@@ -128,13 +129,13 @@ async function applyConsolidationGroup(group, opts) {
   const raw = await opts.llm({
     pass: 'topic-merge',
     system: topicConsolidationSystem(opts.language),
-    prompt: render('mergeTopics', {
+    prompt: boundDreamPrompt(render('mergeTopics', {
       canonical: canonical.path,
       topicContents: available.map(record => [
         `## ${record.path}`,
         record.content || record.memory || record.summary,
       ].join('\n')).join('\n\n'),
-    }, { language: opts.language }),
+    }, { language: opts.language }), opts.maxPromptChars),
   });
   const parsed = parseJsonSafe(raw);
   const mergedContent = String(parsed?.content_md || '').trim();

package/yeaft/dream/triage.js

@@ -36,7 +36,7 @@ import { isValidTopic } from '../memory/store.js';
  * below.)
  */
 
-import { truncateMessage } from './segment.js';
+import { boundDreamPrompt, truncateMessage } from './segment.js';
 import { render } from './prompts/index.js';
 import { resolveTopicRedirect } from '../memory/topic-redirect.js';
 
@@ -106,7 +106,7 @@ export function applyHardRules({ sessionId, chatId, messages }) {
 /**
  * Build the prompt used for Pass-1.
  *
- * @param {{ sessionId: string, messages: Array<object>, topicSummaries: Array<{ path: string, summary: string }> }} ctx
+ * @param {{ sessionId: string, messages: Array<object>, topicSummaries: Array<{ path: string, summary: string }>, maxPromptChars?: number }} ctx
  */
 export function buildPass1Prompt(ctx) {
   const topicSummaries = (!ctx.topicSummaries || ctx.topicSummaries.length === 0)
@@ -119,11 +119,11 @@ export function buildPass1Prompt(ctx) {
     conv.push(truncateMessage(m.body || ''));
     conv.push('');
   }
-  return render('triagePass1', {
+  return boundDreamPrompt(render('triagePass1', {
     sessionId: ctx.sessionId,
     topicSummaries,
     conversation: conv.join('\n').trimEnd(),
-  }, { language: ctx.language });
+  }, { language: ctx.language }), ctx.maxPromptChars);
 }
 
 /**
@@ -135,10 +135,10 @@ export function buildPass2Prompt(ctx) {
   const existingTopics = (!ctx.existingTopics || ctx.existingTopics.length === 0)
     ? (String(ctx.language || '').toLowerCase().startsWith('zh') ? '  （无）' : '  (none)')
     : ctx.existingTopics.map(t => `  - ${t.path} — ${oneLine(t.summary)}`).join('\n');
-  return render('triagePass2', {
+  return boundDreamPrompt(render('triagePass2', {
     description: ctx.description,
     existingTopics,
-  }, { language: ctx.language });
+  }, { language: ctx.language }), ctx.maxPromptChars);
 }
 
 /**
@@ -150,12 +150,13 @@ export function buildPass2Prompt(ctx) {
  *   messages: Array<object>,
  *   topicSummaries: Array<{ path: string, summary: string }>,
  *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ *   maxPromptChars?: number,
  * }} args
  * @returns {Promise<Array<{ kind: 'update'|'create', scope: string }>>}
  */
-export async function classifySoft({ root, sessionId, messages, topicSummaries, llm, language }) {
+export async function classifySoft({ root, sessionId, messages, topicSummaries, llm, language, maxPromptChars }) {
   if (!llm) throw new Error('triage.classifySoft: llm callable required');
-  const pass1Prompt = buildPass1Prompt({ sessionId, messages, topicSummaries, language });
+  const pass1Prompt = buildPass1Prompt({ sessionId, messages, topicSummaries, language, maxPromptChars });
   const pass1Raw = await llm({ pass: 'triage-pass1', prompt: pass1Prompt, system: triageSystem(language) });
   const pass1 = parseJsonSafe(pass1Raw);
   const out = [];
@@ -174,6 +175,7 @@ export async function classifySoft({ root, sessionId, messages, topicSummaries,
       description: description.trim(),
       existingTopics: topicSummaries || [],
       language,
+      maxPromptChars,
     });
     const pass2Raw = await llm({ pass: 'triage-pass2', prompt: pass2Prompt, system: triageSystem(language) });
     const pass2 = parseJsonSafe(pass2Raw);
@@ -224,11 +226,12 @@ export async function triageOneSegment(args) {
  *   segments: Array<{ messages: Array<object> }>,
  *   topicSummaries: Array<{ path: string, summary: string }>,
  *   llm: (req: { pass: string, prompt: string, system: string }) => Promise<string>,
+ *   maxPromptChars?: number,
  *   onProgress?: (event: object) => void,
  * }} args
  * @returns {Promise<Array<{ kind: 'update'|'create', scope: string }>>}
  */
-export async function triageGroupSegments({ root, sessionId, segments, topicSummaries, llm, onProgress, language }) {
+export async function triageGroupSegments({ root, sessionId, segments, topicSummaries, llm, onProgress, language, maxPromptChars }) {
   let acc = [];
   let i = 0;
   for (const seg of (segments || [])) {
@@ -241,6 +244,7 @@ export async function triageGroupSegments({ root, sessionId, segments, topicSumm
       topicSummaries,
       llm,
       language,
+      maxPromptChars,
     });
     acc = dedupeActions([...acc, ...segActions]);
   }

package/yeaft/engine.js

@@ -691,7 +691,7 @@ export class Engine {
   /** @type {import('./stats/tool-usage.js').ToolUsageStats|null} — per-tool call/latency counters */
   #toolStats = null;
 
-  /** @type {object|null} — Config override for internal tasks (recall, consolidation, dream) using fastModel */
+  /** @type {object|null} — Config override for internal compact/recall tasks using fastModel; Dream uses the Session primary model */
   #fastConfig;
 
   /** @type {((agentId: string, evt: object) => void) | null} */
@@ -5437,7 +5437,7 @@ export class Engine {
   /** @returns {string|null} */
   get yeaftDir() { return this.#yeaftDir; }
 
-  /** @returns {object} — Config with fastModel as model (for internal tasks) */
+  /** @returns {object} — Config with fastModel as model (for compact and other non-Dream internal tasks) */
   get fastConfig() { return this.#fastConfig; }
 
   /**