@yeaft/webchat-agent 0.1.408 → 0.1.409

package/unify/engine.js

@@ -2,22 +2,33 @@
  * engine.js — Yeaft query loop
  *
  * The engine is the core orchestrator:
- *   1. Build messages array
- *   2. Call adapter.stream()
- *   3. Collect text + tool_calls from stream events
- *   4. If tool_calls → execute tools → append results → goto 2
- *   5. If end_turn → done
- *   6. If max_tokens → done (Phase 2: auto-continue)
+ *   1. Before first turn: recall memories → inject into system prompt
+ *   2. Build messages array (with compact summary if available)
+ *   3. Call adapter.stream()
+ *   4. Collect text + tool_calls from stream events
+ *   5. If tool_calls → execute tools → append results → goto 3
+ *   6. If end_turn → persist messages → check consolidation → done
+ *   7. If max_tokens → auto-continue (up to maxContinueTurns)
+ *   8. On LLMContextError → force compact → retry
+ *   9. On retryable error with fallbackModel → switch model → retry
  *
  * Pattern derived from Claude Code's query loop (src/query.ts).
+ *
+ * Reference: yeaft-unify-implementation-plan.md §3.1, §4 (Phase 2)
  */
 
 import { randomUUID } from 'crypto';
 import { buildSystemPrompt } from './prompts.js';
+import { LLMContextError } from './llm/adapter.js';
+import { recall } from './memory/recall.js';
+import { shouldConsolidate, consolidate } from './memory/consolidate.js';
 
 /** Maximum number of turns before the engine stops to prevent infinite loops. */
 const MAX_TURNS = 25;
 
+/** Maximum auto-continue turns when stopReason is 'max_tokens'. */
+const MAX_CONTINUE_TURNS = 3;
+
 // ─── Engine Events (superset of adapter events) ──────────────────
 
 /**
@@ -25,8 +36,11 @@ const MAX_TURNS = 25;
  * @typedef {{ type: 'turn_end', turnNumber: number, stopReason: string }} TurnEndEvent
  * @typedef {{ type: 'tool_start', id: string, name: string, input: object }} ToolStartEvent
  * @typedef {{ type: 'tool_end', id: string, name: string, output: string, isError: boolean }} ToolEndEvent
+ * @typedef {{ type: 'consolidate', archivedCount: number, extractedCount: number }} ConsolidateEvent
+ * @typedef {{ type: 'recall', entryCount: number, cached: boolean }} RecallEvent
+ * @typedef {{ type: 'fallback', from: string, to: string, reason: string }} FallbackEvent
  *
- * @typedef {import('./llm/adapter.js').StreamEvent | TurnStartEvent | TurnEndEvent | ToolStartEvent | ToolEndEvent} EngineEvent
+ * @typedef {import('./llm/adapter.js').StreamEvent | TurnStartEvent | TurnEndEvent | ToolStartEvent | ToolEndEvent | ConsolidateEvent | RecallEvent | FallbackEvent} EngineEvent
  */
 
 // ─── Engine ──────────────────────────────────────────────────────
@@ -47,15 +61,29 @@ export class Engine {
   /** @type {string} */
   #traceId;
 
+  /** @type {import('./conversation/persist.js').ConversationStore|null} */
+  #conversationStore;
+
+  /** @type {import('./memory/store.js').MemoryStore|null} */
+  #memoryStore;
+
   /**
-   * @param {{ adapter: import('./llm/adapter.js').LLMAdapter, trace: object, config: object }} params
+   * @param {{
+   *   adapter: import('./llm/adapter.js').LLMAdapter,
+   *   trace: object,
+   *   config: object,
+   *   conversationStore?: import('./conversation/persist.js').ConversationStore,
+   *   memoryStore?: import('./memory/store.js').MemoryStore
+   * }} params
    */
-  constructor({ adapter, trace, config }) {
+  constructor({ adapter, trace, config, conversationStore, memoryStore }) {
     this.#adapter = adapter;
     this.#trace = trace;
     this.#config = config;
     this.#tools = new Map();
     this.#traceId = randomUUID();
+    this.#conversationStore = conversationStore || null;
+    this.#memoryStore = memoryStore || null;
   }
 
   /**
@@ -94,19 +122,120 @@ export class Engine {
   }
 
   /**
-   * Build the system prompt.
+   * Build the system prompt with memory and compact summary.
    *
-   * @param {string} mode — 'chat' | 'work' | 'dream'
+   * @param {string} mode
+   * @param {{ profile?: string, entries?: object[] }} [memory]
+   * @param {string} [compactSummary]
    * @returns {string}
    */
-  #buildSystemPrompt(mode) {
+  #buildSystemPrompt(mode, memory, compactSummary) {
     return buildSystemPrompt({
       language: this.#config.language || 'en',
       mode,
       toolNames: Array.from(this.#tools.keys()),
+      memory,
+      compactSummary,
     });
   }
 
+  /**
+   * Perform memory recall for a given prompt.
+   *
+   * @param {string} prompt
+   * @returns {Promise<{ profile: string, entries: object[] }|null>}
+   */
+  async #recallMemory(prompt) {
+    if (!this.#memoryStore) return null;
+
+    const memory = { profile: '', entries: [] };
+
+    // Read user profile
+    memory.profile = this.#memoryStore.readProfile();
+
+    // Recall relevant entries
+    try {
+      const result = await recall({
+        prompt,
+        adapter: this.#adapter,
+        config: this.#config,
+        memoryStore: this.#memoryStore,
+      });
+      memory.entries = result.entries;
+    } catch {
+      // Recall failure is non-critical
+    }
+
+    return memory;
+  }
+
+  /**
+   * Read compact summary from conversation store.
+   *
+   * @returns {string}
+   */
+  #getCompactSummary() {
+    if (!this.#conversationStore) return '';
+    return this.#conversationStore.readCompactSummary();
+  }
+
+  /**
+   * Persist user message and assistant response to conversation store.
+   *
+   * @param {string} userContent
+   * @param {string} assistantContent
+   * @param {string} mode
+   * @param {object[]} [toolCalls]
+   */
+  #persistMessages(userContent, assistantContent, mode, toolCalls) {
+    if (!this.#conversationStore) return;
+
+    // Persist user message
+    this.#conversationStore.append({
+      role: 'user',
+      content: userContent,
+      mode,
+    });
+
+    // Persist assistant message
+    const assistantMsg = {
+      role: 'assistant',
+      content: assistantContent,
+      mode,
+      model: this.#config.model,
+    };
+    if (toolCalls && toolCalls.length > 0) {
+      assistantMsg.toolCalls = toolCalls;
+    }
+    this.#conversationStore.append(assistantMsg);
+  }
+
+  /**
+   * Check and trigger consolidation if needed.
+   *
+   * @returns {Promise<{ archivedCount: number, extractedCount: number }|null>}
+   */
+  async #maybeConsolidate() {
+    if (!this.#conversationStore || !this.#memoryStore) return null;
+
+    const budget = this.#config.messageTokenBudget || 8192;
+    if (!shouldConsolidate(this.#conversationStore, budget)) return null;
+
+    try {
+      const result = await consolidate({
+        conversationStore: this.#conversationStore,
+        memoryStore: this.#memoryStore,
+        adapter: this.#adapter,
+        config: this.#config,
+        budget,
+      });
+      return { archivedCount: result.archivedCount, extractedCount: result.extractedEntries.length };
+    } catch {
+      // Consolidation failure is non-critical
+      return null;
+    }
+  }
+
   /**
    * Run a query — the main loop.
    *
@@ -126,7 +255,14 @@ export class Engine {
       return;
     }
 
-    const systemPrompt = this.#buildSystemPrompt(mode);
+    // ─── Pre-query: Recall + Compact Summary ────────────────
+    const memory = await this.#recallMemory(prompt);
+    if (memory && memory.entries.length > 0) {
+      yield { type: 'recall', entryCount: memory.entries.length, cached: false };
+    }
+
+    const compactSummary = this.#getCompactSummary();
+    const systemPrompt = this.#buildSystemPrompt(mode, memory, compactSummary);
 
     // Build conversation: existing messages + new user message
     const conversationMessages = [
@@ -136,6 +272,9 @@ export class Engine {
 
     const toolDefs = this.#getToolDefs();
     let turnNumber = 0;
+    let continueTurns = 0; // auto-continue counter
+    let fullResponseText = '';
+    let currentModel = this.#config.model;
 
     while (true) {
       turnNumber++;
@@ -166,9 +305,8 @@ export class Engine {
 
       try {
         // Stream from adapter
-        // Note: pass a snapshot of messages so later mutations don't affect the adapter
         for await (const event of this.#adapter.stream({
-          model: this.#config.model,
+          model: currentModel,
           system: systemPrompt,
           messages: [...conversationMessages],
           tools: toolDefs.length > 0 ? toolDefs : undefined,
@@ -202,10 +340,9 @@ export class Engine {
           }
         }
       } catch (err) {
-        // Adapter threw an exception (network, auth, etc.)
         const latencyMs = Date.now() - startTime;
         this.#trace.endTurn(turnId, {
-          model: this.#config.model,
+          model: currentModel,
           inputTokens: totalUsage.inputTokens,
           outputTokens: totalUsage.outputTokens,
           stopReason: 'error',
@@ -213,6 +350,26 @@ export class Engine {
           responseText,
         });
 
+        // ─── LLMContextError → force compact → retry ──────
+        if (err instanceof LLMContextError && this.#conversationStore && this.#memoryStore) {
+          const consolidated = await this.#maybeConsolidate();
+          if (consolidated && consolidated.archivedCount > 0) {
+            yield { type: 'consolidate', archivedCount: consolidated.archivedCount, extractedCount: consolidated.extractedCount };
+            yield { type: 'turn_end', turnNumber, stopReason: 'context_overflow_retry' };
+            continue; // retry with fewer messages
+          }
+        }
+
+        // ─── Fallback model ──────────────────────────────
+        const fallbackModel = this.#config.fallbackModel;
+        if (fallbackModel && fallbackModel !== currentModel &&
+            (err.name === 'LLMRateLimitError' || err.name === 'LLMServerError')) {
+          yield { type: 'fallback', from: currentModel, to: fallbackModel, reason: err.message };
+          currentModel = fallbackModel;
+          yield { type: 'turn_end', turnNumber, stopReason: 'fallback_retry' };
+          continue; // retry with fallback model
+        }
+
         yield {
           type: 'error',
           error: err,
@@ -226,7 +383,7 @@ export class Engine {
 
       // Record turn in debug trace
       this.#trace.endTurn(turnId, {
-        model: this.#config.model,
+        model: currentModel,
         inputTokens: totalUsage.inputTokens,
         outputTokens: totalUsage.outputTokens,
         stopReason,
@@ -244,10 +401,29 @@ export class Engine {
         }));
       }
       conversationMessages.push(assistantMsg);
+      fullResponseText += responseText;
+
+      // ─── Handle max_tokens → auto-continue ────────────
+      if (stopReason === 'max_tokens' && continueTurns < MAX_CONTINUE_TURNS) {
+        continueTurns++;
+        // Append a "Continue" user message
+        conversationMessages.push({ role: 'user', content: 'Continue' });
+        yield { type: 'turn_end', turnNumber, stopReason: 'max_tokens_continue' };
+        continue; // loop back to call adapter again
+      }
 
       // If no tool calls, we're done
       if (stopReason !== 'tool_use' || toolCalls.length === 0) {
         yield { type: 'turn_end', turnNumber, stopReason };
+
+        // ─── Post-query: Persist + Consolidate ────────────
+        this.#persistMessages(prompt, fullResponseText, mode, assistantMsg.toolCalls);
+
+        const consolidated = await this.#maybeConsolidate();
+        if (consolidated && consolidated.archivedCount > 0) {
+          yield { type: 'consolidate', archivedCount: consolidated.archivedCount, extractedCount: consolidated.extractedCount };
+        }
+
         break;
       }
 
@@ -316,4 +492,20 @@ export class Engine {
   get toolNames() {
     return Array.from(this.#tools.keys());
   }
+
+  /**
+   * Get the conversation store (for external access, e.g., CLI commands).
+   * @returns {import('./conversation/persist.js').ConversationStore|null}
+   */
+  get conversationStore() {
+    return this.#conversationStore;
+  }
+
+  /**
+   * Get the memory store (for external access, e.g., CLI commands).
+   * @returns {import('./memory/store.js').MemoryStore|null}
+   */
+  get memoryStore() {
+    return this.#memoryStore;
+  }
 }

package/unify/index.js

@@ -19,3 +19,9 @@ export {
 export { MODEL_REGISTRY, resolveModel, listModels, isKnownModel } from './models.js';
 export { buildSystemPrompt, SUPPORTED_LANGUAGES } from './prompts.js';
 export { Engine } from './engine.js';
+export { ConversationStore, parseMessage, estimateTokens } from './conversation/persist.js';
+export { searchMessages } from './conversation/search.js';
+export { MemoryStore, parseEntry, serializeEntry, MEMORY_KINDS } from './memory/store.js';
+export { recall, extractKeywords, computeFingerprint, clearRecallCache } from './memory/recall.js';
+export { extractMemories } from './memory/extract.js';
+export { consolidate, shouldConsolidate } from './memory/consolidate.js';

package/unify/memory/consolidate.js

@@ -0,0 +1,187 @@
+/**
+ * consolidate.js — Consolidate = compact + extract (one LLM call)
+ *
+ * Triggered when hot_tokens > MESSAGE_TOKEN_BUDGET.
+ * One LLM call does two things simultaneously:
+ *   1. Generate compact summary → append to compact.md ("short-term memory")
+ *   2. Extract memory entries → write to entries/ ("long-term memory")
+ *
+ * After consolidation:
+ *   - Processed messages moved from messages/ to cold/
+ *   - index.md + scopes.md updated
+ *
+ * Reference: yeaft-unify-core-systems.md §3.1, §4.2
+ *            yeaft-unify-design.md §6.1
+ */
+
+import { extractMemories } from './extract.js';
+
+// ─── Constants ──────────────────────────────────────────────────
+
+/** Default MESSAGE_TOKEN_BUDGET (context * 4%, default ~8192). */
+export const DEFAULT_MESSAGE_TOKEN_BUDGET = 8192;
+
+/** After compact, keep this fraction of the budget. */
+export const COMPACT_KEEP_RATIO = 0.4;
+
+/** Minimum messages to keep hot (newest). */
+const MIN_KEEP_MESSAGES = 3;
+
+// ─── Consolidate ────────────────────────────────────────────────
+
+/**
+ * Check if consolidation should be triggered.
+ *
+ * @param {import('../conversation/persist.js').ConversationStore} conversationStore
+ * @param {number} [budget] — MESSAGE_TOKEN_BUDGET
+ * @returns {boolean}
+ */
+export function shouldConsolidate(conversationStore, budget = DEFAULT_MESSAGE_TOKEN_BUDGET) {
+  const hotTokens = conversationStore.hotTokens();
+  return hotTokens > budget;
+}
+
+/**
+ * Determine which messages to archive (move to cold).
+ * Strategy: from oldest, accumulate tokens until remaining ≤ budget * 40%.
+ * Always keep at least MIN_KEEP_MESSAGES.
+ *
+ * @param {object[]} messages — all hot messages, sorted chronologically
+ * @param {number} budget — MESSAGE_TOKEN_BUDGET
+ * @returns {{ toArchive: object[], toKeep: object[] }}
+ */
+export function partitionMessages(messages, budget = DEFAULT_MESSAGE_TOKEN_BUDGET) {
+  if (messages.length <= MIN_KEEP_MESSAGES) {
+    return { toArchive: [], toKeep: messages };
+  }
+
+  const keepBudget = Math.floor(budget * COMPACT_KEEP_RATIO);
+
+  // Work backwards from newest: accumulate tokens until we hit keepBudget
+  let keepTokens = 0;
+  let keepStart = messages.length;
+
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msgTokens = messages[i].tokens_est || 0;
+    if (keepTokens + msgTokens > keepBudget && (messages.length - i) >= MIN_KEEP_MESSAGES) {
+      keepStart = i + 1;
+      break;
+    }
+    keepTokens += msgTokens;
+    if (i === 0) keepStart = 0;
+  }
+
+  // Ensure at least MIN_KEEP_MESSAGES are kept
+  keepStart = Math.min(keepStart, messages.length - MIN_KEEP_MESSAGES);
+  keepStart = Math.max(keepStart, 0);
+
+  return {
+    toArchive: messages.slice(0, keepStart),
+    toKeep: messages.slice(keepStart),
+  };
+}
+
+/**
+ * Generate a compact summary of messages.
+ *
+ * @param {object[]} messages — messages to summarize
+ * @param {object} adapter — LLM adapter with .call()
+ * @param {object} config — { model }
+ * @returns {Promise<string>} — compact summary text
+ */
+async function generateSummary(messages, adapter, config) {
+  const conversation = messages.map(m => {
+    const prefix = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : m.role;
+    return `[${prefix}]: ${(m.content || '').slice(0, 500)}`;
+  }).join('\n\n');
+
+  const system = 'You are a conversation summarizer. Summarize the conversation concisely in 2-3 paragraphs, preserving key decisions, facts, and context. Write in the same language as the conversation.';
+
+  try {
+    const result = await adapter.call({
+      model: config.model,
+      system,
+      messages: [{ role: 'user', content: `Summarize this conversation:\n\n${conversation}` }],
+      maxTokens: 1024,
+    });
+    return result.text.trim();
+  } catch {
+    // Fallback: simple concatenation of first/last messages
+    const first = messages[0]?.content?.slice(0, 200) || '';
+    const last = messages[messages.length - 1]?.content?.slice(0, 200) || '';
+    return `[Auto-summary failed] Started with: ${first}... Ended with: ${last}`;
+  }
+}
+
+/**
+ * Run the full Consolidate pipeline.
+ *
+ * 1. Partition messages (what to archive vs keep)
+ * 2. Generate compact summary (LLM call)
+ * 3. Extract memory entries (LLM call)
+ * 4. Move archived messages to cold/
+ * 5. Update compact.md, index.md, scopes.md
+ *
+ * @param {{
+ *   conversationStore: import('../conversation/persist.js').ConversationStore,
+ *   memoryStore: import('./store.js').MemoryStore,
+ *   adapter: object,
+ *   config: object,
+ *   budget?: number
+ * }} params
+ * @returns {Promise<{ compactSummary: string, extractedEntries: string[], archivedCount: number }>}
+ */
+export async function consolidate({ conversationStore, memoryStore, adapter, config, budget = DEFAULT_MESSAGE_TOKEN_BUDGET }) {
+  // Load all hot messages
+  const messages = conversationStore.loadAll();
+
+  if (messages.length <= MIN_KEEP_MESSAGES) {
+    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
+  }
+
+  // Step 1: Partition
+  const { toArchive, toKeep } = partitionMessages(messages, budget);
+
+  if (toArchive.length === 0) {
+    return { compactSummary: '', extractedEntries: [], archivedCount: 0 };
+  }
+
+  // Step 2: Generate compact summary
+  const compactSummary = await generateSummary(toArchive, adapter, config);
+
+  // Step 3: Extract memory entries
+  const extracted = await extractMemories({ messages: toArchive, adapter, config });
+
+  // Step 4: Move archived messages to cold
+  const archiveIds = toArchive.map(m => m.id).filter(Boolean);
+  conversationStore.moveToColdBatch(archiveIds);
+
+  // Step 5a: Update compact.md
+  if (compactSummary) {
+    conversationStore.updateCompactSummary(compactSummary);
+  }
+
+  // Step 5b: Write extracted memory entries
+  const entryNames = [];
+  for (const entry of extracted) {
+    const slug = memoryStore.writeEntry(entry);
+    entryNames.push(slug);
+  }
+
+  // Step 5c: Update index.md
+  const lastMsg = toKeep[toKeep.length - 1];
+  conversationStore.updateIndex({
+    lastMessageId: lastMsg?.id || null,
+  });
+
+  // Step 5d: Rebuild scopes.md
+  if (entryNames.length > 0) {
+    memoryStore.rebuildScopes();
+  }
+
+  return {
+    compactSummary,
+    extractedEntries: entryNames,
+    archivedCount: archiveIds.length,
+  };
+}

package/unify/memory/extract.js

@@ -0,0 +1,97 @@
+/**
+ * extract.js — Extract memory-worthy entries from conversation
+ *
+ * Called by consolidate.js during the Consolidate lifecycle.
+ * Uses a single LLM call to identify facts, preferences, skills,
+ * lessons, contexts, and relations from conversation messages.
+ *
+ * Reference: yeaft-unify-core-systems.md §3.1, yeaft-unify-design.md §6.1
+ */
+
+import { MEMORY_KINDS } from './store.js';
+
+/**
+ * Build the extraction prompt.
+ * @param {object[]} messages — conversation messages to analyze
+ * @returns {string}
+ */
+function buildExtractionPrompt(messages) {
+  const conversation = messages.map(m => {
+    const prefix = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : 'System';
+    return `[${prefix}]: ${m.content}`;
+  }).join('\n\n');
+
+  return `Analyze the following conversation and extract any memorable information worth saving to long-term memory.
+
+For each memory, provide:
+- **name**: A short slug-friendly name (e.g., "user-prefers-typescript", "project-uses-vue3")
+- **kind**: One of: ${MEMORY_KINDS.join(', ')}
+- **scope**: A tree path (e.g., "global", "tech/typescript", "work/project-name")
+- **tags**: Relevant keywords as an array
+- **importance**: "high", "normal", or "low"
+- **content**: 1-3 sentences describing the memory
+
+Memory kinds explained:
+- fact: Objective facts (project structure, tech stack)
+- preference: User preferences (coding style, tools)
+- skill: How to do something (patterns, techniques)
+- lesson: Lessons learned (bugs, pitfalls)
+- context: Temporal context (current OKR, progress)
+- relation: People and relationships (teammates, roles)
+
+Do NOT extract:
+- Specific code snippets (too large, will become stale)
+- Temporary debugging information
+- Trivial greetings or small talk
+
+Return a JSON array of memory objects. If nothing is worth remembering, return an empty array [].
+
+Conversation:
+${conversation}`;
+}
+
+/**
+ * Extract memory entries from a set of conversation messages.
+ *
+ * @param {{ messages: object[], adapter: object, config: object }} params
+ * @returns {Promise<object[]>} — extracted memory entries
+ */
+export async function extractMemories({ messages, adapter, config }) {
+  if (!messages || messages.length === 0) return [];
+
+  const system = 'You are a memory extraction assistant. Analyze conversations and extract important facts, preferences, and lessons. Return ONLY a valid JSON array, no other text.';
+
+  const extractionPrompt = buildExtractionPrompt(messages);
+
+  try {
+    const result = await adapter.call({
+      model: config.model,
+      system,
+      messages: [{ role: 'user', content: extractionPrompt }],
+      maxTokens: 2048,
+    });
+
+    const text = result.text.trim();
+
+    // Try to parse JSON array from the response
+    const jsonMatch = text.match(/\[[\s\S]*\]/);
+    if (!jsonMatch) return [];
+
+    const entries = JSON.parse(jsonMatch[0]);
+
+    // Validate and normalize entries
+    return entries
+      .filter(e => e && typeof e === 'object' && e.name && e.content)
+      .map(e => ({
+        name: String(e.name).slice(0, 80),
+        kind: MEMORY_KINDS.includes(e.kind) ? e.kind : 'fact',
+        scope: String(e.scope || 'global'),
+        tags: Array.isArray(e.tags) ? e.tags.map(String) : [],
+        importance: ['high', 'normal', 'low'].includes(e.importance) ? e.importance : 'normal',
+        content: String(e.content),
+      }));
+  } catch {
+    // LLM failure — return empty (non-critical operation)
+    return [];
+  }
+}