@yeaft/webchat-agent 0.1.408 → 0.1.410

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.408",
+  "version": "0.1.410",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/cli.js

@@ -7,6 +7,13 @@
  *   --trace stats|recent|search <keyword>  — Query debug.db
  *   -i / --interactive   — REPL mode with / commands
  *   <prompt>             — One-shot query (Phase 1: engine.query)
+ *
+ * Phase 2 additions:
+ *   /memory              — Show memory status or manage entries
+ *   /history [n]         — Show last N messages from conversation
+ *   /search <keyword>    — Search conversation history
+ *   /compact             — Trigger manual consolidation
+ *   Conversation persistence across REPL sessions
  */
 
 import { createInterface } from 'readline';
@@ -18,6 +25,10 @@ import { createLLMAdapter } from './llm/adapter.js';
 import { Engine } from './engine.js';
 import { listModels, resolveModel } from './models.js';
 import { buildSystemPrompt } from './prompts.js';
+import { ConversationStore } from './conversation/persist.js';
+import { searchMessages } from './conversation/search.js';
+import { MemoryStore } from './memory/store.js';
+import { shouldConsolidate, consolidate } from './memory/consolidate.js';
 
 // ─── Argument parsing ──────────────────────────────────────────
 
@@ -191,21 +202,37 @@ async function runREPL(config, args) {
     dbPath: join(config.dir, 'debug.db'),
   });
 
+  // Initialize conversation and memory stores
+  const conversationStore = new ConversationStore(config.dir);
+  const memoryStore = new MemoryStore(config.dir);
+
   let adapter;
   let engine;
   let currentMode = args.mode;
-  let conversationMessages = []; // persistent conversation for REPL
+
+  // Load persisted conversation as initial messages
+  let conversationMessages = conversationStore.loadRecent(50).map(m => ({
+    role: m.role,
+    content: m.content,
+    ...(m.toolCallId && { toolCallId: m.toolCallId }),
+    ...(m.toolCalls && { toolCalls: m.toolCalls }),
+  }));
 
   // Lazy adapter creation (don't fail on start if no API key for --trace-only usage)
   async function ensureEngine() {
     if (!engine) {
       adapter = await createLLMAdapter(config);
-      engine = new Engine({ adapter, trace, config });
+      engine = new Engine({ adapter, trace, config, conversationStore, memoryStore });
     }
     return engine;
   }
 
+  const hotCount = conversationStore.countHot();
+  const coldCount = conversationStore.countCold();
+  const memStats = memoryStore.stats();
+
   console.log(`Yeaft Unify REPL (model: ${config.model}, mode: ${currentMode})`);
+  console.log(`Conversation: ${hotCount} hot, ${coldCount} cold | Memory: ${memStats.entryCount} entries`);
   console.log('Type /help for commands, /quit to exit.');
   console.log();
 
@@ -233,7 +260,10 @@ async function runREPL(config, args) {
           console.log('  /mode <chat|work|dream>  — Switch mode');
           console.log('  /debug                   — Toggle debug mode');
           console.log('  /trace <stats|recent>    — Query debug trace');
-          console.log('  /memory                  — Show memory status');
+          console.log('  /memory [add|clear|stats] — Memory management');
+          console.log('  /history [n]             — Show last N messages');
+          console.log('  /search <keyword>        — Search conversation history');
+          console.log('  /compact                 — Trigger consolidation');
           console.log('  /context                 — Show context info');
           console.log('  /dry-run                 — Toggle dry-run mode');
           console.log('  /stats                   — Show session stats');
@@ -269,9 +299,126 @@ async function runREPL(config, args) {
           break;
         }
 
-        case 'memory':
-          console.log('Memory status: (not yet implemented — Phase 2)');
+        case 'memory': {
+          const subcmd = cmdArgs[0];
+          if (subcmd === 'add') {
+            // /memory add <section> <entry text>
+            const section = cmdArgs[1];
+            const entryText = cmdArgs.slice(2).join(' ');
+            if (!section || !entryText) {
+              console.log('Usage: /memory add <section> <entry text>');
+            } else {
+              memoryStore.addToSection(section, `- ${entryText}`);
+              console.log(`Added to MEMORY.md [${section}]: ${entryText}`);
+            }
+          } else if (subcmd === 'clear') {
+            memoryStore.clear();
+            console.log('All memory cleared.');
+          } else if (subcmd === 'stats') {
+            const s = memoryStore.stats();
+            console.log('Memory stats:');
+            console.log(`  Entries: ${s.entryCount}`);
+            console.log(`  Scopes: ${s.scopes.join(', ') || '(none)'}`);
+            console.log(`  Kinds:  ${Object.entries(s.kinds).map(([k, v]) => `${k}=${v}`).join(', ') || '(none)'}`);
+          } else if (subcmd === 'search') {
+            const keyword = cmdArgs.slice(1).join(' ');
+            if (!keyword) {
+              console.log('Usage: /memory search <keyword>');
+            } else {
+              const results = memoryStore.search(keyword);
+              if (results.length === 0) {
+                console.log(`No memory entries matching "${keyword}".`);
+              } else {
+                console.log(`Found ${results.length} entries:`);
+                for (const e of results) {
+                  console.log(`  [${e.kind}] ${e.name} (scope: ${e.scope})`);
+                  console.log(`    ${(e.content || '').slice(0, 100)}`);
+                }
+              }
+            }
+          } else {
+            // Default: show MEMORY.md content
+            const profile = memoryStore.readProfile();
+            const s = memoryStore.stats();
+            if (profile) {
+              console.log('--- MEMORY.md ---');
+              console.log(profile.slice(0, 1000));
+              if (profile.length > 1000) console.log('... (truncated)');
+            } else {
+              console.log('MEMORY.md is empty.');
+            }
+            console.log(`\nEntries: ${s.entryCount} | Scopes: ${s.scopes.length}`);
+          }
+          break;
+        }
+
+        case 'history': {
+          const limit = parseInt(cmdArgs[0], 10) || 10;
+          const messages = conversationStore.loadRecent(limit);
+          if (messages.length === 0) {
+            console.log('No messages in history.');
+          } else {
+            console.log(`Last ${messages.length} messages:`);
+            for (const m of messages) {
+              const time = m.time ? new Date(m.time).toLocaleString() : '?';
+              const preview = (m.content || '').slice(0, 100).replace(/\n/g, ' ');
+              console.log(`  [${time}] ${m.role}: ${preview}`);
+            }
+          }
+          break;
+        }
+
+        case 'search': {
+          const keyword = cmdArgs.join(' ');
+          if (!keyword) {
+            console.log('Usage: /search <keyword>');
+          } else {
+            const results = searchMessages(config.dir, keyword, 20);
+            if (results.length === 0) {
+              console.log(`No messages matching "${keyword}".`);
+            } else {
+              console.log(`Found ${results.length} messages:`);
+              for (const m of results) {
+                const time = m.time ? new Date(m.time).toLocaleString() : '?';
+                const preview = (m.content || '').slice(0, 100).replace(/\n/g, ' ');
+                console.log(`  [${time}] ${m.role}: ${preview}`);
+              }
+            }
+          }
           break;
+        }
+
+        case 'compact': {
+          try {
+            const eng = await ensureEngine();
+            const budget = config.messageTokenBudget || 8192;
+            const hotTokens = conversationStore.hotTokens();
+            console.log(`Hot tokens: ${hotTokens} / budget: ${budget}`);
+
+            if (hotTokens <= 0) {
+              console.log('No messages to compact.');
+              break;
+            }
+
+            console.log('Running consolidation...');
+            const result = await consolidate({
+              conversationStore,
+              memoryStore,
+              adapter: adapter || await createLLMAdapter(config),
+              config,
+              budget: Math.min(budget, hotTokens), // force trigger
+            });
+            console.log(`Consolidation complete:`);
+            console.log(`  Archived: ${result.archivedCount} messages`);
+            console.log(`  Extracted: ${result.extractedEntries.length} memory entries`);
+            if (result.compactSummary) {
+              console.log(`  Summary: ${result.compactSummary.slice(0, 200)}...`);
+            }
+          } catch (e) {
+            console.error(`Consolidation error: ${e.message}`);
+          }
+          break;
+        }
 
         case 'context':
           console.log(`Context info:`);
@@ -280,6 +427,10 @@ async function runREPL(config, args) {
           console.log(`  Language: ${config.language}`);
           console.log(`  Max context: ${config.maxContextTokens} tokens`);
           console.log(`  System prompt: ${buildSystemPrompt({ language: config.language, mode: currentMode }).length} chars`);
+          console.log(`  Hot messages: ${conversationStore.countHot()}`);
+          console.log(`  Hot tokens: ${conversationStore.hotTokens()}`);
+          console.log(`  Cold messages: ${conversationStore.countCold()}`);
+          console.log(`  Memory entries: ${memoryStore.stats().entryCount}`);
           break;
 
         case 'dry-run':
@@ -293,6 +444,9 @@ async function runREPL(config, args) {
           console.log(`  Debug: ${config.debug}`);
           console.log(`  Turns: ${s.turnCount}`);
           console.log(`  Tools: ${s.toolCount}`);
+          console.log(`  Hot messages: ${conversationStore.countHot()}`);
+          console.log(`  Cold messages: ${conversationStore.countCold()}`);
+          console.log(`  Memory entries: ${memoryStore.stats().entryCount}`);
           break;
         }
 
@@ -340,8 +494,9 @@ async function runREPL(config, args) {
           break;
 
         case 'clear':
+          conversationStore.clear();
           conversationMessages = [];
-          console.log('Conversation history cleared.');
+          console.log('Conversation history cleared (including persisted messages).');
           break;
 
         case 'quit':
@@ -383,6 +538,19 @@ async function runREPL(config, args) {
               process.stderr.write(`[tool] ${event.name} → ${status}\n`);
             }
             break;
+          case 'recall':
+            if (config.debug) {
+              process.stderr.write(`[recall] ${event.entryCount} entries${event.cached ? ' (cached)' : ''}\n`);
+            }
+            break;
+          case 'consolidate':
+            if (config.debug) {
+              process.stderr.write(`[consolidate] archived=${event.archivedCount}, extracted=${event.extractedCount}\n`);
+            }
+            break;
+          case 'fallback':
+            process.stderr.write(`\n[fallback] ${event.from} → ${event.to}: ${event.reason}\n`);
+            break;
           case 'error':
             process.stderr.write(`\nError: ${event.error.message}\n`);
             break;
@@ -395,7 +563,8 @@ async function runREPL(config, args) {
       }
       console.log(); // newline after response
 
-      // Save both user and assistant messages for multi-turn context
+      // Update in-memory conversation for multi-turn context
+      // (engine.js already persists to disk via conversationStore)
       conversationMessages.push({ role: 'user', content: input });
       if (responseText) {
         conversationMessages.push({ role: 'assistant', content: responseText });
@@ -421,6 +590,10 @@ async function runOnce(config, args) {
     dbPath: join(config.dir, 'debug.db'),
   });
 
+  // Initialize stores for persistence
+  const conversationStore = new ConversationStore(config.dir);
+  const memoryStore = new MemoryStore(config.dir);
+
   try {
     if (args.dryRun) {
       handleDryRun(args, config);
@@ -428,9 +601,21 @@ async function runOnce(config, args) {
     }
 
     const adapter = await createLLMAdapter(config);
-    const engine = new Engine({ adapter, trace, config });
-
-    for await (const event of engine.query({ prompt: args.prompt, mode: args.mode })) {
+    const engine = new Engine({ adapter, trace, config, conversationStore, memoryStore });
+
+    // Load recent conversation as context
+    const priorMessages = conversationStore.loadRecent(20).map(m => ({
+      role: m.role,
+      content: m.content,
+      ...(m.toolCallId && { toolCallId: m.toolCallId }),
+      ...(m.toolCalls && { toolCalls: m.toolCalls }),
+    }));
+
+    for await (const event of engine.query({
+      prompt: args.prompt,
+      mode: args.mode,
+      messages: priorMessages,
+    })) {
       switch (event.type) {
         case 'text_delta':
           process.stdout.write(event.text);
@@ -446,6 +631,19 @@ async function runOnce(config, args) {
             process.stderr.write(`[tool] ${event.name} → ${status}\n`);
           }
           break;
+        case 'recall':
+          if (args.verbose || args.debug) {
+            process.stderr.write(`[recall] ${event.entryCount} entries${event.cached ? ' (cached)' : ''}\n`);
+          }
+          break;
+        case 'consolidate':
+          if (args.verbose || args.debug) {
+            process.stderr.write(`[consolidate] archived=${event.archivedCount}, extracted=${event.extractedCount}\n`);
+          }
+          break;
+        case 'fallback':
+          process.stderr.write(`\n[fallback] ${event.from} → ${event.to}: ${event.reason}\n`);
+          break;
         case 'error':
           process.stderr.write(`\nError: ${event.error.message}\n`);
           break;
@@ -513,12 +711,12 @@ async function main() {
   console.log('Yeaft Unify CLI');
   console.log();
   console.log('Usage:');
-  console.log('  node cli.js "your prompt"          — One-shot query');
-  console.log('  node cli.js -i                     — Interactive REPL');
-  console.log('  node cli.js --dry-run "prompt"      — Show what would be sent');
-  console.log('  node cli.js --trace stats           — Debug trace statistics');
-  console.log('  node cli.js --trace recent           — Recent turns');
-  console.log('  node cli.js --trace search "keyword" — Search traces');
+  console.log('  node cli.js "your prompt"           — One-shot query');
+  console.log('  node cli.js -i                      — Interactive REPL');
+  console.log('  node cli.js --dry-run "prompt"       — Show what would be sent');
+  console.log('  node cli.js --trace stats            — Debug trace statistics');
+  console.log('  node cli.js --trace recent            — Recent turns');
+  console.log('  node cli.js --trace search "keyword"  — Search traces');
   console.log();
   console.log('Options:');
   console.log('  -m, --mode <mode>   Mode: chat, work, dream (default: chat)');

package/unify/config.js

@@ -26,6 +26,8 @@ const DEFAULTS = {
   dir: DEFAULT_YEAFT_DIR,
   maxContextTokens: 200000,
   maxOutputTokens: 16384,
+  messageTokenBudget: 8192, // Phase 2: context * 4%, triggers consolidation
+  maxContinueTurns: 3,      // Phase 2: auto-continue on max_tokens
 };
 
 /**
@@ -217,6 +219,17 @@ export function loadConfig(overrides = {}) {
       overrides.maxOutputTokens ??
       fileConfig.maxOutputTokens ??
       DEFAULTS.maxOutputTokens,
+
+    messageTokenBudget:
+      overrides.messageTokenBudget ??
+      (env.YEAFT_MESSAGE_TOKEN_BUDGET ? parseInt(env.YEAFT_MESSAGE_TOKEN_BUDGET, 10) : null) ??
+      fileConfig.messageTokenBudget ??
+      DEFAULTS.messageTokenBudget,
+
+    maxContinueTurns:
+      overrides.maxContinueTurns ??
+      fileConfig.maxContinueTurns ??
+      DEFAULTS.maxContinueTurns,
   };
 
   // Auto-detect adapter using model registry + credential fallback