@yeaft/webchat-agent 0.1.549 → 0.1.551

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.549",
+  "version": "0.1.551",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/engine.js

@@ -23,6 +23,7 @@ import { LLMContextError, LLMAbortError } from './llm/adapter.js';
 import { recallR6, formatForInjection } from './memory/recall-r6.js';
 import { shouldConsolidate, consolidate } from './memory/consolidate.js';
 import { buildMemoryInjection } from './memory/layout.js';
+import { buildUserProfile } from './memory/user-memory-store.js';
 import { runStopHooks } from './stop-hooks.js';
 import { getThreadStore, MAIN_THREAD_ID } from './threads/store.js';
 import { pickEffort, parseEffortPrefix } from './effort.js';
@@ -293,9 +294,10 @@ export class Engine {
    * @param {string} [compactSummary]
    * @param {string} [prompt] — user prompt (for skill relevance matching)
    * @param {string} [memoryInjection] — task-287: prebuilt memory block (index + prefs + project)
+   * @param {string} [userProfile] — user profile from user-memory shard store
    * @returns {string}
    */
-  #buildSystemPrompt(memory, compactSummary, prompt, memoryInjection) {
+  #buildSystemPrompt(memory, compactSummary, prompt, memoryInjection, userProfile) {
     // Get relevant skill content if SkillManager is wired
     let skillContent = '';
     if (this.#skillManager && prompt) {
@@ -314,6 +316,7 @@ export class Engine {
       memoryInjection,
       compactSummary,
       skillContent,
+      userProfile,
       // task-334f: memory_trace tool is now registered (49 → 51 tools), so
       // unlock the core_memory meta-line behind 334e's feature flag.
       memoryTraceAvailable: true,
@@ -359,9 +362,20 @@ export class Engine {
   async #recallMemory(prompt) {
     const memory = { profile: '', entries: [], formatted: '' };
 
-    // Read user profile from legacy store if available
-    if (this.#memoryStore) {
-      memory.profile = this.#memoryStore.readProfile();
+    // Build user profile from user-memory shard store (R6 path),
+    // falling back to legacy readProfile if shard store unavailable.
+    try {
+      const profile = buildUserProfile(this.#memoryShardStore);
+      if (profile) {
+        memory.profile = profile;
+      } else if (this.#memoryStore) {
+        memory.profile = this.#memoryStore.readProfile();
+      }
+    } catch {
+      // Non-critical — fall through to legacy
+      if (this.#memoryStore) {
+        try { memory.profile = this.#memoryStore.readProfile(); } catch { /* */ }
+      }
     }
 
     // R6 shard-based recall (preferred path)
@@ -603,7 +617,8 @@ export class Engine {
     }
 
     const compactSummary = this.#getCompactSummary();
-    const systemPrompt = this.#buildSystemPrompt(undefined, compactSummary, prompt, memoryInjection);
+    const userProfile = recallResult?.profile || '';
+    const systemPrompt = this.#buildSystemPrompt(undefined, compactSummary, prompt, memoryInjection, userProfile);
 
     // Build conversation: existing messages + new user message
     const conversationMessages = [

package/unify/memory/dream-scheduler.js

@@ -23,7 +23,7 @@
 
 import { dreamShard } from './dream-shard.js';
 import { checkRecompression } from './recompression.js';
-import { dreamExtract } from './dream-extract.js';
+import { runUserDreamJob } from './user-memory-store.js';
 
 /** Default idle timeout before dream triggers (ms). */
 export const DREAM_IDLE_MS = 30 * 60 * 1000; // 30 min
@@ -36,6 +36,8 @@ const MAX_CONCURRENT_DREAMS = 2;
  *
  * @param {{
  *   memoryShardStore: object | null,
+ *   userMemoryStore: object | null,
+ *   conversationStore: object | null,
  *   adapter: object | null,
  *   config: object,
  *   group?: import('../groups/group-store.js').GroupHandle | null,
@@ -50,6 +52,8 @@ const MAX_CONCURRENT_DREAMS = 2;
 export function createDreamScheduler(opts = {}) {
   const {
     memoryShardStore,
+    userMemoryStore,
+    conversationStore,
     adapter,
     config,
     group = null,
@@ -123,24 +127,7 @@ export function createDreamScheduler(opts = {}) {
     try {
       onDreamStart?.(vpId);
 
-      // Phase A: Extract new memories from conversation (§Δ26)
-      let extractResult = null;
-      if (group && memoryDir) {
-        try {
-          extractResult = await dreamExtract({
-            group,
-            shardStore: memoryShardStore,
-            adapter,
-            config: { model: config?.primaryModel || config?.model || 'default' },
-            memoryDir,
-          });
-        } catch (err) {
-          // Non-fatal — proceed to compact even if extract fails
-          extractResult = { error: err.message };
-        }
-      }
-
-      // Phase B: Shard-based compact/merge/prune
+      // Shard-based compact/merge/prune
       const result = await dreamShard({
         shardStore: memoryShardStore,
         adapter,
@@ -158,8 +145,18 @@ export function createDreamScheduler(opts = {}) {
         // Non-fatal
       }
 
-      // Attach extract result
-      result.extract = extractResult;
+      // Post-dream: run user-memory extract + compact (334-w7b)
+      try {
+        const userDreamResult = await runUserDreamJob({
+          store: userMemoryStore || undefined,
+          conversationStore: conversationStore || undefined,
+          adapter: adapter || undefined,
+          config: { model: config?.primaryModel || config?.model || 'default' },
+        });
+        result.userDream = userDreamResult;
+      } catch {
+        // Non-fatal
+      }
 
       messagesSinceLastDream = 0;
       lastDreamAt = Date.now();

package/unify/memory/user-memory-store.js

@@ -15,10 +15,11 @@
 import { homedir } from 'os';
 import { join } from 'path';
 import { randomUUID } from 'crypto';
-import { existsSync } from 'fs';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
 import { openMemoryShardStore } from './shard-store.js';
 import { USER_SHARDS } from './schema.js';
 import { scanShards, runCompactJob } from './dream-shard.js';
+import { pickEffort } from '../effort.js';
 
 /** Default storage root for user memory. */
 export const USER_MEMORY_DIR = join(homedir(), '.yeaft', 'user', 'memory');
@@ -193,19 +194,45 @@ export function buildUserProfile(store, opts = {}) {
 // ─── Dream Job ───────────────────────────────────────────────
 
 /**
- * Run user-memory dream maintenance (compact low-utilization shards).
- * Reuses dream-shard.js compact framework — no LLM calls needed for
- * user-memory (user-authored entries don't need merge/prune by an LLM;
- * we only compact to reclaim superseded/removed tombstones).
+ * Run user-memory dream maintenance: extract phase + compact.
+ * Extract reads conversation messages since the last watermark, uses LLM to
+ * identify user-relevant facts, then writes them to the appropriate shards.
+ * Compact phase reclaims superseded/removed tombstones (unchanged from 334g).
  *
- * @param {{ store?: object, onPhase?: (phase:string, data:any) => void }} [opts]
- * @returns {{ scan: object, compact: object } | null}
+ * @param {{
+ *   store?: object,
+ *   conversationStore?: object,
+ *   adapter?: object,
+ *   config?: object,
+ *   onPhase?: (phase: string, data: any) => void,
+ * }} [opts]
+ * @returns {Promise<{ extract: object|null, scan: object, compact: object } | null>}
  */
-export function runUserDreamJob(opts = {}) {
+export async function runUserDreamJob(opts = {}) {
   const store = 'store' in opts ? opts.store : getUserMemoryStore();
   if (!store) return null;
 
+  let extractResult = null;
+
   try {
+    // ── Phase 1: Extract (LLM) ─────────────────────────────
+    if (opts.conversationStore && opts.adapter && opts.config) {
+      opts.onPhase?.('extract', 'starting');
+      try {
+        extractResult = await dreamExtract({
+          store,
+          conversationStore: opts.conversationStore,
+          adapter: opts.adapter,
+          config: opts.config,
+        });
+        opts.onPhase?.('extract', extractResult);
+      } catch (err) {
+        console.warn('[user-memory-store] extract phase failed:', err.message);
+        extractResult = { error: err.message, extracted: 0 };
+      }
+    }
+
+    // ── Phase 2: Compact ───────────────────────────────────
     const scan = scanShards(store);
     const compact = runCompactJob({
       shardStore: store,
@@ -214,9 +241,212 @@ export function runUserDreamJob(opts = {}) {
         ? (shard, r) => opts.onPhase('compact', { shard, ...r })
         : undefined,
     });
-    return { scan: { totalEntries: scan.totalEntries, totalBytes: scan.totalBytes }, compact };
+    return {
+      extract: extractResult,
+      scan: { totalEntries: scan.totalEntries, totalBytes: scan.totalBytes },
+      compact,
+    };
   } catch (err) {
     console.warn('[user-memory-store] dream job failed:', err.message);
     return null;
   }
 }
+
+// ─── Watermark ──────────────────────────────────────────────
+
+/**
+ * Watermark format (shared with 334-w7b):
+ *   { lastMessageId: string, lastMessageTs: number, updatedAt: string }
+ *
+ * Stored at <storeDir>/.watermark.json (alongside shard files).
+ */
+
+const WATERMARK_FILE = '.watermark.json';
+
+/**
+ * Read the extract watermark for a user-memory store.
+ * Returns null if no watermark exists yet.
+ *
+ * @param {string} dir — store directory (e.g. ~/.yeaft/user/memory)
+ * @returns {{ lastMessageId: string, lastMessageTs: number, updatedAt: string } | null}
+ */
+export function readWatermark(dir) {
+  try {
+    const p = join(dir, WATERMARK_FILE);
+    if (!existsSync(p)) return null;
+    return JSON.parse(readFileSync(p, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Write the extract watermark.
+ *
+ * @param {string} dir
+ * @param {{ lastMessageId: string, lastMessageTs: number }} wm
+ */
+export function writeWatermark(dir, wm) {
+  try {
+    const p = join(dir, WATERMARK_FILE);
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(p, JSON.stringify({
+      lastMessageId: wm.lastMessageId,
+      lastMessageTs: wm.lastMessageTs,
+      updatedAt: new Date().toISOString(),
+    }, null, 2));
+  } catch (err) {
+    console.warn('[user-memory-store] writeWatermark failed:', err.message);
+  }
+}
+
+// ─── Extract Phase ──────────────────────────────────────────
+
+/** Max messages to process in a single extract pass. */
+const EXTRACT_MAX_MESSAGES = 50;
+
+/** Min messages required to trigger an extract. */
+const EXTRACT_MIN_MESSAGES = 3;
+
+/**
+ * Build the user-memory extraction prompt.
+ * Tailored for user-relevant facts (not VP/task memory).
+ *
+ * @param {object[]} messages
+ * @returns {string}
+ */
+export function buildUserExtractPrompt(messages) {
+  const conversation = messages.map(m => {
+    const prefix = m.role === 'user' ? 'User' : m.role === 'assistant' ? 'Assistant' : 'System';
+    return `[${prefix}]: ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`;
+  }).join('\n\n');
+
+  return `Analyze the following conversation and extract facts about THE USER that are worth remembering long-term.
+
+Focus on these categories:
+- **profile**: Name, job title, company, location, background, expertise areas
+- **preferences**: Coding style, tool preferences, language preferences, communication style
+- **projects**: Projects they work on, tech stacks, repositories, products
+- **goals**: Current goals, objectives, what they're trying to achieve
+- **relations**: Team members, colleagues, managers, collaborators mentioned
+
+For each fact, provide:
+- **shard**: One of: profile, preferences, projects, goals, relations
+- **body**: 1-2 sentences describing the fact clearly
+- **tags**: 1-3 keyword tags as an array
+
+Do NOT extract:
+- Specific code snippets or technical instructions
+- Temporary debugging context
+- Facts about the assistant (only about the user)
+- Information already implied by the conversation being about coding
+
+Return a JSON array. If nothing about the user is worth remembering, return [].
+
+Conversation:
+${conversation}`;
+}
+
+/**
+ * Extract user-relevant facts from conversation messages and write to user-memory shards.
+ *
+ * @param {{
+ *   store: object,
+ *   conversationStore: object,
+ *   adapter: object,
+ *   config: object,
+ *   dir?: string,
+ * }} params
+ * @returns {Promise<{ extracted: number, skipped: number, watermark: object|null }>}
+ */
+export async function dreamExtract({ store, conversationStore, adapter, config, dir }) {
+  const storeDir = dir || USER_MEMORY_DIR;
+  const wm = readWatermark(storeDir);
+
+  // Load all messages and filter to those after watermark
+  const allMessages = conversationStore.loadAll();
+  let newMessages;
+
+  if (wm && wm.lastMessageId) {
+    const idx = allMessages.findIndex(m => m.id === wm.lastMessageId);
+    newMessages = idx >= 0 ? allMessages.slice(idx + 1) : allMessages;
+  } else {
+    // No watermark — process all messages (first run)
+    newMessages = allMessages;
+  }
+
+  // Filter to user + assistant messages only (skip system)
+  newMessages = newMessages.filter(m => m.role === 'user' || m.role === 'assistant');
+
+  if (newMessages.length < EXTRACT_MIN_MESSAGES) {
+    return { extracted: 0, skipped: 0, watermark: wm };
+  }
+
+  // Cap to prevent huge LLM calls
+  const batch = newMessages.slice(-EXTRACT_MAX_MESSAGES);
+
+  // LLM extraction call
+  const system = 'You are a user profile extraction assistant. Analyze conversations and extract facts about the user. Return ONLY a valid JSON array, no other text.';
+  const prompt = buildUserExtractPrompt(batch);
+
+  let candidates = [];
+  try {
+    const result = await adapter.call({
+      model: config.model || config.primaryModel || 'default',
+      system,
+      messages: [{ role: 'user', content: prompt }],
+      maxTokens: 2048,
+      effort: pickEffort({ scenario: 'dream' }),
+    });
+
+    const text = result.text.trim();
+    const jsonMatch = text.match(/\[[\s\S]*\]/);
+    if (jsonMatch) {
+      candidates = JSON.parse(jsonMatch[0]);
+    }
+  } catch {
+    return { extracted: 0, skipped: 0, watermark: wm, error: 'llm_failed' };
+  }
+
+  if (!Array.isArray(candidates)) {
+    return { extracted: 0, skipped: 0, watermark: wm };
+  }
+
+  // Validate and write candidates
+  let extracted = 0;
+  let skipped = 0;
+
+  for (const c of candidates) {
+    if (!c || typeof c !== 'object' || !c.body) { skipped++; continue; }
+
+    // Use classifyUserMemoryShard if shard not provided or invalid
+    const shard = USER_SHARDS.includes(c.shard)
+      ? c.shard
+      : classifyUserMemoryShard(c.body, c.tags);
+
+    const id = writeUserMemory(store, {
+      text: c.body,
+      tags: Array.isArray(c.tags) ? c.tags.map(String) : [],
+      sourceRef: { origin: 'dream-extract' },
+    });
+
+    if (id) {
+      extracted++;
+    } else {
+      skipped++;
+    }
+  }
+
+  // Update watermark to last processed message
+  const lastMsg = batch[batch.length - 1];
+  if (lastMsg) {
+    const newWm = {
+      lastMessageId: lastMsg.id || '',
+      lastMessageTs: lastMsg.ts || Date.now(),
+    };
+    writeWatermark(storeDir, newWm);
+    return { extracted, skipped, watermark: newWm };
+  }
+
+  return { extracted, skipped, watermark: wm };
+}

package/unify/session.js

@@ -34,6 +34,7 @@ import { initInputQueueStore } from './input-queue/store.js';
 import { createDispatcher } from './pipeline/dispatcher.js';
 import { ensureDefaultGroupIfEmpty } from './groups/group-crud.js';
 import { createDreamScheduler } from './memory/dream-scheduler.js';
+import { getUserMemoryStore } from './memory/user-memory-store.js';
 import { join } from 'path';
 import { existsSync as existsSyncSafe, readFileSync as readFileSyncSafe } from 'fs';
 
@@ -236,6 +237,8 @@ export async function loadSession(options = {}) {
   // ─── 9a. Create dream scheduler (wave-6b) ─────────────
   const dreamScheduler = createDreamScheduler({
     memoryShardStore,
+    userMemoryStore: getUserMemoryStore(),
+    conversationStore,
     adapter,
     config,
     onDreamStart: (vpId) => {