create-walle 0.9.21 → 0.9.23

package/template/claude-task-manager/lib/session-stream.js

@@ -3,7 +3,8 @@ const crypto = require('crypto');
 const fsp = require('fs/promises');
 const EventEmitter = require('events');
 const { looksLikeCompactContinuation } = require('./compact-stitch');
-const { codexMessageFromEntry, codexUserKey } = require('./session-history');
+const { codexMessageFromEntry, createCodexUserDeduper } = require('./session-history');
+const { isProviderGeneratedUserContextText } = require('./provider-user-context');
 
 const MAX_READ_BYTES = 1024 * 1024; // 1MB cap per incremental read
 const MAX_TEXT_LEN = 50 * 1024;     // 50KB text cap per event
@@ -136,10 +137,17 @@ class JsonlTailer {
 function extractText(content) {
   if (typeof content === 'string') return content;
   if (!Array.isArray(content)) return '';
-  return content
-    .filter(b => b.type === 'text')
-    .map(b => b.text)
-    .join('\n');
+  const parts = [];
+  let imageCount = 0;
+  for (const block of content) {
+    if (!block || typeof block !== 'object') continue;
+    if ((block.type === 'text' || block.type === 'input_text' || block.type === 'output_text') && block.text) {
+      parts.push(block.text);
+    } else if (block.type === 'image' || block.type === 'input_image' || block.type === 'image_ref') {
+      parts.push(`[Image #${++imageCount}]`);
+    }
+  }
+  return parts.join('\n');
 }
 
 function stripBase64Blobs(contentBlocks) {
@@ -522,7 +530,7 @@ class SessionStream extends EventEmitter {
       jsonlPath: jsonlPath || null,
       ring: [],
       userPromptCache: [],  // Last 10 cleaned user text prompts (survives ring buffer churn)
-      codexUserSeen: new Set(),
+      codexUserDeduper: createCodexUserDeduper(),
       seq: 0,
       lastActivity: 0,
       lastPtyActivity: 0,
@@ -641,6 +649,7 @@ class SessionStream extends EventEmitter {
       const content = entry.message.content;
       let text = extractText(content);
       if (text.length > MAX_TEXT_LEN) text = text.slice(0, MAX_TEXT_LEN) + `...truncated (${text.length} chars)`;
+      if (isProviderGeneratedUserContextText(text)) return;
 
       const evt = decorateStreamEvent({
         type: 'user',
@@ -829,9 +838,8 @@ class SessionStream extends EventEmitter {
     if (!msg) return false;
 
     if (msg.role === 'user') {
-      const key = codexUserKey(msg.text);
-      if (!key || st.codexUserSeen.has(key)) return true;
-      st.codexUserSeen.add(key);
+      if (!st.codexUserDeduper) st.codexUserDeduper = createCodexUserDeduper();
+      if (!st.codexUserDeduper.remember(msg.text, msg.timestamp)) return true;
       const timestamp = eventTimestampMs({ timestamp: msg.timestamp });
       const evt = decorateStreamEvent({
         type: 'user',
@@ -875,6 +883,9 @@ class SessionStream extends EventEmitter {
       newStatus,
       reason,
       timestamp: Date.now(),
+      lastActivity: st.lastActivity || 0,
+      lastPtyActivity: st.lastPtyActivity || 0,
+      lastJsonlActivity: st.lastJsonlActivity || 0,
     };
     this.emit('status', statusEvt);
   }
@@ -952,9 +963,15 @@ class SessionStream extends EventEmitter {
       const sourceIds = typeof this._dbModule.getSessionConversationSourceIds === 'function'
         ? this._dbModule.getSessionConversationSourceIds(sessionId, { includeCtmFallback: true })
         : [sessionId];
+      // Prefer the blob-free meta row — the summary path only parses `messages` when the
+      // hot ring is incomplete (lazy via _dbMessagesForRow), so pulling the multi-MB blob
+      // here on every broadcast was pure waste (~20s of boot CPU, profiled).
+      const fetchRow = typeof this._dbModule.getSessionConversationMeta === 'function'
+        ? (id) => this._dbModule.getSessionConversationMeta(id)
+        : (id) => this._dbModule.getSessionConversation(id);
       return sourceIds
-        .map(sourceId => this._dbModule.getSessionConversation(sourceId))
-        .find(row => row && (row.last_user_content || row.messages || row.first_message)) || null;
+        .map(sourceId => fetchRow(sourceId))
+        .find(row => row && (row.last_user_content || row.messages_len || row.messages || row.first_message)) || null;
     } catch {
       return null;
     }
@@ -969,6 +986,29 @@ class SessionStream extends EventEmitter {
     return 0;
   }
 
+  // Resolve the `messages` JSON blob for a summary row, loading it lazily. A blob-free
+  // meta row (from getSessionConversationMeta) carries messages_len but not messages, so
+  // we fetch the blob on demand here — only when the caller actually needs the turns.
+  _dbMessagesForRow(row) {
+    if (!row || typeof row !== 'object') return null;
+    if (row.messages) return row.messages; // legacy/full row or test mock
+    // Memoize per row so the prompt-entries and turns paths share one blob load per
+    // getSummary (a meta row from getSessionConversationMeta has messages_len but no blob).
+    if (Object.prototype.hasOwnProperty.call(row, '_lazyMessages')) return row._lazyMessages;
+    if (!row.messages_len) { row._lazyMessages = null; return null; }
+    const id = row.ctm_session_id;
+    let blob = null;
+    if (id && this._dbModule) {
+      if (typeof this._dbModule.getSessionConversationMessages === 'function') {
+        try { blob = this._dbModule.getSessionConversationMessages(id) || null; } catch { blob = null; }
+      } else if (typeof this._dbModule.getSessionConversation === 'function') {
+        try { blob = this._dbModule.getSessionConversation(id)?.messages || null; } catch { blob = null; }
+      }
+    }
+    row._lazyMessages = blob;
+    return blob;
+  }
+
   _latestTurnTimestamp(turns) {
     let latest = 0;
     for (const turn of Array.isArray(turns) ? turns : []) {
@@ -979,7 +1019,7 @@ class SessionStream extends EventEmitter {
   }
 
   _shouldLoadDbTurnsForSummary(ringTurns, turns, dbRow) {
-    if (!dbRow || !dbRow.messages) return Array.isArray(ringTurns) ? ringTurns.length < turns : true;
+    if (!dbRow || !(dbRow.messages_len || dbRow.messages)) return Array.isArray(ringTurns) ? ringTurns.length < turns : true;
     if (!Array.isArray(ringTurns) || ringTurns.length < turns) return true;
     const dbTimestamp = this._dbRowTimestamp(dbRow);
     const ringTimestamp = this._latestTurnTimestamp(ringTurns);
@@ -998,9 +1038,15 @@ class SessionStream extends EventEmitter {
       if (entry) entries.push(entry);
     };
 
-    if (row.messages) {
+    // Reuse the blob only if it was ALREADY loaded for this summary (the turns path
+    // loads it when the hot ring is incomplete and memoizes it on the row). Don't
+    // trigger a fresh multi-MB load just for the prompt ledger — last_user_content
+    // carries the same text below, and the hot prompt cache is the authority when the
+    // ring is rich. This keeps the blob to at most one load per getSummary.
+    const rowMessages = row.messages || row._lazyMessages || null;
+    if (rowMessages) {
       try {
-        const dbMessages = JSON.parse(row.messages);
+        const dbMessages = JSON.parse(rowMessages);
         if (Array.isArray(dbMessages)) {
           for (let i = dbMessages.length - 1; i >= 0; i--) {
             const m = dbMessages[i];
@@ -1027,10 +1073,11 @@ class SessionStream extends EventEmitter {
 
   _dbTurnsForSummary(sessionId, limit = 6, dbRow) {
     if (arguments.length < 3) dbRow = this._dbConversationRowForSummary(sessionId);
-    if (!dbRow || !dbRow.messages) return [];
+    const rowMessages = this._dbMessagesForRow(dbRow);
+    if (!rowMessages) return [];
     try {
       const maxTurns = Math.max(1, Number(limit) || 6);
-      const dbMessages = JSON.parse(dbRow.messages);
+      const dbMessages = JSON.parse(rowMessages);
       if (!Array.isArray(dbMessages)) return [];
       return dbMessages
         .filter(m => m && (m.role === 'user' || m.role === 'assistant'))
@@ -1162,6 +1209,7 @@ class SessionStream extends EventEmitter {
    *  filters framework preamble. */
   _cleanUserText(text) {
     if (!text) return '';
+    if (isProviderGeneratedUserContextText(text)) return '';
 
     // /compact synthetic user message: discard the entire block, not just the
     // first preamble line. The body that follows ("Summary:\n1. Primary...")

package/template/claude-task-manager/lib/session-timeline-summary.js

@@ -0,0 +1,260 @@
+'use strict';
+
+const MESSAGE_ROLES = new Set(['user', 'assistant']);
+const RECENT_TURN_LIMIT = 6;
+
+function _text(value) {
+  return String(value ?? '').replace(/\s+/g, ' ').trim();
+}
+
+function _messageText(message) {
+  return _text(message?.text ?? message?.content ?? message?.message ?? '');
+}
+
+function _timestampMs(value) {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  const parsed = Date.parse(String(value || ''));
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function _timestampIso(value) {
+  const ms = _timestampMs(value);
+  return ms ? new Date(ms).toISOString() : '';
+}
+
+function _cleanUserPrompt(text) {
+  const cleaned = _text(text)
+    .replace(/<image\b[\s\S]*?<\/image>\s*/gi, '')
+    .replace(/\[input_image\]\s*/g, '')
+    .trim();
+  if (!cleaned) return '';
+  if (/^this session is being continued from a previous conversation/i.test(cleaned)) return '';
+  return cleaned;
+}
+
+function _isAckOnlyPrompt(text) {
+  const normalized = _text(text).toLowerCase();
+  return /^(ok|okay|yes|yep|go ahead|continue|thanks|thank you|done|sounds good|looks good)$/i.test(normalized);
+}
+
+function _isContentRichPrompt(text) {
+  const cleaned = _cleanUserPrompt(text);
+  if (!cleaned || _isAckOnlyPrompt(cleaned)) return false;
+  if (cleaned.length >= 24) return true;
+  if (/[$/@#]|https?:\/\/|\b(fix|debug|review|implement|test|commit|merge|explain|why|how|what|where|find|update|design|use)\b/i.test(cleaned)) return true;
+  return false;
+}
+
+function _promptPayload(entry, source, freshness) {
+  if (!entry?.text) return null;
+  return {
+    text: entry.text,
+    source,
+    freshness,
+    confidence: _isContentRichPrompt(entry.text) ? 'medium' : 'low',
+    timestamp: entry.timestamp,
+    updatedAt: entry.timestamp,
+    isContentRich: _isContentRichPrompt(entry.text),
+  };
+}
+
+function _summaryActivityMs(summary) {
+  if (!summary || typeof summary !== 'object') return 0;
+  const values = [
+    summary.lastActivity,
+    summary.progress?.updatedAt,
+    summary.currentTask?.updatedAt,
+    summary.currentTask?.timestamp,
+    summary.currentTask?.promptTimestamp,
+    summary.intent?.updatedAt,
+    summary.latestPrompt?.updatedAt,
+    summary.latestPrompt?.timestamp,
+    summary.displayPrompt?.updatedAt,
+    summary.displayPrompt?.timestamp,
+    summary.aiSummary?.updatedAt,
+  ];
+  return values.reduce((latest, value) => Math.max(latest, _timestampMs(value)), 0);
+}
+
+function _summaryPromptMs(summary) {
+  if (!summary || typeof summary !== 'object') return 0;
+  return Math.max(
+    _timestampMs(summary.latestPrompt?.timestamp || summary.latestPrompt?.updatedAt),
+    _timestampMs(summary.displayPrompt?.timestamp || summary.displayPrompt?.updatedAt),
+    _timestampMs(summary.currentTask?.promptTimestamp || summary.currentTask?.timestamp),
+    _timestampMs(summary.intent?.timestamp || summary.intent?.updatedAt)
+  );
+}
+
+function _summaryProgressMs(summary) {
+  if (!summary || typeof summary !== 'object') return 0;
+  return Math.max(
+    _timestampMs(summary.progress?.updatedAt),
+    _timestampMs(summary.lastActivity)
+  );
+}
+
+function _hasPrompt(summary) {
+  return !!_text(summary?.latestPrompt?.text || summary?.displayPrompt?.text || summary?.lastPrompt || summary?.currentTask?.text);
+}
+
+function _hasProgress(summary) {
+  return !!_text(summary?.progress?.summary || summary?.progress?.text || summary?.progress || summary?.summary);
+}
+
+function summarizeMessagesForTimeline(messages, options = {}) {
+  const source = options.source || 'conversation-timeline';
+  const freshness = options.freshness || 'durable';
+  const sessionId = String(options.sessionId || '').trim();
+  const ctmSessionId = String(options.ctmSessionId || sessionId || '').trim();
+  const agentSessionId = String(options.agentSessionId || '').trim() || null;
+  const turns = [];
+
+  for (const message of Array.isArray(messages) ? messages : []) {
+    const role = String(message?.role || '').trim();
+    if (!MESSAGE_ROLES.has(role)) continue;
+    const text = role === 'user' ? _cleanUserPrompt(_messageText(message)) : _messageText(message);
+    if (!text) continue;
+    const timestamp = _timestampMs(message?.timestamp || message?.created_at || message?.createdAt || message?.time);
+    const turn = {
+      role,
+      text,
+      timestamp,
+      source: message?.source || source,
+    };
+    turns.push(turn);
+  }
+
+  turns.sort((a, b) => {
+    if (a.timestamp && b.timestamp && a.timestamp !== b.timestamp) return a.timestamp - b.timestamp;
+    if (a.timestamp && !b.timestamp) return -1;
+    if (!a.timestamp && b.timestamp) return 1;
+    return 0;
+  });
+  const userEntries = turns.filter(turn => turn.role === 'user');
+  const assistantEntries = turns.filter(turn => turn.role === 'assistant');
+
+  const latestUser = userEntries[userEntries.length - 1] || null;
+  const displayUser = [...userEntries].reverse().find(entry => _isContentRichPrompt(entry.text)) || latestUser;
+  const taskUser = latestUser && _isContentRichPrompt(latestUser.text) ? latestUser : displayUser;
+  const latestAssistantAfterTask = [...assistantEntries]
+    .reverse()
+    .find(entry => !taskUser?.timestamp || !entry.timestamp || entry.timestamp >= taskUser.timestamp)
+    || assistantEntries[assistantEntries.length - 1]
+    || null;
+  const latestActivity = turns.reduce((latest, turn) => Math.max(latest, turn.timestamp || 0), 0);
+  const latestPrompt = _promptPayload(latestUser, source, freshness);
+  const displayPrompt = _promptPayload(displayUser, source, freshness);
+  const currentTask = _promptPayload(taskUser, source, freshness) || {
+    text: null,
+    source,
+    freshness: 'missing',
+    confidence: 'low',
+    timestamp: 0,
+    updatedAt: 0,
+    promptTimestamp: 0,
+  };
+  if (currentTask?.text) {
+    currentTask.prompt = currentTask.text;
+    currentTask.fullPrompt = currentTask.text;
+    currentTask.promptTimestamp = currentTask.timestamp || 0;
+  }
+
+  const progress = latestAssistantAfterTask ? {
+    phase: '',
+    summary: latestAssistantAfterTask.text,
+    bullets: [latestAssistantAfterTask.text],
+    next: null,
+    source,
+    updatedAt: latestAssistantAfterTask.timestamp || latestActivity || 0,
+  } : {
+    phase: '',
+    summary: null,
+    bullets: [],
+    next: null,
+    source: 'none',
+    updatedAt: 0,
+  };
+
+  return {
+    sessionId: agentSessionId || sessionId,
+    ctmSessionId,
+    agentSessionId,
+    summary: null,
+    summarySource: null,
+    lastPrompt: latestPrompt?.text || '',
+    displayPrompt: displayPrompt?.text || '',
+    latestPrompt,
+    currentTask,
+    intent: currentTask?.text ? {
+      text: currentTask.text,
+      source: currentTask.source,
+      freshness: currentTask.freshness,
+      confidence: currentTask.confidence,
+      prompt: currentTask.text,
+      fullPrompt: currentTask.text,
+      timestamp: currentTask.timestamp || 0,
+      updatedAt: currentTask.updatedAt || 0,
+    } : null,
+    progress,
+    recentTurns: turns.slice(-RECENT_TURN_LIMIT).map(turn => ({
+      role: turn.role,
+      text: turn.text,
+      timestamp: turn.timestamp ? _timestampIso(turn.timestamp) : '',
+      source: turn.source,
+    })),
+    turnCount: turns.length,
+    lastActivity: latestActivity,
+    timelineSource: source,
+    timelineFreshness: freshness,
+  };
+}
+
+function mergeTimelineSummaries(primary, timeline) {
+  if (!primary) return timeline || null;
+  if (!timeline) return primary || null;
+
+  const merged = { ...primary };
+  const primaryPromptMs = _summaryPromptMs(primary);
+  const timelinePromptMs = _summaryPromptMs(timeline);
+  const timelinePromptWins = timelinePromptMs && (!primaryPromptMs || timelinePromptMs >= primaryPromptMs || !_hasPrompt(primary));
+  const primaryAiFresh = primary?.aiSummary?.status === 'fresh'
+    && _timestampMs(primary.aiSummary.updatedAt) >= timelinePromptMs;
+
+  if (timelinePromptWins) {
+    merged.lastPrompt = timeline.lastPrompt || merged.lastPrompt;
+    merged.displayPrompt = timeline.displayPrompt || merged.displayPrompt;
+    merged.latestPrompt = timeline.latestPrompt || merged.latestPrompt;
+    if (!primaryAiFresh) {
+      merged.currentTask = timeline.currentTask || merged.currentTask;
+      merged.intent = timeline.intent || merged.intent;
+    }
+  }
+
+  const primaryProgressMs = _summaryProgressMs(primary);
+  const timelineProgressMs = _summaryProgressMs(timeline);
+  if (timelineProgressMs && (!primaryProgressMs || timelineProgressMs >= primaryProgressMs || !_hasProgress(primary))) {
+    merged.progress = timeline.progress || merged.progress;
+  }
+
+  if ((!Array.isArray(primary.recentTurns) || primary.recentTurns.length === 0) && Array.isArray(timeline.recentTurns)) {
+    merged.recentTurns = timeline.recentTurns;
+  }
+  const latestActivity = Math.max(_summaryActivityMs(primary), _summaryActivityMs(timeline));
+  if (latestActivity) merged.lastActivity = latestActivity;
+  merged.timelineSource = timeline.timelineSource || merged.timelineSource;
+  merged.timelineFreshness = timeline.timelineFreshness || merged.timelineFreshness;
+  return merged;
+}
+
+module.exports = {
+  summarizeMessagesForTimeline,
+  mergeTimelineSummaries,
+  _private: {
+    _cleanUserPrompt,
+    _isContentRichPrompt,
+    _summaryActivityMs,
+    _summaryPromptMs,
+    _summaryProgressMs,
+  },
+};