osai-agent 4.0.0

package/src/agent/loop/stream-parser.js

@@ -0,0 +1,515 @@
+import { TOOLS, COMPLETION_SIGNALS, MODES } from '../../utils/constants.js';
+import { logger } from '../../utils/logger.js';
+
+export default {
+  async _parseResponse(response) {
+    if (!response || typeof response !== 'object' || !response.body) {
+      if (response && response.toolCalls) return response;
+      return { toolCalls: [], fullResponse: '' };
+    }
+
+    const decoder = new TextDecoder();
+    let lineBuffer = '';
+    let fullResponse = '';
+    let completionSignal = null;
+    let badgeDisplayed = false;
+    let narrativeBuffer = '';
+    let jsonAccumulator = '';
+    let inJsonMode = false;
+    let inToolXmlMode = false;
+    let toolXmlAccumulator = '';
+    let inCodePreview = false;
+    let braceDepth = 0;
+    let pendingToolJsonFence = false;
+    let inThinkBlock = false;
+    let thinkContentAccumulator = '';
+
+    const reader = response.body.getReader();
+
+    // Yield rapide via setImmediate pour ne pas bloquer l'event loop
+    const quickYield = () => new Promise(r => setImmediate(r));
+    // Yield de rendu via setTimeout pour laisser Ink re-render
+    // en passant par la phase Timers du event loop
+    const renderYield = () => new Promise(r => setTimeout(r, 0));
+
+    const processLines = async (lines) => {
+      let shouldBreak = false;
+      let lineCount = 0;
+      for (const line of lines) {
+        lineCount++;
+        // Yield de rendu toutes les 25 lignes pour laisser Ink re-render
+        if (lineCount % 25 === 0) {
+          await renderYield();
+          if (this._cancelled) break;
+        // Yield rapide toutes les 5 lignes pour éviter la starvation
+        } else if (lineCount % 5 === 0) {
+          await quickYield();
+          if (this._cancelled) break;
+        }
+        if (!line.startsWith('data: ')) continue;
+        const data = line.slice(6).trim();
+        if (!data) continue;
+        if (data === '[DONE]') { shouldBreak = true; continue; }
+
+        try {
+          const event = JSON.parse(data);
+          if (event.type === 'thinking' && event.content) {
+            if (!this._isThinking) {
+              this._isThinking = true;
+              this.onThinkingStart();
+            }
+            this.onThought(event.content);
+          } else if (event.type === 'chunk' && event.content) {
+            if (event.provider) this.currentProvider = event.provider;
+            if (event.model) this.currentModel = event.model;
+            if (this._isThinking) {
+              const trimmed = event.content.trim();
+              const isToolJson = trimmed.startsWith('{"tool"') || trimmed.startsWith('{\\"tool\\"');
+              if (!isToolJson) {
+                this._isThinking = false;
+                this.onThinkingEnd();
+              }
+            }
+            const content = event.content;
+            fullResponse += content;
+
+            const trimmedChunk = content.trim();
+            const hasToolJsonStart = content.includes('{"tool"') || content.includes('{\\"tool\\"');
+            const hasToolLikeJsonKeys = /"(?:tool|server|mcpTool|params|path|cmd|url|query|find|replace|content|description|startLine|endLine|filePattern|pattern|source|destination)"\s*:/i.test(content)
+              || /\\"(?:tool|server|mcpTool|params|path|cmd|url|query|find|replace|content|description|startLine|endLine|filePattern|pattern|source|destination)\\"\s*:/i.test(content);
+            const trimmedStart = content.trimStart();
+            const startsLikeJsonObject = trimmedStart.startsWith('{') || trimmedStart.startsWith('{\\"') || trimmedStart.startsWith('{"');
+            const looksLikeSplitToolJsonStart = this.mode === MODES.CODING && startsLikeJsonObject && !content.includes('\n\n');
+            const isStandaloneJsonFence = /^```json\s*$/i.test(trimmedChunk);
+            const looksLikeNonJsonToolCall = /^[A-Z][A-Z_0-9]+\s+\w+\s*=/.test(trimmedChunk);
+            const hasToolXmlStart = content.includes('<tool') && (content.includes('>') || content.includes('/>'));
+
+            // Detect <think> blocks and route to thought handler
+            // Must run before tool XML/JSON mode checks so think content
+            // is not consumed by tool accumulators.
+            const thinkOpenIdx = content.indexOf('<think>');
+            const thinkCloseIdx = content.indexOf('</think>');
+            if (inThinkBlock || thinkOpenIdx !== -1) {
+              if (inThinkBlock) {
+                // Already inside a think block
+                if (thinkCloseIdx !== -1) {
+                  const beforeClose = content.slice(0, thinkCloseIdx);
+                  if (beforeClose.trim()) {
+                    thinkContentAccumulator += beforeClose;
+                    this.onThought(beforeClose);
+                  }
+                  inThinkBlock = false;
+                  thinkContentAccumulator = '';
+                  this.onThinkingEnd();
+                  content = content.slice(thinkCloseIdx + 8);
+                } else {
+                  thinkContentAccumulator += content;
+                  this.onThought(content);
+                  content = '';
+                }
+              } else {
+                // Think block starts in this chunk
+                const beforeThink = content.slice(0, thinkOpenIdx);
+                const afterThink = content.slice(thinkOpenIdx + 7);
+                inThinkBlock = true;
+                thinkContentAccumulator = '';
+                this.onThinkingStart();
+                const closeInRemainder = afterThink.indexOf('</think>');
+                if (closeInRemainder !== -1) {
+                  const thinkContent = afterThink.slice(0, closeInRemainder);
+                  if (thinkContent.trim()) {
+                    thinkContentAccumulator = thinkContent;
+                    this.onThought(thinkContent);
+                  }
+                  inThinkBlock = false;
+                  thinkContentAccumulator = '';
+                  this.onThinkingEnd();
+                  content = (beforeThink || '') + afterThink.slice(closeInRemainder + 8);
+                } else {
+                  if (afterThink.trim()) {
+                    thinkContentAccumulator = afterThink;
+                    this.onThought(afterThink);
+                  }
+                  content = beforeThink || '';
+                }
+              }
+            }
+
+            if (!inJsonMode && looksLikeNonJsonToolCall && narrativeBuffer.trim()) {
+              this.onMarkdown(narrativeBuffer.trim());
+              narrativeBuffer = '';
+              shouldBreak = true;
+              break;
+            }
+
+            if (!inJsonMode && isStandaloneJsonFence) {
+              pendingToolJsonFence = true;
+              continue;
+            }
+            if (!inJsonMode && pendingToolJsonFence && !(hasToolJsonStart || hasToolLikeJsonKeys || looksLikeSplitToolJsonStart)) {
+              narrativeBuffer += '```json\n';
+              pendingToolJsonFence = false;
+            }
+            if (!inJsonMode && (hasToolJsonStart || hasToolLikeJsonKeys || looksLikeSplitToolJsonStart)) {
+              pendingToolJsonFence = false;
+              inJsonMode = true;
+            if (narrativeBuffer) { this.onMarkdown(narrativeBuffer); narrativeBuffer = ''; }
+              const rawIdx = content.indexOf('{"tool"');
+              const escapedIdx = content.indexOf('{\\"tool\\"');
+              const braceIdx = content.indexOf('{');
+              let toolStart = rawIdx >= 0 && escapedIdx >= 0 ? Math.min(rawIdx, escapedIdx) : Math.max(rawIdx, escapedIdx);
+              if (toolStart < 0) toolStart = braceIdx >= 0 ? braceIdx : 0;
+              jsonAccumulator = content.slice(toolStart);
+              braceDepth = (jsonAccumulator.match(/{/g) || []).length - (jsonAccumulator.match(/}/g) || []).length;
+              if (braceDepth <= 0) {
+                inJsonMode = false;
+                shouldBreak = true;
+                break;
+              }
+            } else if (inJsonMode) {
+              jsonAccumulator += content;
+              braceDepth += (content.match(/{/g) || []).length - (content.match(/}/g) || []).length;
+              if (braceDepth <= 0) {
+                inJsonMode = false;
+                shouldBreak = true;
+                break;
+              }
+            } else if (inToolXmlMode) {
+              toolXmlAccumulator += content;
+              if (content.includes('/>') || content.includes('</tool>')) {
+                inToolXmlMode = false;
+                shouldBreak = true;
+                break;
+              }
+            } else if (!inJsonMode && !inToolXmlMode && hasToolXmlStart) {
+              if (narrativeBuffer) { this.onMarkdown(narrativeBuffer); narrativeBuffer = ''; }
+              inToolXmlMode = true;
+              toolXmlAccumulator = content;
+              if (content.includes('/>') || content.includes('</tool>')) {
+                inToolXmlMode = false;
+                shouldBreak = true;
+                break;
+              }
+            } else {
+              if (content.includes('[DONE]')) completionSignal = COMPLETION_SIGNALS.DONE;
+              if (content.includes('[INCOMPLETE]')) completionSignal = COMPLETION_SIGNALS.INCOMPLETE;
+              if (content.includes('[BLOCKED]')) completionSignal = COMPLETION_SIGNALS.BLOCKED;
+
+              let cleanContent = content
+                .replace(/\[DONE\]/gi, '').replace(/\[INCOMPLETE\]/gi, '').replace(/\[BLOCKED\]/gi, '')
+                .replace(/\[TOOL_CALL\]\s*/g, '')
+                .replace(/THE TOOL\s*CALL[\s\S]*?(?=\n\n|$)/gi, '');
+
+              const trailingJsonFence = cleanContent.match(/(?:^|\n)\s*```json\s*$/i);
+              if (trailingJsonFence && trailingJsonFence.index != null) {
+                cleanContent = cleanContent.slice(0, trailingJsonFence.index);
+                pendingToolJsonFence = true;
+              }
+
+              if (!inCodePreview && cleanContent) {
+                const fileContentScore = [
+                  /;\s*\n/.test(cleanContent),
+                  /\{\s*\n/.test(cleanContent),
+                  /background-color\s*:/.test(cleanContent),
+                  /display\s*:\s*(flex|grid|block)/.test(cleanContent),
+                  /font-size\s*:/.test(cleanContent),
+                  /margin|padding/.test(cleanContent) && /:\s*\d/.test(cleanContent),
+                  /<\/(div|html|body|head|script|style|button)>/.test(cleanContent),
+                  /\bfunction\s+\w+\s*\(/.test(cleanContent),
+                  /document\.getElementById/.test(cleanContent),
+                  /\\n\s{2,}/.test(cleanContent),
+                ].filter(Boolean).length;
+
+                if (fileContentScore >= 3) {
+                  const sentences = cleanContent.split(/(?<=[.!?])\s+/);
+                  let safeNarrative = '';
+                  for (const s of sentences) {
+                    if ([/;\s*\n/, /background-color/, /display\s*:/, /<\/(div|html)>/, /\\n\s{2,}/]
+                          .some(re => re.test(s))) break;
+                    safeNarrative += s + ' ';
+                  }
+                  if (safeNarrative.trim()) { narrativeBuffer += safeNarrative.trim() + ' '; }
+                  inCodePreview = true;
+                  cleanContent = '';
+                  setImmediate(() => { inCodePreview = false; });
+                }
+              }
+
+              if (cleanContent.includes('{"tool"') || cleanContent.includes('{\\"tool\\"') || /^\s*\{\s*$/.test(cleanContent) || /{"\s*$/.test(cleanContent)) {
+                const rawToolIdx = cleanContent.indexOf('{"tool"');
+                const escapedToolIdx = cleanContent.indexOf('{\\"tool\\"');
+                const braceIdx = cleanContent.indexOf('{');
+                let toolIdx = rawToolIdx >= 0 && escapedToolIdx >= 0 ? Math.min(rawToolIdx, escapedToolIdx) : Math.max(rawToolIdx, escapedToolIdx);
+                if (toolIdx < 0) toolIdx = braceIdx >= 0 ? braceIdx : 0;
+                const before = cleanContent.slice(0, toolIdx).trim();
+                if (before) { narrativeBuffer += before; }
+                cleanContent = '';
+              }
+              if (this.mode === MODES.CODING && cleanContent) {
+                const pseudoToolIdx = cleanContent.search(/\b(?:LOCAL_CMD|SSH_CMD|READ_FILE|WRITE_FILE|EDIT_FILE|APPEND_FILE|DELETE_FILE|LIST_DIR|SEARCH_FILE|CREATE_DIR|TREE_VIEW|RUN_SCRIPT|MOVE_FILE|COPY_FILE|FILE_INFO|FETCH_URL|WEB_SEARCH|TODO_ADD|TODO_COMPLETE|TODO_UPDATE|TODO_LIST|TODO_CLEAR|POWERSHELL|GLOB|GREP|GIT|DIAG_POST_EDIT|ASK_USER|PLAN_MODE|SKILL_LIST|LOAD_SKILL|CREATE_SKILL|TASK)\b\s*\n\s*\{\s*"?(?:path|cmd|url|query|find|replace|content|description|startLine|endLine|filePattern|pattern|source|destination|name|skill|prompt)"?\s*:/i);
+                if (pseudoToolIdx >= 0) {
+                  const before = cleanContent.slice(0, pseudoToolIdx).trim();
+                  if (before) narrativeBuffer += before + ' ';
+                  cleanContent = '';
+                }
+                const leakIdx = cleanContent.search(/"(?:tool|server|mcpTool|params|path|find|replace|content|description|cmd|url|query|startLine|endLine|filePattern|pattern|source|destination)"\s*:/i);
+                if (leakIdx >= 0) {
+                  const before = cleanContent.slice(0, leakIdx).trim();
+                  if (before) narrativeBuffer += before + ' ';
+                  cleanContent = '';
+                }
+              }
+
+              if (cleanContent) {
+                narrativeBuffer += cleanContent;
+                // Flush moins fr\u00E9quent (1000 car au lieu de 300) pour r\u00E9duire le nombre
+                // de re-rendus Ink pendant le streaming \u2192 terminal plus r\u00E9actif
+                const shouldFlush = /[.!?]\s+[A-Z\u00C0-\u024F]/.test(narrativeBuffer) || narrativeBuffer.includes('\n\n') || narrativeBuffer.length > 1000;
+                if (shouldFlush) {
+                  this.onMarkdown(narrativeBuffer);
+                  narrativeBuffer = '';
+                  // Yield de rendu après flush pour laisser Ink afficher
+                  await renderYield();
+                }
+              }
+            }
+          } else if (event.type === 'done') {
+            if (this._isThinking) {
+              this._isThinking = false;
+              this.onThinkingEnd();
+            }
+            if (inThinkBlock) {
+              inThinkBlock = false;
+              thinkContentAccumulator = '';
+              this.onThinkingEnd();
+            }
+            if (narrativeBuffer.trim()) { this.onMarkdown(narrativeBuffer.trim()); narrativeBuffer = ''; }
+            if (!badgeDisplayed) { this._showCompletionBadge(completionSignal); badgeDisplayed = true; }
+            shouldBreak = true;
+          } else if (event.type === 'error') {
+            if (this._cancelled) {
+              shouldBreak = true;
+              continue;
+            }
+            const streamErr = new Error(event.message || 'Worker stream error');
+            streamErr.name = 'WorkerStreamError';
+            throw streamErr;
+          }
+        } catch (e) {
+          if (e.name === 'WorkerStreamError') throw e;
+          continue;
+        }
+      }
+      return shouldBreak;
+    };
+
+    try {
+      while (true) {
+        if (this._cancelled) break;
+        const { done, value } = await reader.read();
+        if (done) break;
+        const chunk = decoder.decode(value, { stream: true });
+        lineBuffer += chunk;
+        const lines = lineBuffer.split('\n');
+        lineBuffer = lines.pop() || '';
+
+        if (await processLines(lines)) {
+          reader.cancel().catch(() => {});
+          break;
+        }
+
+        // Yield de rendu entre les chunks pour laisser Ink re-render
+        await renderYield();
+      }
+      if (lineBuffer.trim()) {
+        await processLines([lineBuffer]);
+      }
+    } catch (err) {
+      if (err.name === 'WorkerStreamError') throw err;
+      logger.error('Stream parsing error', { error: err.message });
+    }
+
+    // Clean up any unclosed think block
+    if (inThinkBlock) {
+      inThinkBlock = false;
+      thinkContentAccumulator = '';
+      this.onThinkingEnd();
+    }
+
+    // Libérer l'event loop avant le cleanup final synchrone
+    await renderYield();
+
+    if (narrativeBuffer.trim()) {
+      const safe = narrativeBuffer
+        .replace(/\{"tool"[\s\S]*$/, '')
+        .replace(/\{\\"tool\\"[\s\S]*$/, '')
+        .replace(/\{"tool"[^}]*\}/g, '')
+        .replace(/\{\\"tool\\"[^}]*\}/g, '')
+        .replace(/<tool\b[^>]*>[\s\S]*?<\/tool\s*>|<tool\b[^>]*\/>/g, '')
+        .replace(
+          /(?:^|\n)\s*(?:LOCAL_CMD|SSH_CMD|READ_FILE|WRITE_FILE|EDIT_FILE|APPEND_FILE|DELETE_FILE|LIST_DIR|SEARCH_FILE|CREATE_DIR|TREE_VIEW|RUN_SCRIPT|MOVE_FILE|COPY_FILE|FILE_INFO|FETCH_URL|WEB_SEARCH|TODO_ADD|TODO_COMPLETE|TODO_UPDATE|TODO_LIST|TODO_CLEAR|POWERSHELL|GLOB|GREP|GIT|DIAG_POST_EDIT|ASK_USER|PLAN_MODE|SKILL_LIST|LOAD_SKILL|CREATE_SKILL|TASK)\s*\n\s*\{[\s\S]*?(?=\n\s*\n|$)/g,
+          '\n',
+        )
+        .replace(/(?:^|\n)\s*```json\s*(?=\n|$)/gi, '\n')
+        .replace(/"content"\s*:\s*"[\s\S]{200,}?"(?=[,}])/g, '')
+        .replace(/"(?:tool|server|mcpTool|params|path|find|replace|content|description|cmd|url|query|startLine|endLine|filePattern|pattern|source|destination)"\s*:\s*".*$/i, '')
+        .trim();
+      if (safe) this.onMarkdown(safe);
+    }
+    if (!badgeDisplayed) this._showCompletionBadge(completionSignal);
+    this._lastCompletionSignal = completionSignal;
+
+    const cleanResponse = fullResponse
+      .replace(/\[DONE\]/gi, '').replace(/\[INCOMPLETE\]/gi, '').replace(/\[BLOCKED\]/gi, '')
+      .replace(/^\s*\{(?:\\")?tool(?:\\")?\s*:\s*.*$/gim, '')
+      .replace(/<tool\b[^>]*>[\s\S]*?<\/tool\s*>|<tool\b[^>]*\/>/gim, '')
+      .trim();
+    if (cleanResponse) this._appendConversationMessage('assistant', cleanResponse);
+
+    // Libérer l'event loop avant le scan char-par-char _extractToolCalls
+    await renderYield();
+
+    const extractedToolCalls = this._extractToolCalls(fullResponse);
+    if (extractedToolCalls.length === 0 && jsonAccumulator.trim()) {
+      try {
+        const parsed = JSON.parse(jsonAccumulator.trim());
+        if (parsed && parsed.tool) {
+          if (parsed.tool === TOOLS.POWERSHELL) parsed.tool = TOOLS.LOCAL_CMD;
+          extractedToolCalls.push(parsed);
+        }
+      } catch {}
+    }
+    if (extractedToolCalls.length === 0 && toolXmlAccumulator.trim()) {
+      const xmlResult = this._extractToolCalls(toolXmlAccumulator);
+      extractedToolCalls.push(...xmlResult);
+    }
+    return { toolCalls: extractedToolCalls, fullResponse: cleanResponse };
+  },
+
+  _detectCompletionSignal(text) {
+    if (text.includes('[DONE]')) this._lastCompletionSignal = COMPLETION_SIGNALS.DONE;
+    else if (text.includes('[INCOMPLETE]')) this._lastCompletionSignal = COMPLETION_SIGNALS.INCOMPLETE;
+    else if (text.includes('[BLOCKED]')) this._lastCompletionSignal = COMPLETION_SIGNALS.BLOCKED;
+  },
+
+  _showCompletionBadge(signal) {
+    if (!signal) return;
+    this.onBadge({ signal, provider: this.currentProvider, model: this.currentModel });
+  },
+
+  _extractToolCalls(text) {
+    const toolCalls = [];
+    const toolNames = Object.values(TOOLS);
+    const toolSet = new Set(toolNames);
+
+    // 1. Parse JSON tool calls: {"tool":"NAME","key":"value"}
+    let i = 0;
+    const len = text.length;
+
+    while (i < len) {
+      const openBrace = text.indexOf('{', i);
+      if (openBrace === -1) break;
+
+      const ahead = text.slice(openBrace, openBrace + 300);
+      if (!ahead.includes('"tool"')) {
+        i = openBrace + 1;
+        continue;
+      }
+
+      let depth = 0;
+      let inString = false;
+      let escaped = false;
+      let closeIdx = -1;
+
+      for (let j = openBrace; j < len; j++) {
+        const ch = text[j];
+
+        if (escaped) { escaped = false; continue; }
+        if (ch === '\\' && inString) { escaped = true; continue; }
+        if (ch === '"' && !escaped) { inString = !inString; continue; }
+
+        if (!inString) {
+          if (ch === '{') depth++;
+          else if (ch === '}') {
+            depth--;
+            if (depth === 0) { closeIdx = j + 1; break; }
+          }
+        }
+      }
+
+      if (closeIdx !== -1) {
+        const candidate = text.slice(openBrace, closeIdx);
+        try {
+          const parsed = JSON.parse(candidate);
+          if (parsed && parsed.tool && toolSet.has(parsed.tool)) {
+            if (parsed.tool === TOOLS.POWERSHELL) parsed.tool = TOOLS.LOCAL_CMD;
+            toolCalls.push(parsed);
+          }
+        } catch {}
+        i = closeIdx;
+      } else {
+        i = openBrace + 1;
+      }
+    }
+
+    // 2. Parse XML-style tool calls: <tool>NAME key="value"</tool> or <tool name="NAME" key="value" />
+    if (toolCalls.length === 0) {
+      const xmlPattern = /<tool\b([^>]*)>([\s\S]*?)<\/tool\s*>|<tool\b([^>]*)\/>/gi;
+      let xmlMatch;
+      while ((xmlMatch = xmlPattern.exec(text)) !== null) {
+        const outerAttrs = xmlMatch[1] || xmlMatch[3] || '';
+        const innerContent = xmlMatch[2] || '';
+        let fullArgs = outerAttrs;
+        if (innerContent) fullArgs += ' ' + innerContent.trim();
+
+        const toolNameMatch = fullArgs.match(/^(?:name\s*=\s*["'])?([A-Z][A-Z_0-9]+)(?:["'])?\s*/);
+        if (!toolNameMatch) continue;
+        const toolName = toolNameMatch[1];
+        if (!toolSet.has(toolName)) continue;
+
+        const toolCall = { tool: toolName };
+        const argsStr = fullArgs.slice(toolNameMatch[0].length).trim();
+        const argRegex = /(\w+)\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+))/g;
+        let argMatch;
+        while ((argMatch = argRegex.exec(argsStr)) !== null) {
+          const value = argMatch[2] !== undefined ? argMatch[2]
+            : argMatch[3] !== undefined ? argMatch[3]
+            : argMatch[4];
+          if (value !== undefined) {
+            if (toolName === 'POWERSHELL') toolCall.tool = TOOLS.LOCAL_CMD;
+            toolCall[argMatch[1]] = value;
+          }
+        }
+        if (Object.keys(toolCall).length > 1) {
+          toolCalls.push(toolCall);
+        }
+      }
+    }
+
+    // 3. Legacy non-JSON fallback: NAME key="value" (no <tool> wrapper)
+    if (toolCalls.length === 0) {
+      const toolNamePattern = new RegExp(
+        `(?:^|\\n)\\s*(${toolNames.join('|')})\\s+([\\s\\S]*?)$`, 'im'
+      );
+      const toolMatch = text.match(toolNamePattern);
+      if (toolMatch) {
+        const toolName = toolMatch[1].toUpperCase();
+        if (toolSet.has(toolName)) {
+          const argsStr = toolMatch[2].trim();
+          const toolCall = { tool: toolName };
+          const argRegex = /(\w+)\s*=\s*(?:"([^"]*)"|(\S+))/g;
+          let argMatch;
+          while ((argMatch = argRegex.exec(argsStr)) !== null) {
+            const value = argMatch[2] !== undefined ? argMatch[2] : argMatch[3];
+            toolCall[argMatch[1]] = value;
+          }
+          if (Object.keys(toolCall).length > 1) {
+            toolCalls.push(toolCall);
+          }
+        }
+      }
+    }
+
+    return toolCalls;
+  },
+};