@semalt-ai/code 1.8.1 → 1.8.4

package/lib/agent.js

@@ -2,8 +2,14 @@
 
 const { logToolCall } = require('./audit');
 const { Metrics } = require('./metrics');
-const { SYSTEM_PROMPT } = require('./prompts');
+const { getSystemPrompt } = require('./prompts');
 const { TAG_REGISTRY } = require('./constants');
+const { mapInvokeToCall } = require('./tools');
+const { UI_THEME } = require('./ui/theme');
+const { RST } = require('./ui/ansi');
+const { getCols: _getCols, repeatToWidth } = require('./ui/utils');
+const writer = require('./ui/writer');
+const messages = require('./ui/messages');
 
 class StreamParser {
   constructor(onToken, onTagOpen, onTagContent, onTagClose) {
@@ -40,14 +46,18 @@ class StreamParser {
         const tagRaw = this.buffer.slice(1, gtIdx).trim();
         const selfClose = tagRaw.endsWith('/');
         const tagBody = selfClose ? tagRaw.slice(0, -1).trim() : tagRaw;
-        const spaceIdx = tagBody.search(/\s/);
-        const tagName = (spaceIdx === -1 ? tagBody : tagBody.slice(0, spaceIdx)).toLowerCase();
-        const attrStr = spaceIdx === -1 ? '' : tagBody.slice(spaceIdx + 1);
+        // Split on whitespace OR `=` so both MiniMax-style `<parameter name="x">`
+        // and Qwen3-Coder `<parameter=x>` resolve to the same tagName.
+        const delimIdx = tagBody.search(/[\s=]/);
+        const tagName = (delimIdx === -1 ? tagBody : tagBody.slice(0, delimIdx)).toLowerCase();
+        const attrStr = delimIdx === -1 ? '' : tagBody.slice(delimIdx + 1);
 
         const attrs = {};
-        const attrRe = /(\w+)="([^"]*)"/g;
+        const attrReDouble = /(\w+)="([^"]*)"/g;
+        const attrReSingle = /(\w+)='([^']*)'/g;
         let m;
-        while ((m = attrRe.exec(attrStr)) !== null) attrs[m[1]] = m[2];
+        while ((m = attrReDouble.exec(attrStr)) !== null) attrs[m[1]] = m[2];
+        while ((m = attrReSingle.exec(attrStr)) !== null) attrs[m[1]] = m[2];
 
         this.buffer = this.buffer.slice(gtIdx + 1);
 
@@ -70,11 +80,37 @@ class StreamParser {
       } else {
         const closing = '</' + this.insideTag + '>';
         const closeIdx = this.buffer.toLowerCase().indexOf(closing);
+        const entry = TAG_REGISTRY[this.insideTag];
+        const streamInner = entry && entry.type === 'final';
         if (closeIdx === -1) {
-          this.tagContent += this.buffer;
-          this.buffer = '';
+          if (streamInner) {
+            // Emit content live through onToken, but hold back any trailing
+            // substring that could be a prefix of the closing tag (chunk
+            // boundary splitting `</final_answer>` into e.g. `</fin` + `al…`).
+            const lowBuf = this.buffer.toLowerCase();
+            const lowClose = closing;
+            let safeUpTo = this.buffer.length;
+            const ltIdx = lowBuf.lastIndexOf('<');
+            if (ltIdx !== -1) {
+              const tail = lowBuf.slice(ltIdx);
+              if (lowClose.startsWith(tail)) safeUpTo = ltIdx;
+            }
+            if (safeUpTo > 0) {
+              const emit = this.buffer.slice(0, safeUpTo);
+              this.onToken(emit);
+              this.tagContent += emit;
+              this.buffer = this.buffer.slice(safeUpTo);
+            }
+          } else {
+            this.tagContent += this.buffer;
+            this.buffer = '';
+          }
           break;
         }
+        if (streamInner) {
+          const emit = this.buffer.slice(0, closeIdx);
+          if (emit) this.onToken(emit);
+        }
         this.tagContent += this.buffer.slice(0, closeIdx);
         this.buffer = this.buffer.slice(closeIdx + closing.length);
         this.onTagContent(this.insideTag, this.tagContent);
@@ -99,7 +135,7 @@ function cleanAssistantContent(raw) {
   }
 
   for (const [tag, entry] of Object.entries(TAG_REGISTRY)) {
-    if (entry.type === 'strip') {
+    if (entry.type === 'strip' || entry.type === 'final') {
       // Strip only the wrapper tags; keep the inner content
       text = text.replace(new RegExp(`<${tag}[^>]*>`, 'gi'), '');
       text = text.replace(new RegExp(`<\\/${tag}>`, 'gi'), '');
@@ -112,13 +148,277 @@ function cleanAssistantContent(raw) {
     }
   }
 
-  text = text.replace(/<\/?[a-zA-Z_][a-zA-Z0-9_]*(\s[^>]*)?>/g, '');
-  text = text.replace(/\n{2,}/g, '\n');
-
   return text.trim();
 }
 
-function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agentExecFile, ui }) {
+function estimateTokens(text) {
+  return Math.floor((text || '').length / 4);
+}
+
+// User-initiated aborts surface through several shapes depending on where in
+// the Node http stack the signal fires: `new Error('Aborted')` from our own
+// abort paths in api.js, or AbortError/ABORT_ERR from Node's built-ins. The
+// authoritative check is the signal itself — this helper is the fallback.
+function isAbortError(err) {
+  if (!err) return false;
+  if (err.name === 'AbortError') return true;
+  if (err.code === 'ABORT_ERR' || err.code === 'ERR_ABORTED') return true;
+  if (typeof err.message === 'string' && /^Aborted$/i.test(err.message)) return true;
+  return false;
+}
+
+function abortableSleep(ms, signal) {
+  return new Promise((resolve) => {
+    if (signal && signal.aborted) { resolve(); return; }
+    const t = setTimeout(resolve, ms);
+    if (signal) {
+      signal.addEventListener('abort', () => {
+        clearTimeout(t);
+        resolve();
+      }, { once: true });
+    }
+  });
+}
+
+function detectFormat(reply, toolCalls) {
+  if (!reply || !reply.trim()) return 'empty';
+  if (/<(minimax:tool_call|qwen:tool_call|tool_call|function_call)\b/i.test(reply)) return 'tool_call';
+  if (toolCalls && toolCalls.length > 0) return 'command';
+  return 'text';
+}
+
+// Spot known-tag names that are present in the reply but didn't produce any
+// parsed tool calls. Common culprits are attribute-required tags (create_file,
+// write_file, etc.) emitted without the `path` attribute, usually because the
+// model put nonsense like `<attrs: path=...>` inside the body. Returning a
+// specific hint lets the agent loop push a corrective user message and keep
+// going instead of silently stalling.
+function detectMalformedTags(text) {
+  const issues = [];
+  const PATH_REQUIRED = ['create_file', 'write_file', 'append_file'];
+  for (const tag of PATH_REQUIRED) {
+    const re = new RegExp(`<${tag}\\b([^>]*)>`, 'g');
+    for (const m of text.matchAll(re)) {
+      const attrs = m[1] || '';
+      if (!/\bpath\s*=\s*['"]/.test(attrs)) {
+        issues.push({
+          tag,
+          hint: `Use <${tag} path="/absolute/path">FILE CONTENT HERE</${tag}>. Put the path as a quoted attribute on the opening tag, and the actual file contents between the tags — not a nested pseudo-tag.`,
+        });
+      }
+    }
+  }
+  // Deduplicate by tag so we don't spam the model with the same hint per occurrence.
+  const seen = new Set();
+  return issues.filter((i) => (seen.has(i.tag) ? false : (seen.add(i.tag), true)));
+}
+
+function previewCommand(call) {
+  if (!call) return 'NONE — CLIENT WILL STALL';
+  const tag = call[0] || 'unknown';
+  const arg = call[1] || '';
+  const oneLine = String(arg).replace(/\s+/g, ' ').trim();
+  const trimmed = oneLine.length > 80 ? oneLine.slice(0, 77) + '...' : oneLine;
+  return trimmed ? `<${tag}> ${trimmed}` : `<${tag}>`;
+}
+
+function formatDebugBlock(sections) {
+  // The debug block is rendered as a tool-output message in the TUI. Chat
+  // history indents output by 5 cols; account for that so the frame still
+  // reaches the visible right edge instead of wrapping.
+  const totalW = Math.max(40, _getCols());
+  const frameW = Math.max(20, totalW - 7);
+  const H = UI_THEME.muted;       // frame glyphs
+  const L = UI_THEME.info;         // block label ("DEBUG TOOL RESULTS")
+  const S = UI_THEME.subtle;       // iteration tag, secondary text
+  const K = UI_THEME.accent;       // section-header bullets ("▸ SUMMARY")
+  const W = UI_THEME.warning;      // warning markers
+
+  const header = sections.title || 'DEBUG';
+  const iter = `[iteration ${sections.iteration}]`;
+
+  const out = [];
+  // Top frame: " LABEL [iteration N] ═════… " — fills to width.
+  const headPrefix = `${H}══ ${RST}${L}${header}${RST} ${S}${iter}${RST} `;
+  const headVisible = 4 + header.length + 1 + iter.length + 1; // "══ " + label + " " + iter + " "
+  out.push(`${headPrefix}${H}${repeatToWidth('═', frameW, headVisible)}${RST}`);
+
+  const pushSection = (title) => out.push(`${K}▸ ${RST}${L}${title}${RST}`);
+
+  for (const [title, rows] of sections.blocks) {
+    pushSection(title);
+    const width = Math.max(...rows.map((r) => r[0].length));
+    for (const [k, v] of rows) {
+      const val = (v === undefined || v === null) ? '—' : String(v);
+      out.push(`  ${S}${k.padEnd(width + 2)}${RST}${val}`);
+    }
+    out.push('');
+  }
+  if (sections.raw !== undefined) {
+    pushSection('RAW RESPONSE');
+    out.push(sections.raw ? sections.raw : `${S}(empty)${RST}`);
+    out.push('');
+    if (sections.rawFooter && sections.rawFooter.length) {
+      pushSection('STREAM FOOTER');
+      const width = Math.max(...sections.rawFooter.map((r) => r[0].length));
+      for (const [k, v] of sections.rawFooter) {
+        const val = (v === undefined || v === null) ? '—' : String(v);
+        out.push(`  ${S}${k.padEnd(width + 2)}${RST}${val}`);
+      }
+      out.push('');
+    }
+  }
+  if (sections.entries && sections.entries.length) {
+    for (const entry of sections.entries) {
+      pushSection(entry.title);
+      if (entry.rows) {
+        const width = Math.max(...entry.rows.map((r) => r[0].length));
+        for (const [k, v] of entry.rows) {
+          const val = (v === undefined || v === null) ? '—' : String(v);
+          out.push(`  ${S}${k.padEnd(width + 2)}${RST}${val}`);
+        }
+      }
+      if (entry.body !== undefined) {
+        const body = entry.body === '' ? `${S}(empty)${RST}` : entry.body;
+        for (const line of String(body).split('\n')) out.push('    ' + line);
+      }
+      out.push('');
+    }
+  }
+  if (sections.warnings && sections.warnings.length) {
+    pushSection('WARNINGS');
+    for (const w of sections.warnings) out.push(`  ${W}⚠ ${w}${RST}`);
+    out.push('');
+  }
+  // Bottom frame: plain full-width rule in muted.
+  out.push(`${H}${repeatToWidth('═', frameW)}${RST}`);
+  return out.join('\n');
+}
+
+function truncateForDebug(text, maxLines = 40, maxChars = 2000) {
+  if (text === undefined || text === null) return '';
+  let s = String(text);
+  if (s.length > maxChars) {
+    s = s.slice(0, maxChars) + `\n… [truncated, ${String(text).length - maxChars} more chars]`;
+  }
+  const lines = s.split('\n');
+  if (lines.length > maxLines) {
+    return lines.slice(0, maxLines).join('\n') + `\n… [truncated, ${lines.length - maxLines} more lines]`;
+  }
+  return s;
+}
+
+// Per-tag meta extractor. Converts a tool-executor return value into the
+// compact meta object consumed by the tool-line formatter — exit codes for
+// shell, byte counts for file ops, status_code + bytes for HTTP, etc. A
+// pure function by design: no UI state, no config reads. The callback
+// layer (commands.js) feeds the meta into formatToolLine together with
+// the tag, so the formatter can produce the 4-segment line in either the
+// pending (live region) or final (scrollback) context.
+function _metaForTool(tag, result) {
+  if (!result || result.error) return null;
+  switch (tag) {
+    case 'shell':
+    case 'exec':
+      return { exit_code: result.exit_code };
+    case 'read':
+    case 'read_file':
+      return {
+        bytes: typeof result.bytes === 'number'
+          ? result.bytes
+          : (result.content ? Buffer.byteLength(String(result.content), 'utf8') : 0),
+      };
+    case 'write':
+    case 'write_file':
+    case 'create_file':
+    case 'append':
+    case 'append_file':
+    case 'upload':
+      return { bytes: typeof result.bytes === 'number' ? result.bytes : 0 };
+    case 'list_dir':
+      return { count: Array.isArray(result.items) ? result.items.length : 0 };
+    case 'search_files':
+      return { count: Array.isArray(result.files) ? result.files.length : 0 };
+    case 'search_in_file':
+      return { count: Array.isArray(result.matches) ? result.matches.length : 0 };
+    case 'replace_in_file':
+      return { count: typeof result.count === 'number' ? result.count : 0 };
+    case 'http_get':
+    case 'download':
+      return {
+        status_code: result.status_code,
+        bytes: typeof result.bytes === 'number'
+          ? result.bytes
+          : (result.body ? Buffer.byteLength(String(result.body), 'utf8') : 0),
+      };
+    case 'file_stat':
+      return {
+        bytes: result.size_kb ? Math.round(parseFloat(result.size_kb) * 1024) : 0,
+        kind: result.type || null,
+      };
+    default:
+      return null;
+  }
+}
+
+// Turn a [action, arg1, arg2, …] call tuple into the `attrs` bag that
+// formatToolLine looks up when building the operation string. Centralized
+// here so the per-tag positional-arg contract is written down in exactly
+// one place — any new tool added to the agent-loop tuple schema also gets
+// its attrs mapping here.
+function _attrsFromCall(call) {
+  if (!Array.isArray(call) || call.length === 0) return {};
+  const [tag, ...args] = call;
+  switch (tag) {
+    case 'shell':
+    case 'exec':
+      return { command: args[0] || '' };
+    case 'read':
+    case 'read_file':
+    case 'list_dir':
+    case 'delete_file':
+    case 'make_dir':
+    case 'remove_dir':
+    case 'file_stat':
+      return { path: args[0] || '' };
+    case 'write':
+    case 'write_file':
+    case 'create_file':
+    case 'append':
+    case 'append_file':
+      return { path: args[0] || '', content: args[1] || '' };
+    case 'upload':
+      return { path: args[0] || '' };
+    case 'move_file':
+    case 'copy_file':
+      return { src: args[0] || '', dst: args[1] || '' };
+    case 'edit_file':
+      return { path: args[0] || '', line: args[1], content: args[2] || '' };
+    case 'search_files':
+      return { pattern: args[0] || '', dir: args[1] || '.' };
+    case 'search_in_file':
+      return { path: args[0] || '', pattern: args[1] || '' };
+    case 'replace_in_file':
+      return { path: args[0] || '', search: args[1] || '', replace: args[2] || '', flags: args[3] || '' };
+    case 'get_env':
+      return { name: args[0] || '' };
+    case 'set_env':
+      return { name: args[0] || '', value: args[1] || '' };
+    case 'download':
+    case 'http_get':
+      return { url: args[0] || '' };
+    case 'ask_user':
+      return { question: args[0] || '' };
+    case 'store_memory':
+      return { key: args[0] || '', value: args[1] || '' };
+    case 'recall_memory':
+      return { key: args[0] || '' };
+    default:
+      return {};
+  }
+}
+
+function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agentExecFile, ui, getConfig }) {
   const { BOLD, FG_DARK, FG_GRAY, FG_TEAL, FG_YELLOW, RST, THEME, getCols } = ui;
 
   function formatFileResult(call, result) {
@@ -140,20 +440,8 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
       case 'file_stat':
         return `Stat ${result.path}: size=${result.size_kb} KB, mtime=${result.mtime}, type=${result.type}, mode=${result.mode}`;
       case 'http_get': {
-        if (result.chunked) {
-          return `HTTP GET ${args[0]} (${result.status_code}) [Part 1/${result.total_parts}]:\n${result.body}\n\n[Response is large and was split into ${result.total_parts} parts. Use <http_get_next key="${args[0]}"/> to retrieve the next part.]`;
-        }
         return `HTTP GET ${args[0]} (${result.status_code}):\n${result.body}`;
       }
-      case 'http_get_next': {
-        if (result.done && !result.body) {
-          return `http_get_next "${args[0]}": No more content available.`;
-        }
-        const more = result.done
-          ? ' [Final part]'
-          : `\n\n[Use <http_get_next key="${args[0]}"/> to retrieve part ${result.part + 1}/${result.total_parts}.]`;
-        return `HTTP content "${args[0]}" [Part ${result.part}/${result.total_parts}]:\n${result.body}${more}`;
-      }
       case 'ask_user':
         return `User answered "${result.question}": ${result.answer}`;
       case 'store_memory':
@@ -257,10 +545,6 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
         const raw = attrs.raw || '';
         return formatFileResult(['http_get', url, raw], await agentExecFile('http_get', url, raw));
       }
-      case 'http_get_next': {
-        const key = attrs.key || content;
-        return formatFileResult(['http_get_next', key], await agentExecFile('http_get_next', key));
-      }
       case 'ask_user': {
         const q = attrs.question || content;
         return formatFileResult(['ask_user', q], await agentExecFile('ask_user', q));
@@ -295,9 +579,9 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
       return;
     }
 
-    if (entry.type === 'strip') return;
+    if (entry.type === 'strip' || entry.type === 'final') return;
 
-    // Tool execution happens in the toolCalls loop after streaming; handleTag only handles visual/strip.
+    // Tool execution happens in the toolCalls loop after streaming; handleTag only handles visual/strip/final.
   }
 
   async function runAgentLoop(messages, model, maxIterations = Infinity, tokenLimit = null, opts = {}) {
@@ -312,9 +596,27 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
     const isAborted = getAbortFlag || (() => false);
     const cb = callbacks;
     const metrics = new Metrics(tokenLimit);
-    const activeSystemPrompt = overrideSystemPrompt !== null ? overrideSystemPrompt : SYSTEM_PROMPT;
     const mode = overrideMode || 'system_role';
 
+    // Route debug blocks to the UI callback when present (interactive TUI mode
+    // overwrites stderr with redraws, losing the output). Fall back to stderr
+    // for one-shot/non-TTY flows where there's no UI to host the block.
+    const emitDebug = (block) => {
+      if (typeof cb.onDebug === 'function') cb.onDebug(block);
+      // audit: allowed — stderr debug under --debug flag (no UI hosting available).
+      else process.stderr.write('\n' + block + '\n');
+    };
+
+    // Resolve native_tools from the active profile (matched by api_base+model).
+    // Fallback to true if no matching profile — mirrors config-normalization default.
+    const _cfg = typeof getConfig === 'function' ? getConfig() : {};
+    const _profile = Array.isArray(_cfg.models)
+      ? _cfg.models.find((p) => p && p.api_base === _cfg.api_base && p.model === model)
+      : null;
+    const nativeTools = _profile && _profile.native_tools === false ? false : true;
+
+    const activeSystemPrompt = overrideSystemPrompt !== null ? overrideSystemPrompt : getSystemPrompt(nativeTools);
+
     for (let iteration = 0; iteration < maxIterations; iteration++) {
       if (isAborted()) break;
       const linePrefix = `${FG_TEAL}${BOLD}◆ ${RST}`;
@@ -366,49 +668,116 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
           }
         : null;
 
-      if (debug) {
-        const header = `\n───── messages sent to agent (iteration ${iteration + 1}) ─────\n`;
-        const footer = `\n───── end messages ─────\n`;
-        process.stderr.write(header + JSON.stringify(messagesWithSystem, null, 2) + footer);
-      }
-
       const MAX_RETRIES = 3;
+      const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
+      const NON_RETRYABLE_STATUS = new Set([400, 401, 403, 404, 413, 422]);
       let result = null;
       let lastApiErr = null;
 
-      for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
-        if (attempt === 1) {
-          callbacks.onRequestSent?.();
-        } else {
-          cb.onRetry?.(attempt, MAX_RETRIES);
-          await new Promise((r) => setTimeout(r, 1000));
-        }
-        try {
-          result = await chatStream(messagesWithSystem, {
-            model,
-            linePrefix: wrappedOnToken ? '' : linePrefix,
-            showThink,
-            onToken: wrappedOnToken,
-            silent: !!wrappedOnToken,
-          });
-          lastApiErr = null;
-          break;
-        } catch (err) {
-          lastApiErr = err;
-          if (debug) {
-            const header = `\n───── raw http error (iteration ${iteration + 1}, attempt ${attempt}/${MAX_RETRIES}) ─────\n`;
-            const footer = `\n───── end raw http error ─────\n`;
-            const status = err.statusCode ? `HTTP ${err.statusCode}` : 'network error';
-            const headerLines = err.responseHeaders
-              ? Object.entries(err.responseHeaders).map(([k, v]) => `${k}: ${v}`).join('\n')
-              : '';
-            const body = err.rawBody !== undefined ? err.rawBody : (err.stack || err.message || String(err));
-            const parts = [status];
-            if (headerLines) parts.push(headerLines);
-            parts.push(body || '(empty body)');
-            process.stderr.write(header + parts.join('\n\n') + footer);
+      // AbortController per iteration: watcher polls isAborted() every 50ms
+      // and flips controller.abort() as soon as the flag flips.
+      const controller = new AbortController();
+      const abortWatcher = setInterval(() => {
+        if (isAborted() && !controller.signal.aborted) controller.abort();
+      }, 50);
+
+      try {
+        for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+          if (attempt === 1) {
+            callbacks.onRequestSent?.();
+          }
+          try {
+            result = await chatStream(messagesWithSystem, {
+              model,
+              nativeTools,
+              linePrefix: wrappedOnToken ? '' : linePrefix,
+              showThink,
+              onToken: wrappedOnToken,
+              silent: !!wrappedOnToken,
+              signal: controller.signal,
+              onTrim: (info) => {
+                // Setter (rather than re-reading config per iteration) keeps
+                // the Metrics instance authoritative: a 400-overflow discovery
+                // mid-loop immediately updates 85%-warning thresholds and the
+                // status bar without threading config access into agent.js.
+                if (info && info.reason === 'overflow-400' && typeof info.limit === 'number' && info.limit > 0) {
+                  metrics.setModelTokenLimit(info.limit);
+                }
+                if (cb.onError) {
+                  cb.onError({
+                    message: `Context trimmed (${info.reason}): ${info.dropped} message(s) dropped, kept ~${info.keptTokens} tokens (limit ${info.limit}).`,
+                    isWarning: true,
+                  });
+                }
+              },
+            });
+            lastApiErr = null;
+            break;
+          } catch (err) {
+            // User-initiated abort: not a transient failure. Skip the retry
+            // counter, the "Retrying (N/M)..." status update, the debug dump,
+            // and the post-loop error surface. The "Interrupted." feedback is
+            // already shown by the input-field abort listener.
+            if (controller.signal.aborted || isAborted() || isAbortError(err)) {
+              lastApiErr = null;
+              break;
+            }
+            lastApiErr = err;
+            if (debug) {
+              const status = err.statusCode ? `HTTP ${err.statusCode}` : 'network error';
+              const body = err.rawBody !== undefined ? err.rawBody : (err.stack || err.message || String(err));
+              const block = formatDebugBlock({
+                iteration: iteration + 1,
+                blocks: [
+                  ['REQUEST', [
+                    ['model:', model],
+                    ['endpoint:', err.endpoint || '(unknown)'],
+                    ['timestamp:', new Date().toISOString()],
+                    ['native_tools:', nativeTools],
+                    ['attempt:', `${attempt}/${MAX_RETRIES}`],
+                  ]],
+                  ['RESPONSE', [
+                    ['status:', status],
+                    ['detail:', err.detail || ''],
+                  ]],
+                ],
+                raw: body || '(empty body)',
+                warnings: [`HTTP error on attempt ${attempt}/${MAX_RETRIES}: ${err.message}`],
+              });
+              emitDebug(block);
+            }
+            const sc = err.statusCode;
+            const retryable = !sc || RETRYABLE_STATUS.has(sc);
+            if (!retryable || NON_RETRYABLE_STATUS.has(sc)) break;
+            if (attempt >= MAX_RETRIES) break;
+            // Backoff: base 1000ms doubling (1s, 2s, 4s). For 429, honor
+            // Retry-After header when it's a plausible seconds value.
+            let delayMs = 1000 * Math.pow(2, attempt - 1);
+            if (sc === 429) {
+              const ra = err.responseHeaders && err.responseHeaders['retry-after'];
+              const raNum = ra !== undefined ? Number(ra) : NaN;
+              if (Number.isFinite(raNum) && raNum >= 0 && raNum <= 30) {
+                delayMs = Math.round(raNum * 1000);
+              }
+            }
+            cb.onRetry?.(attempt + 1, MAX_RETRIES);
+            await abortableSleep(delayMs, controller.signal);
+            // Ctrl+C pressed during backoff: bail without the next attempt.
+            if (controller.signal.aborted || isAborted()) {
+              lastApiErr = null;
+              break;
+            }
           }
         }
+      } finally {
+        clearInterval(abortWatcher);
+      }
+
+      // User-initiated abort: exit the turn quietly. Skip the empty-reply
+      // "connection dropped" warning below — the abort listener already
+      // surfaced "Interrupted." and the outer prompt will return.
+      if (controller.signal.aborted || isAborted()) {
+        break;
       }
 
       if (lastApiErr) {
@@ -420,33 +789,52 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
       const usage = result ? result.usage : null;
       metrics.endTurn(usage, model);
 
-      if (debug) {
-        const header = `\n───── raw ai response (iteration ${iteration + 1}) ─────\n`;
-        const footer = `\n───── end raw response ─────\n`;
-        process.stderr.write(header + (reply || '(empty)') + footer);
-      }
-
       if (cb.onMetricsUpdate) {
         cb.onMetricsUpdate({
           totalTokens: metrics.totalTokens(),
           contextTokens: metrics.contextTokens(),
           turns: metrics.turns.length,
+          tokenLimit: metrics.tokenLimitStatus(),
         });
       }
 
       const limitStatus = metrics.tokenLimitStatus();
-      if (limitStatus !== null && limitStatus.pct >= 85) {
+      if (limitStatus !== null && limitStatus.pct !== null && limitStatus.pct >= 85) {
         const warnMsg = `Context at ${limitStatus.pct}% of limit (${limitStatus.used}/${limitStatus.limit} tokens). Consider /compact.`;
         if (cb.onError) {
           cb.onError({ message: warnMsg, isWarning: true });
         } else {
-          process.stdout.write(
-            `\n  ${THEME.warn}⚠ ${warnMsg}${THEME.reset}\n`
-          );
+          messages.sysWarn(warnMsg);
         }
       }
 
       if (!reply) {
+        if (debug && result) {
+          const block = formatDebugBlock({
+            iteration: iteration + 1,
+            blocks: [
+              ['REQUEST', [
+                ['model:', result.request?.model || model],
+                ['endpoint:', result.endpoint || '(unknown)'],
+                ['timestamp:', new Date().toISOString()],
+                ['native_tools:', result.request?.native_tools ?? nativeTools],
+              ]],
+              ['RESPONSE', [
+                ['finish_reason:', result.finish_reason || '(unknown)'],
+                ['completion_tokens:', usage?.completion_tokens ?? 0],
+                ['latency_ms:', result.elapsed_ms ?? '?'],
+              ]],
+              ['PARSED', [
+                ['detected_format:', 'empty'],
+                ['commands_found:', 0],
+                ['first_command:', 'NONE — CLIENT WILL STALL'],
+              ]],
+            ],
+            raw: '',
+            warnings: ['Agent returned an empty response — connection to model may have dropped'],
+          });
+          emitDebug(block);
+        }
         // Empty reply from the model — stream resolved with no content and no
         // tool_calls. Most common causes: server-side disconnect mid-stream,
         // context-window overflow that slipped past the 400/413 handler, or a
@@ -459,69 +847,185 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
         break;
       }
 
-      const toolCalls = extractToolCalls(reply);
+      // Native function-calling path: the model returned structured tool_calls.
+      // Convert them to the internal [action, ...args] tuple shape so downstream
+      // dispatch is unchanged, and remember the OpenAI id per call so results
+      // can be rooted back to their tool_call on the next turn.
+      const nativeToolCalls = Array.isArray(result?.toolCalls) ? result.toolCalls : [];
+      let toolCalls;
+      let nativeToolCallIds = [];
+      if (nativeToolCalls.length > 0) {
+        toolCalls = [];
+        for (const tc of nativeToolCalls) {
+          let args;
+          try {
+            args = tc.function?.arguments ? JSON.parse(tc.function.arguments) : {};
+          } catch (err) {
+            if (cb.onError) cb.onError({ message: `Failed to parse tool_call arguments for ${tc.function?.name || '(unknown)'}: ${err.message}`, isWarning: true });
+            continue;
+          }
+          const call = mapInvokeToCall(tc.function?.name, args);
+          if (call) {
+            toolCalls.push(call);
+            nativeToolCallIds.push(tc.id);
+          }
+        }
+      } else {
+        toolCalls = extractToolCalls(reply, { model });
+      }
+      const isNativeCall = nativeToolCalls.length > 0;
       const cleanedReply = cleanAssistantContent(reply);
 
-      // Detect mid-tag truncation: an opening tool tag in the raw reply with
-      // no matching close. This happens when the model streams a large
-      // `<write_file>…` body and hits max_tokens or a server-side cutoff
-      // before the closing tag arrives. cleanAssistantContent strips the
-      // unclosed tag + its trailing content, so cleanedReply looks
-      // legitimate (just the planning preamble) and extractToolCalls finds
-      // zero calls — the loop would break silently and the user sees the
-      // planning text followed by nothing. Surface it so the user can retry,
-      // shorten the request, or bump max_tokens.
-      let truncatedTag = null;
-      for (const [tag, entry] of Object.entries(TAG_REGISTRY)) {
-        if (entry.type !== 'tool') continue;
-        let opens = 0;
-        for (const m of reply.matchAll(new RegExp(`<${tag}([^>]*)>`, 'gi'))) {
-          // Skip self-closing (`<tag .../>`) — they don't need a matching close.
-          if (!m[1].trimEnd().endsWith('/')) opens++;
+      if (debug && result) {
+        const lastUserMsg = [...messagesWithSystem].reverse().find((m) => m.role === 'user');
+        const sysMsg = messagesWithSystem.find((m) => m.role === 'system');
+        const systemPromptTok = estimateTokens(sysMsg ? sysMsg.content : activeSystemPrompt);
+        const currentInputTok = estimateTokens(lastUserMsg ? lastUserMsg.content : '');
+        const historyTok = messagesWithSystem.reduce((sum, m) => {
+          if (m === sysMsg || m === lastUserMsg) return sum;
+          return sum + estimateTokens(m.content || '');
+        }, 0);
+        const promptTokens = usage && usage.prompt_tokens != null
+          ? usage.prompt_tokens
+          : systemPromptTok + historyTok + currentInputTok;
+        const completionTokens = usage && usage.completion_tokens != null
+          ? usage.completion_tokens
+          : estimateTokens(reply);
+        const thinkingTokens = result.reasoning ? estimateTokens(result.reasoning) : 0;
+        const visibleTokens = Math.max(completionTokens - thinkingTokens, 0);
+        const contextLimit = tokenLimit || null;
+        const ctxPct = contextLimit ? Math.round((promptTokens / contextLimit) * 100) : null;
+        const detected = detectFormat(reply, toolCalls);
+        const firstCmd = toolCalls.length > 0 ? previewCommand(toolCalls[0]) : previewCommand(null);
+        const toolTags = Object.entries(TAG_REGISTRY)
+          .filter(([, e]) => e.type === 'tool')
+          .map(([t]) => t);
+
+        const warnings = [];
+        if (result.finish_reason === 'length') warnings.push('finish_reason=length  → response truncated, increase max_tokens');
+        if (detected === 'tool_call' && toolCalls.length === 0) {
+          warnings.push('commands_found=0      → agent emitted no command, client will stall');
         }
-        if (opens === 0) continue;
-        const closes = (reply.match(new RegExp(`<\\/${tag}>`, 'gi')) || []).length;
-        if (opens > closes) { truncatedTag = tag; break; }
+        if (ctxPct !== null && ctxPct > 80) warnings.push(`context_used=${ctxPct}%    → approaching context limit`);
+
+        const block = formatDebugBlock({
+          iteration: iteration + 1,
+          blocks: [
+            ['REQUEST', [
+              ['model:', result.request?.model || model],
+              ['endpoint:', result.endpoint || '(unknown)'],
+              ['timestamp:', new Date().toISOString()],
+              ['native_tools:', result.request?.native_tools ?? nativeTools],
+            ]],
+            ['CONTEXT', [
+              ['total_messages:', messagesWithSystem.length],
+              ['system_prompt_tok:', systemPromptTok],
+              ['history_tok:', historyTok],
+              ['current_input_tok:', currentInputTok],
+              ['context_used:', contextLimit
+                ? `${promptTokens} / ${contextLimit}  (${ctxPct}%)`
+                : `${promptTokens} / unknown`],
+            ]],
+            ['PARAMETERS', [
+              ['max_tokens:', result.request?.max_tokens ?? '(default)'],
+              ['temperature:', result.request?.temperature ?? '(default)'],
+              ['stop_sequences:', JSON.stringify(result.request?.stop || [])],
+              ['reasoning_effort:', '(n/a)'],
+              ['tools_enabled:', `${toolTags.length} XML tags (via system prompt)`],
+            ]],
+            ['RESPONSE', [
+              ['finish_reason:', result.finish_reason || '(unknown)'],
+              ['completion_tokens:', completionTokens],
+              ['thinking_tokens:', thinkingTokens],
+              ['visible_tokens:', visibleTokens],
+              ['latency_ms:', result.elapsed_ms ?? '?'],
+            ]],
+            ['PARSED', [
+              ['detected_format:', detected],
+              ['commands_found:', toolCalls.length],
+              ['first_command:', firstCmd],
+            ]],
+          ],
+          raw: reply || '',
+          rawFooter: [
+            ['finish_reason:', result.finish_reason || '(unknown)'],
+            ['total_tokens:', result.usage_from_provider && result.usage
+              ? (result.usage.prompt_tokens || 0) + (result.usage.completion_tokens || 0)
+              : '(no usage)'],
+            ['content_chars:', (result.content || '').length],
+            ['reasoning_chars:', (result.reasoning_details || '').length],
+            ['tool_calls_seen:', result.tool_calls_count > 0 ? `yes (${result.tool_calls_count})` : 'no'],
+            ['native_mode:', isNativeCall ? `yes (${nativeToolCalls.length} call${nativeToolCalls.length === 1 ? '' : 's'})` : 'no'],
+          ],
+          warnings,
+        });
+        emitDebug(block);
       }
+
+      // Detect mid-tag truncation: StreamParser tracks `insideTag` for the
+      // currently-unclosed tag. If it's still set after the stream settles,
+      // the response was cut off while inside a tool tag (hit max_tokens or
+      // a server-side cutoff). cleanAssistantContent strips the unclosed
+      // tag + trailing content, so extractToolCalls would find nothing and
+      // the loop would break silently. Surface it.
+      //
+      // When `cb.onToken` is unset (non-streaming UI), the parser was never
+      // fed — push the final reply through it once so `insideTag` reflects
+      // the terminal state.
+      if (!wrappedOnToken && reply) parser.push(reply);
+      const truncatedTag = parser.insideTag && TAG_REGISTRY[parser.insideTag]?.type === 'tool'
+        ? parser.insideTag
+        : null;
       if (truncatedTag && cb.onError) {
         cb.onError({ message: `Response truncated mid-<${truncatedTag}> tag — likely hit max_tokens or a server-side cutoff. Try again, shorten the request, or raise the model's max_tokens.`, isWarning: true });
       }
 
-      messages.push({ role: 'assistant', content: cleanedReply });
+      const assistantMsg = { role: 'assistant', content: cleanedReply };
+      if (isNativeCall) assistantMsg.tool_calls = nativeToolCalls;
+      messages.push(assistantMsg);
       // When showThink is off and the turn has tool calls, suppress the text bubble —
       // pre-tool reasoning is noise, tool result bubbles already convey what happened.
       const displayReply = (!showThink && toolCalls.length > 0) ? '' : cleanedReply;
       if (cb.onAssistantMessage) cb.onAssistantMessage(displayReply);
 
-      // If nothing meaningful came back (no text to show, no tools to run) but
-      // the reply string wasn't strictly empty, it's usually model wrapper
-      // noise or a stripped-only response. Still a dead-end for the user.
-      if (toolCalls.length === 0 && !cleanedReply.trim()) {
-        if (cb.onError) {
-          cb.onError({ message: 'Agent reply had no visible content and no actions — stopping.', isWarning: true });
-        }
-        break;
-      }
-
       if (toolCalls.length === 0) {
-        // Model narrated next steps but didn't emit a tool tag. Happens when the
-        // model ends a plan with "Let me do that for you." and stops. If we just
-        // break, the user sees a dangling promise and thinks the connection dropped.
-        if (iteration > 0 && /\b(let me|i['’]?ll|i will|i'?m going to|next[, ]|now[, ]? ?(i|we)|going to (create|write|build|add|make|run|do|set up|install))\b/i.test(cleanedReply)) {
+        // Detect malformed known-tag syntax (e.g. <create_file> with no path
+        // attribute, usually paired with nonsense like <attrs: path=...> inside
+        // the body). Push a corrective feedback message and keep looping so
+        // the model self-corrects instead of silently stalling on the user.
+        const malformed = detectMalformedTags(reply);
+        if (malformed.length > 0) {
+          const hintBlock = malformed.map((m) => `- <${m.tag}>: ${m.hint}`).join('\n');
+          const summary = malformed.map((m) => `<${m.tag}>`).join(', ');
           if (cb.onError) {
-            cb.onError({ message: 'Agent described next steps but did not emit a tool call. Reply "continue" (or similar) to push it forward, or restart if it keeps stalling.', isWarning: true });
+            cb.onError({ message: `Detected malformed tool tag(s): ${summary}. Asking the model to retry with correct syntax.`, isWarning: true });
           }
+          messages.push({
+            role: 'user',
+            content: `Your last response contained malformed tool tags that the parser could not execute:\n\n${hintBlock}\n\nRe-emit the tool calls using the exact syntax above. Do not nest pseudo-tags like <attrs: ...> inside the body.`,
+          });
+          continue;
         }
+
+        // No tool calls and non-empty content (the empty case was already
+        // handled by the `!reply` guard above). This is the model's final
+        // answer for this turn — end the loop and return control to the user.
         break;
       }
       if (isAborted()) break;
 
       if (!cb.onToolStart) {
-        process.stdout.write(`\n  ${FG_TEAL}◆${RST} ${FG_GRAY}Found ${toolCalls.length} action(s) to execute${RST}\n`);
+        writer.scrollback(`\n  ${FG_TEAL}◆${RST} ${FG_GRAY}Found ${toolCalls.length} action(s) to execute${RST}`);
       }
 
       const results = [];
+      const debugEntries = debug ? [] : null;
       let aborted = false;
+      // Per-invocation id. Paired across onToolStart/onToolEnd so the UI
+      // layer can track each concurrent tool's activity-region slot and
+      // commit its final line atomically via endActivity. Monotonic —
+      // never reused even if the agent runs the same tag twice.
+      let invocationCounter = 0;
 
       for (const call of toolCalls) {
         if (isAborted()) { aborted = true; break; }
@@ -529,8 +1033,11 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
         const tag = call[0] || 'unknown';
         const arg = call[1] || '';
         const toolStart = Date.now();
+        const invocationId = `tool-${iteration}-${invocationCounter++}-${tag}`;
+        const attrs = _attrsFromCall(call);
+        const startCtx = { id: invocationId, call, attrs, startedAt: toolStart };
 
-        if (cb.onToolStart) cb.onToolStart(tag, arg);
+        if (cb.onToolStart) cb.onToolStart(tag, arg, startCtx);
 
         try {
           if (tag === 'shell') {
@@ -538,16 +1045,29 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
             const ms = Date.now() - toolStart;
             if (shellResult.stderr === 'Permission denied by user') {
               const resultStr = `Command \`${arg}\`: Permission denied by user.`;
-              if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms);
+              if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms, { id: invocationId, call, attrs, meta: null, error: { message: 'denied' }, denied: true });
               results.push(resultStr);
+              if (debugEntries) debugEntries.push({ tag, call, ms, status: 'denied', exitCode: null, result: resultStr });
               aborted = true;
               break;
             } else {
               let out = shellResult.stdout;
               if (shellResult.stderr) out += `\nSTDERR: ${shellResult.stderr}`;
               const resultStr = `Command \`${arg}\`:\nExit code: ${shellResult.exit_code}\n${out}`;
-              if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms);
+              const meta = _metaForTool(tag, shellResult);
+              const error = shellResult.exit_code !== 0
+                ? { message: `exit ${shellResult.exit_code}`, code: shellResult.exit_code }
+                : null;
+              if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms, { id: invocationId, call, attrs, meta, error });
               results.push(resultStr);
+              if (debugEntries) debugEntries.push({
+                tag,
+                call,
+                ms,
+                status: shellResult.exit_code === 0 ? 'ok' : 'nonzero_exit',
+                exitCode: shellResult.exit_code,
+                result: resultStr,
+              });
             }
             continue;
           }
@@ -557,28 +1077,87 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
 
           if (fileResult.error === 'Permission denied') {
             const resultStr = `${tag} ${call[1] || ''}: Permission denied by user.`;
-            if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms);
+            if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms, { id: invocationId, call, attrs, meta: null, error: { message: 'denied' }, denied: true });
             results.push(resultStr);
+            if (debugEntries) debugEntries.push({ tag, call, ms, status: 'denied', exitCode: null, result: resultStr });
             aborted = true;
             break;
           } else {
             const resultStr = formatFileResult(call, fileResult);
-            if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms);
+            const meta = _metaForTool(tag, fileResult);
+            const error = fileResult.error
+              ? { message: fileResult.error, code: fileResult.error_code || null }
+              : null;
+            if (cb.onToolEnd) cb.onToolEnd(tag, resultStr, ms, { id: invocationId, call, attrs, meta, error });
             results.push(resultStr);
+            if (debugEntries) debugEntries.push({
+              tag,
+              call,
+              ms,
+              status: fileResult.error ? 'error' : 'ok',
+              exitCode: null,
+              result: resultStr,
+            });
           }
         } catch (err) {
           const ms = Date.now() - toolStart;
-          if (cb.onToolEnd) cb.onToolEnd(tag, `Error: ${err.message}`, ms);
+          if (cb.onToolEnd) cb.onToolEnd(tag, `Error: ${err.message}`, ms, { id: invocationId, call, attrs, meta: null, error: err });
           if (cb.onError) {
             cb.onError({ message: `Tool error (${tag}): ${err.message}`, isWarning: true });
           } else {
-            process.stdout.write(`\n  ${THEME.warn}⚠ Tool error (${tag}): ${err.message}${THEME.reset}\n`);
+            messages.toolError(tag, err.message);
           }
           logToolCall(tag, { args: call.slice(1) }, false, 'error');
           results.push(`${tag}: Error — ${err.message}`);
+          if (debugEntries) debugEntries.push({ tag, call, ms, status: 'exception', exitCode: null, result: `Error — ${err.message}` });
         }
       }
 
+      if (debug && debugEntries && debugEntries.length > 0) {
+        const totalMs = debugEntries.reduce((s, e) => s + (e.ms || 0), 0);
+        const statusCounts = debugEntries.reduce((acc, e) => {
+          acc[e.status] = (acc[e.status] || 0) + 1;
+          return acc;
+        }, {});
+        const statusSummary = Object.entries(statusCounts)
+          .map(([k, v]) => `${k}=${v}`)
+          .join(', ');
+
+        const entries = debugEntries.map((e, idx) => {
+          const argsPreview = (e.call || []).slice(1).map((a) => {
+            if (a === undefined || a === null) return '';
+            const s = String(a).replace(/\s+/g, ' ').trim();
+            return s.length > 120 ? s.slice(0, 117) + '...' : s;
+          }).filter((s) => s.length > 0).join(' │ ');
+          const rows = [
+            ['tag:', e.tag],
+            ['args:', argsPreview || '(none)'],
+            ['status:', e.status + (e.exitCode !== null && e.exitCode !== undefined ? ` (exit=${e.exitCode})` : '')],
+            ['latency_ms:', e.ms],
+          ];
+          return {
+            title: `TOOL ${idx + 1}/${debugEntries.length}`,
+            rows,
+            body: truncateForDebug(e.result),
+          };
+        });
+
+        const block = formatDebugBlock({
+          title: 'DEBUG TOOL RESULTS',
+          iteration: iteration + 1,
+          blocks: [
+            ['SUMMARY', [
+              ['tools_executed:', debugEntries.length],
+              ['total_latency_ms:', totalMs],
+              ['status_breakdown:', statusSummary],
+              ['aborted:', aborted ? 'yes' : 'no'],
+            ]],
+          ],
+          entries,
+        });
+        emitDebug(block);
+      }
+
       if (aborted) {
         const warnMsg = isAborted()
           ? 'Agent interrupted.'
@@ -586,24 +1165,36 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
         if (cb.onError) {
           cb.onError({ message: warnMsg, isWarning: true });
         } else {
-          process.stdout.write(`\n  ${FG_YELLOW}⚠${RST} ${FG_GRAY}${warnMsg}${RST}`);
+          messages.sysWarn(warnMsg);
         }
         // Push whatever results accumulated before the denial so the LLM has
         // context if the user asks to continue.
         if (results.length > 0) {
-          messages.push({
-            role: 'user',
-            content: `Tool execution results (partial — stopped after user denied an action):\n\n${results.join('\n\n')}`,
-          });
+          if (isNativeCall) {
+            for (let i = 0; i < results.length; i++) {
+              messages.push({ role: 'tool', tool_call_id: nativeToolCallIds[i], content: results[i] });
+            }
+          } else {
+            messages.push({
+              role: 'user',
+              content: `Tool execution results (partial — stopped after user denied an action):\n\n${results.join('\n\n')}`,
+            });
+          }
         }
         break;
       }
 
-      const feedback = results.join('\n\n');
-      messages.push({
-        role: 'user',
-        content: `Tool execution results:\n\n${feedback}\n\nContinue with the task. If everything is done, summarize what was accomplished.`,
-      });
+      if (isNativeCall) {
+        for (let i = 0; i < results.length; i++) {
+          messages.push({ role: 'tool', tool_call_id: nativeToolCallIds[i], content: results[i] });
+        }
+      } else {
+        const feedback = results.join('\n\n');
+        messages.push({
+          role: 'user',
+          content: `Tool execution results:\n\n${feedback}\n\nContinue with the task. If everything is done, summarize what was accomplished.`,
+        });
+      }
     }
 
     return { messages, metrics };
@@ -616,4 +1207,5 @@ function createAgentRunner({ chatStream, extractToolCalls, agentExecShell, agent
 
 module.exports = {
   createAgentRunner,
+  formatDebugBlock,
 };