@semalt-ai/code 1.19.0 → 1.20.0

package/test/native-live-narration.test.js

@@ -0,0 +1,254 @@
+'use strict';
+
+// Live token-by-token narration on the NATIVE tool-call rail, gated by a safety
+// signal so implicit reasoning is NEVER streamed (leaked).
+//
+// These tests drive the REAL createTurnHandler callbacks (chat-turn.js) wired to
+// the REAL ChatHistory (chat-history.js, with _commit captured), simulating the
+// per-iteration callback order agent.js + api.js produce:
+//
+//   onRequestSent()                      reset gate + safety signals
+//   onReasoningStart()                   (only when delta.reasoning_content seen)
+//   onStreamStart(nativeRail, inlineR)   first content token — rail + flag (signal b)
+//   onToken(t) …                         each delta.content token
+//   onAssistantMessage(clean,{terminal}) finalize the turn
+//
+// The gate (chat-turn.js onToken) opens eagerly ONLY when nativeRail AND
+// (reasoningSeen OR inline_reasoning:false). Otherwise it keeps the buffered-
+// until-boundary behavior (the no-leak fallback, also the entire XML-rail path).
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const { createTurnHandler } = require('../lib/commands/chat-turn');
+const { ChatHistory } = require('../lib/ui/chat-history');
+const { stripAnsi } = require('../lib/ui/utils');
+const { normalizeConfig } = require('../lib/config');
+
+// ---------------------------------------------------------------------------
+// Config: inline_reasoning (signal b) normalizes to an explicit boolean only.
+// ---------------------------------------------------------------------------
+test('inline_reasoning persists only as an explicit boolean on a models[] profile', () => {
+  const base = { api_base: 'http://x', api_key: 'k', model: 'm' };
+  const norm = (extra) => normalizeConfig({ models: [{ ...base, ...extra }] }).models[0];
+
+  assert.strictEqual('inline_reasoning' in norm({}), false, 'unset by default → assume might inline (safe)');
+  assert.strictEqual(norm({ inline_reasoning: false }).inline_reasoning, false, 'explicit false persists');
+  assert.strictEqual(norm({ inline_reasoning: true }).inline_reasoning, true, 'explicit true persists');
+  // Non-boolean junk is dropped (stays unset → safe default), never coerced.
+  assert.strictEqual('inline_reasoning' in norm({ inline_reasoning: 'false' }), false, 'string is dropped, not coerced');
+  assert.strictEqual('inline_reasoning' in norm({ inline_reasoning: 0 }), false, 'number is dropped, not coerced');
+});
+
+// Build a harness around the REAL createTurnHandler + REAL ChatHistory.
+// `streamed`  — tokens the gate passed live to chatHistory.streamToken.
+// `committed` — everything that reached scrollback (via the captured _commit).
+// `statuses`  — statusBar.update (state,label) pairs, to assert the transition.
+function harness(opts = {}) {
+  const streamed = [];
+  const committed = [];
+  const statuses = [];
+
+  const chatHistory = new ChatHistory();
+  chatHistory._commit = (t) => committed.push(t);
+  chatHistory._setDetail = () => {};
+  chatHistory._commitDetail = (t) => { if (t) committed.push(t); };
+  const origStream = chatHistory.streamToken.bind(chatHistory);
+  chatHistory.streamToken = (t) => { streamed.push(t); origStream(t); };
+
+  const statusBar = {
+    update: (state, label) => statuses.push([state, label]),
+    onToken() {}, addPendingTokens() {}, updateMetrics() {}, setCost() {},
+  };
+  const inputField = {
+    on() {}, removeListener() {}, releaseNavigation() {}, setDisabled() {},
+  };
+  const writerModule = {
+    startActivity() {}, updateActivity() {}, endActivity() {}, scrollback() {},
+  };
+
+  let scenario = async () => {};
+  const runAgentLoop = async (messages, model, maxIter, limit, loopOpts) => {
+    await scenario(loopOpts.callbacks);
+    return { messages, metrics: { turns: [] }, withheldActions: [] };
+  };
+
+  const ctx = {
+    inputField, statusBar, chatHistory, writerModule, runAgentLoop,
+    getConfig: () => ({ auth_token: 'tok', max_iterations: 50, show_cost: false, system_prompt_mode: 'system_role' }),
+    approxTokens: () => 0,
+    resolveCommand: () => null,
+    opts: { showThink: !!opts.showThink },
+    TAG_REGISTRY: {},
+    collapseListMsg() {}, handlePendingSelection() {}, showPendingStep() {},
+    activateNavCapture() {}, finalizeListMsg() {},
+    createChatIfNeeded: async () => {}, saveTurnToDashboard: async () => {}, saveSession() {},
+    messages: [], currentModel: 'm', debugMode: false, pendingImages: [],
+    chatSync: async () => '', resolvedSystemPrompt: '', resolvedTokenLimit: null, planMode: false,
+  };
+
+  const handler = createTurnHandler(ctx, {});
+  return { streamed, committed, statuses, handler, setScenario: (fn) => { scenario = fn; } };
+}
+
+// Simulate one streaming iteration in the exact callback order agent.js drives.
+function iteration(cb, { native, inlineReasoning, reasoningSeen, tokens, finalContent }) {
+  cb.onRequestSent();
+  if (reasoningSeen) cb.onReasoningStart();      // api.js wrappedOnReasoning (signal a)
+  cb.onStreamStart(native, inlineReasoning);     // first content token (rail + signal b)
+  for (const t of tokens) cb.onToken(t);
+  cb.onAssistantMessage(finalContent, { terminal: true });
+}
+
+function committedText(committed) { return stripAnsi(committed.join('')); }
+function countHeaders(committed) {
+  return (committedText(committed).match(/▸ AI-agent/g) || []).length;
+}
+
+// ---------------------------------------------------------------------------
+// Case 1 — native rail + reasoning_content seen → narration streams LIVE
+// ---------------------------------------------------------------------------
+test('native rail + reasoning_content seen: narration streams live (gate opens before finalize)', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: true, inlineReasoning: undefined, reasoningSeen: true,
+      tokens: ['Hello', ' world'], finalContent: 'Hello world',
+    });
+  });
+  await h.handler('hi');
+
+  assert.deepStrictEqual(h.streamed, ['Hello', ' world'], 'both narration tokens streamed live');
+  // Status bar transitioned to streaming on eager-open (not stuck on Thinking).
+  assert.ok(h.statuses.some(([s, l]) => s === 'streaming' && l === 'Streaming response'),
+    'status bar transitions to streaming on eager-open');
+});
+
+// ---------------------------------------------------------------------------
+// Case 2 — native rail + inline_reasoning:false → narration streams from token 1
+// ---------------------------------------------------------------------------
+test('native rail + inline_reasoning:false: narration streams live from token 1', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: true, inlineReasoning: false, reasoningSeen: false,
+      tokens: ['The', ' answer'], finalContent: 'The answer',
+    });
+  });
+  await h.handler('hi');
+
+  assert.deepStrictEqual(h.streamed, ['The', ' answer'], 'streams live from the first token, no reasoning_content needed');
+  assert.ok(h.statuses.some(([s, l]) => s === 'streaming' && l === 'Streaming response'),
+    'status bar transitions to streaming on eager-open');
+});
+
+// ---------------------------------------------------------------------------
+// Case 3 — native rail, NO signal, bare-text-then-orphan-</think>:
+//          reasoning stays HIDDEN (no leak); narration after </think> streams.
+// ---------------------------------------------------------------------------
+test('native rail + NO signal + implicit think: reasoning hidden, post-</think> narration streams', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: true, inlineReasoning: undefined, reasoningSeen: false,
+      tokens: ['Let', ' me', ' think', '</think>', 'The answer', ' is 42'],
+      finalContent: 'The answer is 42',
+    });
+  });
+  await h.handler('hi');
+
+  // The bare reasoning tokens were buffered+discarded — never streamed, never committed.
+  assert.deepStrictEqual(h.streamed, ['The answer', ' is 42'], 'only post-</think> narration streams');
+  const text = committedText(h.committed);
+  for (const leak of ['Let', 'me', 'think']) {
+    assert.ok(!text.includes(leak), `reasoning token ${JSON.stringify(leak)} must NOT reach scrollback (no leak)`);
+  }
+  assert.ok(text.includes('The answer is 42'), 'narration is rendered');
+});
+
+// ---------------------------------------------------------------------------
+// Case 4 — XML rail (nativeRail false): identical to old behavior even WITH
+//          reasoning_content seen. The eager-open guard excludes the XML rail.
+// ---------------------------------------------------------------------------
+test('XML rail: buffered-until-</think> behavior unchanged even with reasoning seen (regression guard)', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: false, inlineReasoning: false, reasoningSeen: true,
+      tokens: ['secret', ' reasoning', '</think>', 'visible answer'],
+      finalContent: 'visible answer',
+    });
+  });
+  await h.handler('hi');
+
+  assert.deepStrictEqual(h.streamed, ['visible answer'], 'XML rail buffers leading text despite signals');
+  const text = committedText(h.committed);
+  assert.ok(!text.includes('secret'), 'XML rail does not leak buffered reasoning');
+  assert.ok(text.includes('visible answer'), 'XML rail narration after </think> still renders');
+});
+
+// ---------------------------------------------------------------------------
+// Case 5 — finalize after a live stream does NOT double-print (one AI header).
+// ---------------------------------------------------------------------------
+test('finalize after live stream: no double-print (single AI bubble, no _buildAI re-synthesis)', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: true, inlineReasoning: false, reasoningSeen: false,
+      tokens: ['unique-token-α'], finalContent: 'unique-token-α',
+    });
+  });
+  await h.handler('hi');
+
+  const text = committedText(h.committed);
+  assert.strictEqual(countHeaders(h.committed), 1, 'exactly one ▸ AI-agent header (no re-synthesized bubble)');
+  assert.strictEqual((text.match(/unique-token-α/g) || []).length, 1, 'narration committed exactly once');
+});
+
+// ---------------------------------------------------------------------------
+// Case 6 (P2 regression) — native rail, reasoning_content seen (eager-open),
+// content carries a STRAY inline </think>. MiniMax emits reasoning via BOTH
+// reasoning_content AND an orphan </think> terminator in content; the
+// StreamParser passes that orphan tag through verbatim (its `/think` form is not
+// a TAG_REGISTRY key). The eager-open path must DROP it (not stream it) while the
+// surrounding narration streams live. Regression from 938f583: the eager-open
+// fell through to streamToken and skipped the orphan-drop guard for every token.
+// (f) The committed scrollback — what persists/replays — stays clean too.
+// ---------------------------------------------------------------------------
+test('native rail + reasoning_content + inline orphan </think>: stray tag dropped, narration streams (P2)', async () => {
+  const h = harness();
+  h.setScenario(async (cb) => {
+    iteration(cb, {
+      native: true, inlineReasoning: undefined, reasoningSeen: true,
+      tokens: ['Here', ' is', '</think>', ' the answer'],
+      finalContent: 'Here is the answer',
+    });
+  });
+  await h.handler('hi');
+
+  assert.deepStrictEqual(h.streamed, ['Here', ' is', ' the answer'],
+    'orphan </think> dropped from the live stream; surrounding narration streamed live');
+  const text = committedText(h.committed);
+  assert.ok(!text.includes('</think>'), 'the stray </think> never reaches scrollback (live + persisted clean)');
+  assert.ok(text.includes('Here is the answer'), 'narration is rendered');
+});
+
+// Every closing reasoning tag the StreamParser emits raw must drop on the
+// eager-open path — for the WHOLE stream, not just the first token. The set
+// matches the registered visual tags (think/reasoning/reflection/plan); their
+// `/tag` closing form is never a TAG_REGISTRY key, so all stream verbatim.
+test('native rail eager-open: orphan think/reasoning/reflection/plan close tags all dropped (P2)', async () => {
+  for (const tag of ['think', 'reasoning', 'reflection', 'plan']) {
+    const h = harness();
+    h.setScenario(async (cb) => {
+      iteration(cb, {
+        native: true, inlineReasoning: false, reasoningSeen: false,
+        tokens: ['ok ', `</${tag}>`, 'done'],
+        finalContent: 'ok done',
+      });
+    });
+    await h.handler('hi');
+    assert.deepStrictEqual(h.streamed, ['ok ', 'done'], `</${tag}> dropped on the eager-open path`);
+    assert.ok(!committedText(h.committed).includes(`</${tag}>`), `</${tag}> never committed`);
+  }
+});

package/test/output-heredoc-leak.test.js

@@ -0,0 +1,195 @@
+'use strict';
+
+// Output Refactor · Phase 4 (fix A) — the heredoc stray-lines / stuck-spinner
+// leak regression.
+//
+// THE BUG: a multi-line heredoc command (`python3 - <<'PY'\n…\nPY`) reached the
+// status-bar label as a raw `input.slice(0, 40)` with no newline flattening
+// (chat-turn.js:187/197). The embedded \n rode into the live region, so a single
+// LOGICAL live line spanned 2+ PHYSICAL rows. _liveHeight counts logical lines,
+// so the erase (`\x1b[{up}A\r\x1b[J`) moved up too few rows and \x1b[J cleared
+// too low → the top physical row(s) of each repaint leaked into scrollback.
+// Phase 3 made `tool` an ANIM_STATE, so the undercounting erase now runs at the
+// ~10 Hz driver cadence → dozens of stranded `────` rules and a stuck
+// `⣯ Running shell: …` row over a few seconds.
+//
+// THE FIX (A): (1) flatten the label at source via normalizeCmdForDisplay; and
+// (2) harden _fitOneRow to strip embedded control chars so the 1-logical=
+// 1-physical invariant holds regardless of caller; (3) consolidate the erase
+// math into one helper. These tests assert each part and, via a tiny VT model,
+// that no scrollback residue accumulates across repaints.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const path = require('path');
+
+const writer = require('../lib/ui/writer');
+const { normalizeCmdForDisplay } = require('../lib/ui/format');
+
+// ── A minimal ANSI terminal model ────────────────────────────────────────────
+//
+// Interprets the exact escape vocabulary the writer emits (cursor up/right,
+// \r, \x1b[J erase-to-end-of-screen, SGR/mode toggles ignored) over a growing
+// row buffer. `rows.length` is the total physical-row footprint; if the erase
+// undercounts, repeated repaints strand stale rows and the buffer GROWS.
+function makeVT() {
+  const rows = [''];
+  let r = 0, c = 0;
+  function ensure(row) { while (rows.length <= row) rows.push(''); }
+  function put(ch) {
+    ensure(r);
+    const line = rows[r];
+    rows[r] = line.slice(0, c) + ch + line.slice(c + 1);
+    c++;
+  }
+  function write(s) {
+    let i = 0;
+    while (i < s.length) {
+      const ch = s[i];
+      if (ch === '\x1b' && s[i + 1] === '[') {
+        let j = i + 2;
+        let params = '';
+        while (j < s.length && /[0-9;?<>]/.test(s[j])) { params += s[j]; j++; }
+        const final = s[j];
+        const n = parseInt(params, 10) || 1;
+        if (final === 'A') r = Math.max(0, r - n);
+        else if (final === 'B') { r += n; ensure(r); }
+        else if (final === 'C') c += n;
+        else if (final === 'D') c = Math.max(0, c - n);
+        else if (final === 'J') { ensure(r); rows[r] = rows[r].slice(0, c); rows.length = r + 1; }
+        // m / h / l / r / u → presentation only, ignore.
+        i = j + 1;
+        continue;
+      }
+      if (ch === '\x1b') { i += 1; continue; } // bare ESC (shouldn't happen)
+      if (ch === '\n') { r++; c = 0; ensure(r); i++; continue; }
+      if (ch === '\r') { c = 0; i++; continue; }
+      put(ch); i++;
+    }
+  }
+  return { rows, write };
+}
+
+// Drive the real writer against a VT, returning the VT + a reset helper. The
+// writer is a singleton; clearLive() resets its module state between tests.
+function withVT(fn) {
+  const vt = makeVT();
+  const out = process.stdout;
+  const prev = { isTTY: out.isTTY, columns: out.columns, rows: out.rows, write: out.write };
+  out.isTTY = true;
+  out.columns = 80;
+  out.rows = 24;
+  out.write = (s) => { vt.write(String(s)); return true; };
+  return (async () => {
+    try {
+      await fn(vt);
+    } finally {
+      await writer.clearLive();
+      await writer.flush();
+      out.isTTY = prev.isTTY;
+      out.columns = prev.columns;
+      out.rows = prev.rows;
+      out.write = prev.write;
+    }
+  })();
+}
+
+const HEREDOC = "python3 - <<'PY'\nprint('hi')\nPY";          // 2 embedded \n
+const ONE_NL  = "echo a\necho b";                              // 1 embedded \n
+
+// ── Part 1 — label flattening at source ──────────────────────────────────────
+
+test('Part 1: normalizeCmdForDisplay flattens newlines/tabs to a single line', () => {
+  const flat = normalizeCmdForDisplay(HEREDOC);
+  assert.ok(!/[\n\r\t]/.test(flat), 'no embedded control whitespace survives');
+  assert.strictEqual(flat, "python3 - <<'PY' print('hi') PY");
+  // The 40-char slice operates on the flattened text → still single-line.
+  const short = flat.length > 40 ? flat.slice(0, 40) + '…' : flat;
+  assert.ok(!/[\n\r\t]/.test(short), 'sliced label is single-line');
+});
+
+test('Part 1: normalizeCmdForDisplay applied at both label sites in chat-turn.js', () => {
+  const src = fs.readFileSync(path.join(__dirname, '../lib/commands/chat-turn.js'), 'utf8');
+  // The import is present and the helper is used to build the `short` label.
+  assert.ok(/require\('\.\.\/ui\/format'\)/.test(src), 'format module required');
+  const flatUses = (src.match(/normalizeCmdForDisplay\(input\)/g) || []).length;
+  assert.ok(flatUses >= 2, `expected ≥2 flattened label sites, found ${flatUses}`);
+  // And the raw un-flattened slice that caused the leak is gone.
+  assert.ok(!/input\.slice\(0, 40\)/.test(src), 'raw input.slice(0,40) removed');
+});
+
+// ── Part 2 — _fitOneRow control-char hardening (the structural guard) ─────────
+
+test('Part 2: setLive with an embedded \\n draws exactly one physical row', () =>
+  withVT(async (vt) => {
+    await writer.setLive(['alpha\nbeta']);
+    await writer.flush();
+    assert.strictEqual(writer.getLiveHeight(), 1, 'one logical live line');
+    // The fitted row replaced \n with a space → it lives on ONE physical row.
+    const drawn = vt.rows.find((l) => l.includes('alpha'));
+    assert.ok(drawn, 'the live line is present');
+    assert.ok(drawn.includes('alpha beta'), `newline flattened to space: ${JSON.stringify(drawn)}`);
+    assert.ok(!vt.rows.some((l) => /^beta/.test(l)), 'beta did NOT spill onto its own row');
+  }));
+
+test('Part 2: a clean single-line row is unchanged (no regression)', () =>
+  withVT(async (vt) => {
+    await writer.setLive(['just one clean line']);
+    await writer.flush();
+    assert.strictEqual(writer.getLiveHeight(), 1);
+    assert.ok(vt.rows.some((l) => l === 'just one clean line'),
+      'clean line rendered verbatim');
+  }));
+
+// ── The bug (regression): no scrollback growth across repaints ────────────────
+
+test('regression: multi-line heredoc label leaks no rows across repaints', () =>
+  withVT(async (vt) => {
+    // Simulate the live status label being a (deliberately un-flattened) heredoc
+    // command — the writer-level guard (Part 2) must contain it even if a future
+    // caller forgets to flatten. This is the exact pre-fix leak scenario.
+    const label = `⣯ Running shell: ${HEREDOC}`;
+    await writer.setLive([label]);
+    await writer.flush();
+    const baseline = vt.rows.length;
+    assert.strictEqual(writer.getLiveHeight(), 1, 'one logical row');
+
+    // Phase-3 amplification: the animation driver repaints at ~10 Hz. Replay a
+    // burst of repaints with the SAME live content (what redrawLive does each
+    // tick) and assert the committed footprint never grows.
+    for (let i = 0; i < 12; i++) {
+      await writer.redrawLive();
+      await writer.flush();
+      assert.strictEqual(vt.rows.length, baseline,
+        `repaint ${i + 1}: scrollback grew (${vt.rows.length} > ${baseline}) — leak`);
+    }
+    // No stray separator rule or stuck spinner row accumulated.
+    const spinnerRows = vt.rows.filter((l) => l.includes('Running shell')).length;
+    assert.ok(spinnerRows <= 1, `at most one spinner row, found ${spinnerRows}`);
+  }));
+
+test('regression: single-embedded-newline label (stray ──── case) leaks nothing', () =>
+  withVT(async (vt) => {
+    await writer.setLive([`────── ${ONE_NL}`]);
+    await writer.flush();
+    const baseline = vt.rows.length;
+    for (let i = 0; i < 8; i++) {
+      await writer.redrawLive();
+      await writer.flush();
+      assert.strictEqual(vt.rows.length, baseline, `repaint ${i + 1} grew the buffer`);
+    }
+  }));
+
+// ── Part 3 — erase math consolidated into one helper ──────────────────────────
+
+test('Part 3: the erase math lives in exactly one helper', () => {
+  const src = fs.readFileSync(path.join(__dirname, '../lib/ui/writer.js'), 'utf8');
+  const mathCopies = (src.match(/Math\.max\(0, _liveHeight - offset\)/g) || []).length;
+  assert.strictEqual(mathCopies, 1, 'erase math appears once (consolidated)');
+  assert.ok(/function _eraseSeqForHeight\(\)/.test(src), 'shared helper exists');
+  assert.ok(/_eraseLiveSeq[\s\S]{0,80}_eraseSeqForHeight\(\)/.test(src),
+    '_eraseLiveSeq delegates to the shared helper');
+  assert.ok(/parts\.push\(_eraseSeqForHeight\(\)\)/.test(src),
+    'teardown uses the shared helper');
+});