@semalt-ai/code 1.8.5 → 1.20.0

package/test/detail-band-tab-flatten.test.js

@@ -0,0 +1,242 @@
+'use strict';
+
+// Detail-band TAB flattening — the "1 logical line == 1 physical row" invariant.
+//
+// The writer's erase/commit math (_liveHeight via displayRows) assumes each
+// emitted preview line is exactly one physical row, so the band's logical-line
+// count equals its physical-row count. That invariant is FALSE for raw shell
+// output containing TABs (and other C0 controls): the detail band is emitted RAW
+// (writer _drawLiveSeq, bypassing _fitOneRow — the only place embedded TAB/C0/DEL
+// are flattened to spaces), and truncateVisible/termWidth count a TAB as 1 column
+// while the terminal advances it to the next 8-col tab stop. So a body line
+// "fitted to cols-1" still renders WIDER than cols and WRAPS to ≥2 physical rows,
+// and the writer's cursor-up erase math under-counts → stranded/duplicated rows.
+//
+// THE FIX (option a): restore "1 logical line == 1 physical row" by flattening the
+// detail band's lines the same way every other live row already is. Applied at the
+// single seam _renderOutputPreviewLines (the source for the live band, the
+// immediate/committed scrollback, and replay), so all render paths are consistent:
+//   1. C0/DEL controls (TAB included) → space (1:1, width preserved).
+//   2. stray non-SGR escapes (e.g. \x1b[K) dropped, closing the stripAnsi-only-'m'
+//      over-count gap. SGR color is preserved.
+//
+// These tests gate: (a) a TAB-bearing band line == exactly 1 physical row;
+// (b) the real writer renders a large TAB band at its true physical height and the
+// one-shot commit clears it with no wide/stranded rows (real writer + TAB/wrap-aware
+// VT); (c) a stray \x1b[K line counts as 1 row (no over-count); (d) plain-ASCII
+// output is byte-identical (flatten is a no-op); (e) live vs committed render of the
+// same TAB output stay consistent (both flattened).
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+process.stdout.isTTY = true;
+delete process.env.NO_COLOR;
+
+const { ChatHistory } = require('../lib/ui/chat-history');
+const writer = require('../lib/ui/writer');
+const { displayRows } = require('../lib/ui/utils');
+
+// Strip EVERY escape (SGR + non-SGR CSI + bare ESC) so we can measure visible
+// content the way the real terminal renders it.
+const stripAllEsc = (s) =>
+  String(s).replace(/\x1b\[[0-9;?<>]*[A-Za-z]/g, '').replace(/\x1b./g, '');
+
+// Ground-truth physical-row count for a single emitted line on a REAL terminal:
+// expand TABs to the next 8-col tab stop and autowrap at `cols`. This is what
+// displayRows/termWidth CANNOT see (they count a TAB as 1), so it is the honest
+// oracle the fix must satisfy.
+function realPhysicalRows(line, cols) {
+  const plain = stripAllEsc(line);
+  let col = 0;
+  let rows = 1;
+  const advance = () => { col++; if (col >= cols) { rows++; col = 0; } };
+  for (const ch of plain) {
+    if (ch === '\t') {
+      const target = (Math.floor(col / 8) + 1) * 8;
+      for (let k = col; k < target; k++) advance();
+    } else {
+      advance();
+    }
+  }
+  return rows;
+}
+
+function instrument(ch) {
+  const log = [];
+  ch._commit = (t) => log.push({ op: 'scrollback', text: t });
+  ch._setDetail = (lines) => log.push({ op: 'setDetail', lines: lines.slice() });
+  ch._commitDetail = (t) => log.push({ op: 'commitDetail', text: t });
+  ch._notifyLive = () => {};
+  return log;
+}
+
+const lastSetDetail = (log) => log.filter((e) => e.op === 'setDetail').pop().lines;
+
+// A line whose TAB=1 width is ≤ the fit budget (so formatOutputPreview keeps the
+// TABs in the "fitted" line) but whose tab-EXPANDED width blows far past cols.
+// "x\t" × 37 = 74 cols counted (cols-6 budget at 80), but expands to ~300 cols.
+const TAB_LINE = 'x\t'.repeat(37);
+const tabOutput = (n) => Array.from({ length: n }, () => TAB_LINE).join('\n');
+
+// ── (a) a TAB-bearing band line occupies EXACTLY one physical row ─────────────
+
+test('(a) TAB flatten: every preview band line is exactly one physical row; TABs become spaces', () => {
+  const ch = new ChatHistory();
+  const log = instrument(ch);
+  ch.deferToolOutput({ role: 'tool', tag: 'shell', content: '', output: tabOutput(10), previewLines: 5 });
+  const band = lastSetDetail(log);
+
+  // Sanity: the RAW (pre-flatten) line would have wrapped — prove the test bites.
+  assert.ok(realPhysicalRows(`     ${TAB_LINE}`, 80) > 1,
+    'precondition: a raw TAB line tab-expands past cols (would wrap without the fix)');
+
+  const bodyRows = band.filter((l) => /x/.test(stripAllEsc(l)));
+  assert.ok(bodyRows.length >= 5, 'the preview TAB lines surfaced');
+  for (const line of bodyRows) {
+    assert.ok(!stripAllEsc(line).includes('\t'), 'TAB flattened to spaces in the band line');
+    assert.strictEqual(realPhysicalRows(line, 80), 1, 'tab-aware: line is exactly one physical row');
+    assert.strictEqual(displayRows(line, 80), 1, 'writer-side displayRows agrees it is one row');
+  }
+  // Logical-line count == physical-row count → the writer's erase math is honest.
+  assert.strictEqual(writer.physicalRows(band, 80), band.length,
+    'band logical line count equals its physical-row total (the restored invariant)');
+});
+
+// ── (b) the real writer renders the band at its true height; commit clears it ──
+
+// Minimal terminal model that expands TABs to 8-col tab stops AND autowraps at
+// `cols` — so a band line that wraps shows up as MULTIPLE physical rows, exactly
+// what the writer's TAB=1 _liveHeight cannot see. rows.length is the committed
+// physical footprint; an undercounting erase makes it GROW.
+function makeTabVT(cols) {
+  const rows = [''];
+  let r = 0, c = 0;
+  const ensure = (row) => { while (rows.length <= row) rows.push(''); };
+  const putChar = (ch) => {
+    if (c >= cols) { r++; c = 0; ensure(r); }
+    ensure(r);
+    rows[r] = rows[r].slice(0, c) + ch + rows[r].slice(c + 1);
+    c++;
+  };
+  function write(s) {
+    let i = 0;
+    while (i < s.length) {
+      const ch = s[i];
+      if (ch === '\x1b' && s[i + 1] === '[') {
+        let j = i + 2, params = '';
+        while (j < s.length && /[0-9;?<>]/.test(s[j])) { params += s[j]; j++; }
+        const final = s[j]; const n = parseInt(params, 10) || 1;
+        if (final === 'A') r = Math.max(0, r - n);
+        else if (final === 'B') { r += n; ensure(r); }
+        else if (final === 'C') c += n;
+        else if (final === 'D') c = Math.max(0, c - n);
+        else if (final === 'J') { ensure(r); rows[r] = rows[r].slice(0, c); rows.length = r + 1; }
+        i = j + 1; continue;
+      }
+      if (ch === '\x1b') { i += 1; continue; }
+      if (ch === '\n') { r++; c = 0; ensure(r); i++; continue; }
+      if (ch === '\r') { c = 0; i++; continue; }
+      if (ch === '\t') {
+        const target = (Math.floor(c / 8) + 1) * 8;
+        for (let k = c; k < target; k++) putChar(' ');
+        i++; continue;
+      }
+      putChar(ch); i++;
+    }
+  }
+  return { rows, write };
+}
+
+test('(b) real writer: large TAB band renders at its true physical height; commit clears it, no wide rows', () => {
+  const vt = makeTabVT(80);
+  const out = process.stdout;
+  const prev = { isTTY: out.isTTY, columns: out.columns, rows: out.rows, write: out.write };
+  out.isTTY = true; out.columns = 80; out.rows = 24;
+  out.write = (s) => { vt.write(String(s)); return true; };
+  const ch = new ChatHistory();
+  ch._notifyLive = () => {};
+  return (async () => {
+    try {
+      await writer.setLive(['separator', 'status', '> input']); // 3 single-row chrome lines
+      await writer.flush();
+      // Install the deferred band once (no toggling — the band is a one-shot static
+      // path now). Every emitted band line is flattened to a single physical row.
+      ch.deferToolOutput({ role: 'tool', tag: 'shell', content: '', output: tabOutput(8), previewLines: 5 });
+      await writer.flush();
+      const band = writer.getDetailLines();
+      for (const l of band) assert.strictEqual(realPhysicalRows(l, 80), 1, 'each emitted band row is one physical row');
+      assert.strictEqual(writer.getLiveHeight(), band.length + 3,
+        '_liveHeight equals the true physical height (band + chrome)');
+
+      // Commit once → band moves to scrollback, the live region is back to chrome.
+      ch.commitDeferredDetail();
+      await writer.flush();
+      assert.deepStrictEqual(writer.getDetailLines(), [], 'band cleared after commit');
+      assert.ok(vt.rows.every((row) => stripAllEsc(row).length <= 80), 'no row wider than cols remains on screen');
+    } finally {
+      await writer.clearLive(); await writer.flush();
+      out.isTTY = prev.isTTY; out.columns = prev.columns; out.rows = prev.rows; out.write = prev.write;
+    }
+  })();
+});
+
+// ── (c) a stray non-SGR escape (\x1b[K) counts/renders as one row, no over-count ─
+
+test('(c) stray escape: a band line with \\x1b[K is one physical row (no displayRows over-count)', () => {
+  const ch = new ChatHistory();
+  const log = instrument(ch);
+  // Visible content sized to the fit budget (cols-6 = 74), then a stray \x1b[K.
+  // truncateVisible keeps the (0-width) CSI; pre-fix displayRows' SGR-only strip
+  // counts ESC[K as 3 cols → 5(indent)+74+3 = 82 > 80 → a phantom 2nd row.
+  const noisy = 'z'.repeat(74) + '\x1b[K';
+  ch.deferToolOutput({ role: 'tool', tag: 'shell', content: '', output: Array(8).fill(noisy).join('\n'), previewLines: 5 });
+  const band = lastSetDetail(log);
+  const bodyRows = band.filter((l) => /z/.test(stripAllEsc(l)));
+  assert.ok(bodyRows.length >= 5, 'the preview lines surfaced');
+  for (const line of bodyRows) {
+    assert.ok(!/\x1b\[K/.test(line), 'stray non-SGR escape dropped from the band line');
+    assert.strictEqual(displayRows(line, 80), 1, 'no over-count: exactly one physical row');
+    assert.strictEqual(realPhysicalRows(line, 80), 1, 'tab/wrap-aware oracle agrees');
+  }
+});
+
+// ── (d) plain-ASCII output is byte-identical — flatten is a no-op ──────────────
+
+test('(d) plain ASCII: flatten is a no-op — committed band byte-identical to addMessage', () => {
+  const mkLines = (n) => Array.from({ length: n }, (_, i) => `line ${i + 1}`).join('\n');
+  // Reference: the immediate addMessage commit for the same clean output.
+  const ref = new ChatHistory();
+  const refOut = [];
+  ref._commit = (t) => refOut.push(t);
+  ref._notifyLive = () => {};
+  ref.addMessage({ role: 'tool', tag: 'shell', content: '', output: mkLines(20), previewLines: 5 });
+  const expected = refOut.join('');
+
+  const ch = new ChatHistory();
+  const log = instrument(ch);
+  ch.deferToolOutput({ role: 'tool', tag: 'shell', content: '', output: mkLines(20), previewLines: 5 });
+  ch.commitDeferredDetail();
+  const commit = log.filter((e) => e.op === 'commitDetail')[0].text;
+  assert.strictEqual(commit, expected, 'clean output commit is byte-identical to addMessage (flatten no-op)');
+});
+
+// ── (e) live vs committed render of the same TAB output stay consistent ────────
+
+test('(e) live/replay consistency: live band and committed text are BOTH flattened and agree on body', () => {
+  const ch = new ChatHistory();
+  const log = instrument(ch);
+  ch.deferToolOutput({ role: 'tool', tag: 'shell', content: '', output: tabOutput(10), previewLines: 5 });
+  const liveBand = lastSetDetail(log);
+  ch.commitDeferredDetail();             // committed (replay) text
+  const committed = log.filter((e) => e.op === 'commitDetail')[0].text;
+
+  assert.ok(!liveBand.some((l) => l.includes('\t')), 'live band carries no raw TABs');
+  assert.ok(!committed.includes('\t'), 'committed/replay text carries no raw TABs');
+
+  // The flattened body rows match between the live band and the commit (the band is
+  // committed verbatim — installed once, committed once).
+  const liveBody = liveBand.map(stripAllEsc).filter((l) => /x/.test(l));
+  const commitBody = committed.split('\n').map(stripAllEsc).filter((l) => /x/.test(l));
+  assert.deepStrictEqual(commitBody, liveBody, 'live and committed body rows are identical (both flattened)');
+});

package/test/download-allow-anywhere.test.js

@@ -0,0 +1,66 @@
+'use strict';
+
+// Pre-Task 4.0b — the --allow-anywhere positive branch for `download`.
+// isPathSafe reads the --allow-anywhere flag from process.argv exactly once at
+// module load, so this branch must run in its own process with the flag set
+// before lib/tools is required. `node --test` runs each test file in a separate
+// process, giving us that isolation.
+
+const os = require('node:os');
+const fs = require('node:fs');
+const path = require('node:path');
+
+if (!process.argv.includes('--allow-anywhere')) process.argv.push('--allow-anywhere');
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+const http = require('node:http');
+
+const ui = require('../lib/ui');
+const { createPermissionManager } = require('../lib/permissions');
+const { createToolExecutor } = require('../lib/tools');
+
+let CWD;
+let PREV_CWD;
+let OUTSIDE_DIR;
+
+function mkExec() {
+  const pm = createPermissionManager(ui, {});
+  return createToolExecutor(pm, ui, () => ({
+    max_file_size_kb: 512,
+    command_timeout_ms: 30000,
+    download_max_bytes: 1048576,
+  }));
+}
+
+async function withServer(body, fn) {
+  const server = http.createServer((req, res) => { res.writeHead(200); res.end(body); });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  const { port } = server.address();
+  try {
+    return await fn(port);
+  } finally {
+    await new Promise((r) => server.close(r));
+  }
+}
+
+before(() => {
+  PREV_CWD = process.cwd();
+  CWD = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-dl-aa-cwd-'));
+  OUTSIDE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-dl-aa-out-'));
+  process.chdir(CWD);
+});
+
+after(() => {
+  process.chdir(PREV_CWD);
+});
+
+test('download to a destination outside CWD is allowed under --allow-anywhere', async () => {
+  const exec = mkExec();
+  const outside = path.join(OUTSIDE_DIR, 'fetched.txt');
+  await withServer('outside-ok', async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/x`, outside);
+    assert.strictEqual(r.status, 'ok');
+    assert.strictEqual(fs.readFileSync(outside, 'utf8'), 'outside-ok');
+  });
+});

package/test/download-confine.test.js

@@ -0,0 +1,153 @@
+'use strict';
+
+// Pre-Task 4.0b — confine the `download` tool.
+// download was the one write path not routed through isPathSafe, with no byte
+// cap and no --readonly check. These tests pin the new confinement: path
+// refusal, secret-file guard, --readonly block, a byte cap that aborts and
+// cleans up the partial file, and an in-bounds happy path that still works.
+//
+// Home-based paths (config.json / memory.json / audit.log) are redirected into
+// a temp dir BEFORE any lib module loads, so the secret-file guard resolves
+// against the temp config path. The process argv here has neither
+// --allow-anywhere nor --readonly, so isPathSafe defaults to "confined to CWD"
+// (the allow-anywhere positive branch lives in download-allow-anywhere.test.js,
+// which needs the flag set at module load and so runs in its own process).
+
+const os = require('node:os');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-dl-home-'));
+const PREV_HOME = process.env.HOME;
+const PREV_USERPROFILE = process.env.USERPROFILE;
+process.env.HOME = TMP_HOME;
+process.env.USERPROFILE = TMP_HOME;
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+const http = require('node:http');
+
+const ui = require('../lib/ui');
+const { createPermissionManager } = require('../lib/permissions');
+const { createToolExecutor } = require('../lib/tools');
+const { CONFIG_PATH } = require('../lib/constants');
+
+let CWD;
+let PREV_CWD;
+
+// Build a tool executor with an injectable config + permission-manager options.
+function mkExec({ config = {}, pmOpts = {} } = {}) {
+  const pm = createPermissionManager(ui, pmOpts);
+  return createToolExecutor(pm, ui, () => ({
+    max_file_size_kb: 512,
+    command_timeout_ms: 30000,
+    download_max_bytes: 1048576,
+    ...config,
+  }));
+}
+
+// Spin up a localhost server that returns the given body, run fn(port), close.
+async function withServer(body, fn) {
+  const server = http.createServer((req, res) => { res.writeHead(200); res.end(body); });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  const { port } = server.address();
+  try {
+    return await fn(port);
+  } finally {
+    await new Promise((r) => server.close(r));
+  }
+}
+
+before(() => {
+  PREV_CWD = process.cwd();
+  CWD = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-dl-cwd-'));
+  process.chdir(CWD);
+});
+
+after(() => {
+  process.chdir(PREV_CWD);
+  if (PREV_HOME === undefined) delete process.env.HOME; else process.env.HOME = PREV_HOME;
+  if (PREV_USERPROFILE === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = PREV_USERPROFILE;
+});
+
+// ---------------------------------------------------------------------------
+// happy path — an in-bounds download still works
+// ---------------------------------------------------------------------------
+
+test('download saves an in-bounds URL to a file in cwd', async () => {
+  const exec = mkExec();
+  await withServer('filedata', async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/payload.txt`);
+    assert.strictEqual(r.status, 'ok');
+    assert.strictEqual(fs.readFileSync(path.join(CWD, 'payload.txt'), 'utf8'), 'filedata');
+  });
+});
+
+// ---------------------------------------------------------------------------
+// byte cap — oversized download is aborted and the partial file removed
+// ---------------------------------------------------------------------------
+
+test('download aborts past the byte cap and removes the partial file', async () => {
+  const exec = mkExec({ config: { download_max_bytes: 16 } });
+  const big = 'a'.repeat(5000);
+  await withServer(big, async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/big.bin`);
+    assert.ok(r.error, 'should return an error');
+    assert.match(r.error, /cap/i);
+    assert.strictEqual(r.capped, true);
+    assert.strictEqual(fs.existsSync(path.join(CWD, 'big.bin')), false, 'partial file must be cleaned up');
+  });
+});
+
+// ---------------------------------------------------------------------------
+// path confinement — a destination outside CWD is refused
+// ---------------------------------------------------------------------------
+
+test('download to a destination outside CWD is refused', async () => {
+  const exec = mkExec();
+  const outside = path.join(os.tmpdir(), 'semalt-dl-outside-' + process.pid + '.txt');
+  await withServer('data', async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/x`, outside);
+    assert.ok(r.error, 'should be refused');
+    assert.match(r.error, /outside allowed area/i);
+    assert.strictEqual(fs.existsSync(outside), false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// secret-file guard — refuses writing over a protected secret path
+// ---------------------------------------------------------------------------
+
+test('download over a protected secret path is refused by the secret guard', async () => {
+  const exec = mkExec();
+  await withServer('data', async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/x`, CONFIG_PATH);
+    assert.ok(r.error, 'should be refused');
+    assert.match(r.error, /secret|credential/i);
+    assert.strictEqual(fs.existsSync(CONFIG_PATH), false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// --readonly — blocks the mutating download
+// ---------------------------------------------------------------------------
+
+test('download is blocked under --readonly', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  await withServer('data', async (port) => {
+    const r = await exec.agentExecFile('download', `http://127.0.0.1:${port}/ro.txt`);
+    assert.ok(r.error, 'should be blocked');
+    assert.match(r.error, /readonly/i);
+    assert.strictEqual(fs.existsSync(path.join(CWD, 'ro.txt')), false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// describePermission short-circuits download under --readonly (parity with
+// delete/move/copy) so no approval prompt precedes the deterministic block.
+// ---------------------------------------------------------------------------
+
+test('describePermission returns null for download under --readonly', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  assert.strictEqual(await exec.describePermission(['download', 'http://x/y.zip']), null);
+});