@semalt-ai/code 1.8.5 → 1.19.0

package/test/read-paginate.test.js

@@ -0,0 +1,275 @@
+'use strict';
+
+// Task W.7 — Paginate read_file (start_line/end_line + PARTIAL notice + token net).
+//
+// THE CHANGE these tests pin: read_file used to dump the WHOLE file verbatim into
+// context (the only guard was a hard byte refusal at max_file_size_kb). Now it
+// paginates: under a line cap it reads fully (NO regression for the common
+// small-file case), over it returns the first page + a PARTIAL notice telling the
+// model the range, the total, and the start_line for the next page. start_line/
+// end_line return an explicit slice (still line-capped); a token net bounds
+// pathological long lines.
+//
+// Step 0 finding: edit_file is LINE-NUMBER-based (lines[lineNum-1] = content) and
+// replace_in_file is MATCH-based (regex on a search string). A MIX. Decision:
+// line numbers are OPTIONAL, default OFF (show_line_numbers). Rationale —
+// replace_in_file (the match-based path) needs raw, copyable line text, so
+// default-off keeps snippets verbatim AND avoids the ~1.7x token tax on every
+// read; edit_file (line-based) gets correct line refs on demand via
+// show_line_numbers. Tests assert what the MODEL receives (the audit's method).
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const { formatReadResult } = require('../lib/agent');
+const {
+  DEFAULT_READ_LINE_CAP,
+  DEFAULT_READ_MAX_TOKENS,
+} = require('../lib/constants');
+
+// ---------------------------------------------------------------------------
+// Part A — pure model-facing pagination (formatReadResult)
+// ---------------------------------------------------------------------------
+
+function nLines(n, prefix = 'L') {
+  return Array.from({ length: n }, (_, i) => `${prefix}${i + 1}`).join('\n');
+}
+
+test('small file under the line cap, no range, no numbers → byte-for-byte today (NO regression)', () => {
+  const content = 'alpha\nbeta\ngamma\n'; // trailing newline included
+  const out = formatReadResult({ content, path: 'f.txt' });
+  // Exactly the pre-W.7 shape: `File <path>:\n<content>` with nothing added.
+  assert.strictEqual(out, `File f.txt:\n${content}`);
+  assert.doesNotMatch(out, /PARTIAL/);
+  assert.doesNotMatch(out, /token-capped/);
+});
+
+test('split/join round-trips exactly — equivalence holds for content without a trailing newline', () => {
+  const content = 'one\ntwo\nthree';
+  const out = formatReadResult({ content, path: 'g.txt' });
+  assert.strictEqual(out, `File g.txt:\n${content}`);
+});
+
+test('large file over the cap, no range → first page (lineCap lines) + a PARTIAL notice', () => {
+  const total = DEFAULT_READ_LINE_CAP + 234; // e.g. 2234
+  const content = nLines(total);
+  const out = formatReadResult({ content, path: 'big.txt' });
+
+  // First page present: line 1 .. line lineCap.
+  assert.match(out, /^File big\.txt:\nL1\n/, 'starts at the top');
+  assert.match(out, new RegExp(`(^|\\n)L${DEFAULT_READ_LINE_CAP}(\\n|$|\\s)`), 'last line of the page present');
+  // Beyond the page is NOT included.
+  assert.doesNotMatch(out, new RegExp(`(^|\\n)L${DEFAULT_READ_LINE_CAP + 1}(\\n|$)`), 'next page not dumped');
+  assert.doesNotMatch(out, new RegExp(`(^|\\n)L${total}(\\n|$)`), 'EOF line not dumped');
+
+  // PARTIAL notice states range, total, and the next start_line.
+  assert.match(out, /PARTIAL/);
+  assert.match(out, new RegExp(`lines 1[–-]${DEFAULT_READ_LINE_CAP} of ${total}`));
+  assert.match(out, new RegExp(`start_line=${DEFAULT_READ_LINE_CAP + 1}`));
+});
+
+test('PAIRED: a file exactly AT the cap reads fully with NO PARTIAL notice', () => {
+  const content = nLines(DEFAULT_READ_LINE_CAP);
+  const out = formatReadResult({ content, path: 'atcap.txt' });
+  assert.strictEqual(out, `File atcap.txt:\n${content}`);
+  assert.doesNotMatch(out, /PARTIAL/);
+});
+
+test('explicit start_line/end_line → exactly that slice', () => {
+  const content = nLines(500);
+  const out = formatReadResult({ content, path: 'r.txt', startLine: 100, endLine: 105 });
+  // Lines 100..105 inclusive, nothing outside.
+  for (const n of [100, 101, 102, 103, 104, 105]) {
+    assert.match(out, new RegExp(`(^|\\n)L${n}(\\n|$)`), `L${n} present`);
+  }
+  assert.doesNotMatch(out, /(^|\n)L99(\n|$)/, 'before-range excluded');
+  assert.doesNotMatch(out, /(^|\n)L106(\n|$)/, 'after-range excluded');
+  // EOF (500) is past the slice → there IS more, so a PARTIAL/continuation hint.
+  assert.match(out, /start_line=106/);
+});
+
+test('a HUGE explicit range is still line-capped (cannot dump everything)', () => {
+  const total = DEFAULT_READ_LINE_CAP * 3;
+  const content = nLines(total);
+  const out = formatReadResult({ content, path: 'huge.txt', startLine: 1, endLine: total });
+  // Only the first lineCap lines come back even though the range asked for all.
+  assert.match(out, new RegExp(`(^|\\n)L${DEFAULT_READ_LINE_CAP}(\\n|$|\\s)`));
+  assert.doesNotMatch(out, new RegExp(`(^|\\n)L${DEFAULT_READ_LINE_CAP + 1}(\\n|$)`));
+  assert.match(out, /PARTIAL/);
+  assert.match(out, new RegExp(`start_line=${DEFAULT_READ_LINE_CAP + 1}`));
+});
+
+test('start_line offset window is line-capped relative to the start', () => {
+  const content = nLines(10000);
+  const out = formatReadResult({ content, path: 'w.txt', startLine: 5000 });
+  const end = 5000 + DEFAULT_READ_LINE_CAP - 1;
+  assert.match(out, new RegExp(`(^|\\n)L5000(\\n|$)`), 'window starts at start_line');
+  assert.match(out, new RegExp(`(^|\\n)L${end}(\\n|$|\\s)`), 'window ends one cap later');
+  assert.doesNotMatch(out, new RegExp(`(^|\\n)L${end + 1}(\\n|$)`));
+  assert.match(out, new RegExp(`start_line=${end + 1}`));
+});
+
+test('show_line_numbers default OFF → no number prefixes; ON → correct, aligned numbers', () => {
+  const content = 'first\nsecond\nthird';
+  const off = formatReadResult({ content, path: 'n.txt' });
+  assert.strictEqual(off, `File n.txt:\n${content}`, 'default-off is verbatim (copyable for replace_in_file)');
+
+  const on = formatReadResult({ content, path: 'n.txt', showLineNumbers: true });
+  // Each line carries its 1-based number; the number aligns with edit_file's
+  // lines[lineNum-1] addressing (line N => lines[N-1]).
+  assert.match(on, /(^|\n)1\tfirst(\n|$)/);
+  assert.match(on, /(^|\n)2\tsecond(\n|$)/);
+  assert.match(on, /(^|\n)3\tthird(\n|$)/);
+});
+
+test('show_line_numbers on a paginated window numbers by ABSOLUTE line', () => {
+  const content = nLines(500);
+  const out = formatReadResult({ content, path: 'a.txt', startLine: 250, endLine: 252, showLineNumbers: true });
+  assert.match(out, /(^|\n)250\tL250(\n|$)/);
+  assert.match(out, /(^|\n)251\tL251(\n|$)/);
+  assert.match(out, /(^|\n)252\tL252(\n|$)/);
+});
+
+test('token net: pathological long lines are token-bounded even within the line cap', () => {
+  // 3 lines, each 60k chars → ~45k tokens at char/4, over the 25k default budget.
+  const content = ['x'.repeat(200000), 'y'.repeat(200000), 'z'.repeat(200000)].join('\n');
+  const out = formatReadResult({ content, path: 'long.txt' });
+  assert.match(out, /token-capped/, 'token safety net fired');
+  // Bounded to ~the token budget in chars (25k tokens * 4 ≈ 100k), far below
+  // the 600k raw chars.
+  assert.ok(out.length < content.length / 4, `token-bounded: ${out.length} << ${content.length}`);
+  assert.ok(out.length < DEFAULT_READ_MAX_TOKENS * 4 + 2000, 'bounded near the token budget');
+});
+
+test('PAIRED: a normal full-size page is NOT token-clipped (pagination shows the whole page)', () => {
+  const content = nLines(DEFAULT_READ_LINE_CAP); // short lines, well under the token net
+  const out = formatReadResult({ content, path: 'normal.txt' });
+  assert.doesNotMatch(out, /token-capped/);
+  assert.strictEqual(out, `File normal.txt:\n${content}`);
+});
+
+test('config-supplied lineCap/maxTokens override the defaults', () => {
+  const content = nLines(40);
+  const out = formatReadResult({ content, path: 'c.txt', lineCap: 10 });
+  assert.match(out, /(^|\n)L10(\n|$|\s)/);
+  assert.doesNotMatch(out, /(^|\n)L11(\n|$)/);
+  assert.match(out, /lines 1[–-]10 of 40/);
+  assert.match(out, /start_line=11/);
+});
+
+test('non-string / empty content handled without throwing', () => {
+  assert.strictEqual(formatReadResult({ content: '', path: 'e.txt' }), 'File e.txt:\n');
+  assert.strictEqual(formatReadResult({ content: null, path: 'e.txt' }), 'File e.txt:\n');
+});
+
+test('start_line past EOF degrades to a clear note, not a crash', () => {
+  const content = nLines(5);
+  const out = formatReadResult({ content, path: 'p.txt', startLine: 99 });
+  assert.match(out, /past end of file|past EOF|beyond/i);
+  assert.doesNotMatch(out, /PARTIAL/);
+});
+
+// ---------------------------------------------------------------------------
+// Part B — both rails: XML + native produce parity tuples with start_line/end_line
+// ---------------------------------------------------------------------------
+
+const { extractToolCalls, mapInvokeToCall } = require('../lib/tools');
+
+test('XML attr rail: start_line/end_line/show_line_numbers parse onto the tuple', () => {
+  const calls = extractToolCalls('<read_file path="a.txt" start_line="10" end_line="20" show_line_numbers="true"/>');
+  assert.deepStrictEqual(calls, [['read', 'a.txt', 10, 20, true]]);
+});
+
+test('XML inline rail: no range → nulls + false, parity preserved', () => {
+  const calls = extractToolCalls('<read_file>a.txt</read_file>');
+  assert.deepStrictEqual(calls, [['read', 'a.txt', null, null, false]]);
+});
+
+test('native rail: fromParams maps start_line/end_line/show_line_numbers identically', () => {
+  assert.deepStrictEqual(
+    mapInvokeToCall('read_file', { path: 'a.txt', start_line: 10, end_line: 20, show_line_numbers: true }),
+    ['read', 'a.txt', 10, 20, true],
+  );
+  // Absent → nulls + false, byte-identical to the XML inline rail.
+  assert.deepStrictEqual(
+    mapInvokeToCall('read_file', { path: 'a.txt' }),
+    ['read', 'a.txt', null, null, false],
+  );
+});
+
+test('RAIL PARITY: XML and native produce the SAME tuple for the same intent', () => {
+  const xml = extractToolCalls('<read_file path="b.txt" start_line="3" end_line="7"/>')[0];
+  const native = mapInvokeToCall('read_file', { path: 'b.txt', start_line: 3, end_line: 7 });
+  assert.deepStrictEqual(xml, native);
+});
+
+// ---------------------------------------------------------------------------
+// Part C — executor + the read→edit loop (Step 0 alignment)
+// ---------------------------------------------------------------------------
+
+const ui = require('../lib/ui');
+const { createToolExecutor } = require('../lib/tools');
+const { createPermissionManager } = require('../lib/permissions');
+
+let TMP, CWD, exec, ef;
+let prevKey;
+before(() => {
+  prevKey = process.env.SEMALT_API_KEY;
+  process.env.SEMALT_API_KEY = 'test-key';
+  TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'w7-read-'));
+  CWD = TMP;
+  const ui2 = ui;
+  const pm = createPermissionManager(ui2, { skipPermissions: true });
+  pm.setUICallbacks({ onAddMessage: () => {}, onShowModal: () => {}, onCloseModal: () => {}, onCaptureNavigation: () => () => {} });
+  exec = createToolExecutor(pm, ui2, () => ({ max_file_size_kb: 512, command_timeout_ms: 30000 }));
+  ef = (action, ...args) => exec.agentExecFile(action, ...args);
+});
+after(() => {
+  if (prevKey === undefined) delete process.env.SEMALT_API_KEY;
+  else process.env.SEMALT_API_KEY = prevKey;
+  try { fs.rmSync(TMP, { recursive: true, force: true }); } catch { /* ignore */ }
+});
+
+test('executor read returns the FULL content (pagination is a context-boundary concern, not a read truncation)', async () => {
+  const big = path.join(TMP, 'big.txt');
+  const content = nLines(DEFAULT_READ_LINE_CAP + 50);
+  fs.writeFileSync(big, content);
+  const r = await ef('read', big);
+  assert.strictEqual(r.error, undefined);
+  assert.strictEqual(r.content, content, 'executor does not truncate — formatReadResult paginates for context');
+});
+
+test('byte backstop still hard-refuses a file over an explicitly small max_file_size_kb', async () => {
+  const pm = createPermissionManager(ui, { skipPermissions: true });
+  const tinyExec = createToolExecutor(pm, ui, () => ({ max_file_size_kb: 1 }));
+  const big = path.join(TMP, 'over.txt');
+  fs.writeFileSync(big, 'x'.repeat(4000));
+  const r = await tinyExec.agentExecFile('read', big);
+  assert.ok(r.error && /too large/i.test(r.error), 'byte backstop remains as a hard ceiling when configured');
+});
+
+test('read→edit alignment: line-based edit_file targets the line show_line_numbers labels', async () => {
+  const file = path.join(TMP, 'edit.txt');
+  fs.writeFileSync(file, 'aaa\nbbb\nccc\nddd');
+  // Read with numbers; the label N must equal edit_file's lines[N-1].
+  const r = await ef('read', file);
+  const numbered = formatReadResult({ content: r.content, path: file, showLineNumbers: true });
+  assert.match(numbered, /(^|\n)3\tccc(\n|$)/, 'line 3 is labeled ccc');
+  // edit_file line 3 → replaces lines[2] (ccc).
+  await ef('edit_file', file, 3, 'CCC');
+  assert.strictEqual(fs.readFileSync(file, 'utf8'), 'aaa\nbbb\nCCC\nddd');
+});
+
+test('read→edit alignment: match-based replace_in_file uses a snippet copied verbatim from default read', async () => {
+  const file = path.join(TMP, 'replace.txt');
+  fs.writeFileSync(file, 'const x = 1;\nconst y = 2;\n');
+  const r = await ef('read', file);
+  const shown = formatReadResult({ content: r.content, path: file }); // default: NO line numbers
+  // The default output carries the raw line verbatim → copyable as a search string.
+  assert.ok(shown.includes('const y = 2;'), 'snippet is present verbatim (no number prefix to corrupt the match)');
+  await ef('replace_in_file', file, 'const y = 2;', 'const y = 42;', '');
+  assert.strictEqual(fs.readFileSync(file, 'utf8'), 'const x = 1;\nconst y = 42;\n');
+});

package/test/readonly-tools.test.js

@@ -0,0 +1,177 @@
+'use strict';
+
+// Pre-Task 5.0c — complete `--readonly` enforcement.
+// The re-audit found --readonly did NOT block edit_file, replace_in_file,
+// make_dir, remove_dir, or upload — so a "read-only" session could still
+// mutate files via those tools. These tests pin the completed contract:
+//   - the five newly-added file-mutating tools are each refused under --readonly
+//     (no mutation reaches disk),
+//   - the previously-blocked set still refused (no regression),
+//   - shell is NOT blocked by --readonly (the documented decision: --readonly
+//     governs FILE TOOLS only; shell side effects are confined by the sandbox /
+//     deny-list, not --readonly),
+//   - a read-only tool still works under --readonly.
+//
+// Home-based paths are redirected into a temp dir BEFORE any lib module loads,
+// matching test/download-confine.test.js.
+
+const os = require('node:os');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-ro-home-'));
+const PREV_HOME = process.env.HOME;
+const PREV_USERPROFILE = process.env.USERPROFILE;
+process.env.HOME = TMP_HOME;
+process.env.USERPROFILE = TMP_HOME;
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+
+const ui = require('../lib/ui');
+const { createPermissionManager } = require('../lib/permissions');
+const { createToolExecutor } = require('../lib/tools');
+
+let CWD;
+let PREV_CWD;
+
+function mkExec({ config = {}, pmOpts = {} } = {}) {
+  const pm = createPermissionManager(ui, pmOpts);
+  return createToolExecutor(pm, ui, () => ({
+    max_file_size_kb: 512,
+    command_timeout_ms: 30000,
+    max_output_lines: 50,
+    ...config,
+  }));
+}
+
+before(() => {
+  PREV_CWD = process.cwd();
+  CWD = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-ro-cwd-'));
+  process.chdir(CWD);
+});
+
+after(() => {
+  process.chdir(PREV_CWD);
+  if (PREV_HOME === undefined) delete process.env.HOME; else process.env.HOME = PREV_HOME;
+  if (PREV_USERPROFILE === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = PREV_USERPROFILE;
+});
+
+// ---------------------------------------------------------------------------
+// readonlyBlock — the full file-mutating set is blocked, reads are not
+// ---------------------------------------------------------------------------
+
+test('readonlyBlock covers every file-mutating tool when --readonly is set', () => {
+  const ro = createPermissionManager(ui, { readonly: true });
+  // newly added (Pre-Task 5.0c)
+  for (const tag of ['edit_file', 'replace_in_file', 'make_dir', 'remove_dir', 'upload']) {
+    assert.deepStrictEqual(ro.readonlyBlock(tag), { error: 'blocked by --readonly' }, `${tag} must be blocked`);
+  }
+  // previously blocked — no regression
+  for (const tag of ['write_file', 'append_file', 'delete_file', 'move_file', 'copy_file', 'download']) {
+    assert.deepStrictEqual(ro.readonlyBlock(tag), { error: 'blocked by --readonly' }, `${tag} must stay blocked`);
+  }
+  // read-class operations are allowed even in readonly mode
+  for (const tag of ['read_file', 'list_dir', 'grep', 'glob', 'search_in_file', 'file_stat']) {
+    assert.strictEqual(ro.readonlyBlock(tag), null, `${tag} must be allowed`);
+  }
+});
+
+test('readonlyBlock does NOT block shell/exec — shell is governed by the sandbox, not --readonly', () => {
+  const ro = createPermissionManager(ui, { readonly: true });
+  assert.strictEqual(ro.readonlyBlock('exec'), null, 'exec is not a file tool; --readonly does not block it');
+  assert.strictEqual(ro.readonlyBlock('shell'), null, 'shell is not a file tool; --readonly does not block it');
+});
+
+// ---------------------------------------------------------------------------
+// executor-level — each newly-added tool is refused and leaves no mutation
+// ---------------------------------------------------------------------------
+
+test('edit_file is blocked under --readonly (no mutation)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const f = path.join(CWD, 'edit.txt');
+  fs.writeFileSync(f, 'original\n');
+  const r = await exec.agentExecFile('edit_file', f, 1, 'changed');
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.readFileSync(f, 'utf8'), 'original\n', 'file must be untouched');
+});
+
+test('replace_in_file is blocked under --readonly (no mutation)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const f = path.join(CWD, 'replace.txt');
+  fs.writeFileSync(f, 'foo bar\n');
+  const r = await exec.agentExecFile('replace_in_file', f, 'foo', 'baz', '');
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.readFileSync(f, 'utf8'), 'foo bar\n', 'file must be untouched');
+});
+
+test('make_dir is blocked under --readonly (no mutation)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const d = path.join(CWD, 'newdir');
+  const r = await exec.agentExecFile('make_dir', d);
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.existsSync(d), false, 'directory must not be created');
+});
+
+test('remove_dir is blocked under --readonly (no mutation)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const d = path.join(CWD, 'keepdir');
+  fs.mkdirSync(d, { recursive: true });
+  const r = await exec.agentExecFile('remove_dir', d);
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.existsSync(d), true, 'directory must still exist');
+});
+
+test('upload is blocked under --readonly (no mutation)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const f = path.join(CWD, 'uploaded.bin');
+  const r = await exec.agentExecFile('upload', f, Buffer.from('hi').toString('base64'));
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.existsSync(f), false, 'file must not be written');
+});
+
+// ---------------------------------------------------------------------------
+// no regression — the previously-blocked set still refused at the executor
+// ---------------------------------------------------------------------------
+
+test('write_file is still blocked under --readonly (no regression)', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const f = path.join(CWD, 'write.txt');
+  const r = await exec.agentExecFile('write', f, 'data');
+  assert.ok(r.error, 'should be blocked');
+  assert.match(r.error, /readonly/i);
+  assert.strictEqual(fs.existsSync(f), false, 'file must not be written');
+});
+
+// ---------------------------------------------------------------------------
+// describePermission short-circuits the newly-added tools under --readonly
+// (parity with delete/move/copy/download) so no approval prompt precedes the
+// deterministic block.
+// ---------------------------------------------------------------------------
+
+test('describePermission returns null for the newly-added tools under --readonly', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  assert.strictEqual(await exec.describePermission(['edit_file', '/x', 1, 'y']), null);
+  assert.strictEqual(await exec.describePermission(['replace_in_file', '/x', 'a', 'b', '']), null);
+  assert.strictEqual(await exec.describePermission(['make_dir', '/x']), null);
+  assert.strictEqual(await exec.describePermission(['remove_dir', '/x']), null);
+  assert.strictEqual(await exec.describePermission(['upload', '/x', 'aGk=']), null);
+});
+
+// ---------------------------------------------------------------------------
+// a read-only tool still works under --readonly
+// ---------------------------------------------------------------------------
+
+test('read_file still works under --readonly', async () => {
+  const exec = mkExec({ pmOpts: { readonly: true } });
+  const f = path.join(CWD, 'readme.txt');
+  fs.writeFileSync(f, 'hello\n');
+  const r = await exec.agentExecFile('read', f);
+  assert.ok(!r.error, 'read must not be blocked');
+  assert.match(r.content || '', /hello/);
+});