@semalt-ai/code 1.8.5 → 1.20.0

package/test/hooks-agent.test.js

@@ -0,0 +1,238 @@
+'use strict';
+
+// Integration tests for lifecycle hooks (Task 3.4) driving the REAL runAgentLoop
+// against the mock-LLM harness, with the REAL createHookRunner reading
+// config.hooks (so spawnSync actually runs the hook commands). Hook commands use
+// `node -e …` so they are portable across the CI matrix (Linux/macOS/Windows).
+// Sentinel paths are passed via env vars (merged into the hook's environment) to
+// avoid embedding OS-specific path separators in a `node -e` string literal.
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const ui = require('../lib/ui');
+const { createApiClient } = require('../lib/api');
+const { createToolExecutor, extractToolCalls } = require('../lib/tools');
+const { createPermissionManager } = require('../lib/permissions');
+const { createAgentRunner } = require('../lib/agent');
+const { startMockLLM } = require('./harness/mock-llm');
+
+let prevKey;
+before(() => { prevKey = process.env.SEMALT_API_KEY; process.env.SEMALT_API_KEY = 'test-key'; });
+after(() => {
+  if (prevKey === undefined) delete process.env.SEMALT_API_KEY;
+  else process.env.SEMALT_API_KEY = prevKey;
+});
+
+const NODE = JSON.stringify(process.execPath);
+
+// buildRunner mirrors agent-loop.test.js, but threads `hooks` into the config so
+// the real hook runner (built inside createAgentRunner from getConfig) sees them.
+function buildRunner(base, hooks) {
+  const config = {
+    api_base: base, api_key: 'test-key', default_model: 'test-model',
+    temperature: 0.5, request_timeout_ms: 5000, stream: true, models: [],
+    hooks: hooks || {},
+    // This suite tests hook ORCHESTRATION, not the OS sandbox (covered by
+    // hooks-verify-sandbox.test.js). Disable the sandbox so the command hooks
+    // run deterministically across the CI matrix regardless of bwrap/Seatbelt.
+    sandbox: { mode: 'off' },
+  };
+  const getConfig = () => config;
+  const saveConfig = (c) => Object.assign(config, c);
+  const api = createApiClient({ getConfig, saveConfig, ui });
+  const pm = createPermissionManager(ui, { skipPermissions: true });
+  pm.setUICallbacks({ onAddMessage: () => {}, onShowModal: () => {}, onCloseModal: () => {}, onCaptureNavigation: () => () => {} });
+  const { agentExecShell, agentExecFile, describePermission } = createToolExecutor(pm, ui, getConfig);
+  const runner = createAgentRunner({
+    chatStream: api.chatStream, extractToolCalls, agentExecShell, agentExecFile,
+    describePermission, permissionManager: pm, ui, getConfig,
+  });
+  return { runner, config };
+}
+
+function collector() {
+  const ev = { tools: [], errors: [], assistants: [] };
+  const cb = {
+    onToolEnd: (tag, result) => ev.tools.push({ tag, result }),
+    onError: (e) => ev.errors.push(e),
+    onAssistantMessage: (m) => ev.assistants.push(m),
+  };
+  return { ev, cb };
+}
+
+function tmpdir() { return fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-hooks-')); }
+
+// ---------------------------------------------------------------------------
+// 1. PreToolUse hook blocks a tool (non-zero exit) — the tool never runs
+// ---------------------------------------------------------------------------
+
+test('PreToolUse hook with a non-zero exit BLOCKS the tool; it does not run and the agent gets the reason', async () => {
+  const dir = tmpdir();
+  const sentinel = path.join(dir, 'written.txt');
+  const hooks = { PreToolUse: [{
+    type: 'command', matcher: '*',
+    command: `${NODE} -e "process.stdout.write('blocked: writes are frozen'); process.exit(1)"`,
+  }] };
+  const mock = await startMockLLM();
+  // The model tries to write a file; the hook must stop it.
+  mock.replyWith(`<write_file path="${sentinel}">DATA</write_file>`);
+  mock.replyWith('Understood, I will not write.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { ev, cb } = collector();
+    const messages = [{ role: 'user', content: 'write the file' }];
+    await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+
+    assert.ok(!fs.existsSync(sentinel), 'the write tool was blocked — no file on disk');
+    assert.strictEqual(ev.tools.length, 0, 'a blocked tool never reaches onToolEnd');
+    const fed = messages.find((m) => m.role === 'user' && /Tool execution results/.test(m.content));
+    assert.ok(fed, 'the blocked-tool result is fed back to the model');
+    assert.match(fed.content, /BLOCKED by a PreToolUse hook/);
+    assert.match(fed.content, /writes are frozen/, 'hook stdout is the block reason');
+    assert.ok(messages.some((m) => m.role === 'assistant' && m.content === 'Understood, I will not write.'));
+  } finally {
+    await mock.close();
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 2. PostToolUse hook observes a result and injects feedback
+// ---------------------------------------------------------------------------
+
+test('PostToolUse hook observes the tool result and its stdout is appended (untrusted-fenced)', async () => {
+  const dir = tmpdir();
+  const file = path.join(dir, 'fixture.txt');
+  fs.writeFileSync(file, 'FILE_BODY_7', 'utf8');
+  const hooks = { PostToolUse: [{
+    type: 'command',
+    // Echo back the tool name the hook saw via env — proof it observed the call.
+    command: `${NODE} -e "process.stdout.write('POSTHOOK_SAW=' + process.env.SEMALT_TOOL_NAME)"`,
+  }] };
+  const mock = await startMockLLM();
+  mock.replyWith(`<read_file>${file}</read_file>`);
+  mock.replyWith('Read it.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { ev, cb } = collector();
+    const messages = [{ role: 'user', content: 'read the file' }];
+    await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+
+    assert.strictEqual(ev.tools.length, 1);
+    assert.strictEqual(ev.tools[0].tag, 'read');
+    const fed = messages.find((m) => m.role === 'user' && /Tool execution results/.test(m.content));
+    assert.ok(fed);
+    assert.match(fed.content, /FILE_BODY_7/, 'the real tool result is present');
+    assert.match(fed.content, /POSTHOOK_SAW=read/, 'PostToolUse stdout was appended');
+    assert.match(fed.content, /UNTRUSTED_EXTERNAL_CONTENT/, 'hook feedback is fenced as untrusted');
+  } finally {
+    await mock.close();
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 3. UserPromptSubmit hook injects context before the loop
+// ---------------------------------------------------------------------------
+
+test('UserPromptSubmit hook stdout is injected into the conversation as context', async () => {
+  const hooks = { UserPromptSubmit: [{
+    type: 'command',
+    command: `${NODE} -e "process.stdout.write('INJECTED_CONTEXT_42')"`,
+  }] };
+  const mock = await startMockLLM();
+  mock.replyWith('Acknowledged.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { cb } = collector();
+    const messages = [{ role: 'user', content: 'hello' }];
+    await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+
+    const injected = messages.find((m) => m.role === 'user' && /INJECTED_CONTEXT_42/.test(m.content));
+    assert.ok(injected, 'hook output was injected as a user message');
+    assert.match(injected.content, /UNTRUSTED_EXTERNAL_CONTENT/, 'injected context is fenced as untrusted');
+  } finally {
+    await mock.close();
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 4. A failing hook does not crash the loop
+// ---------------------------------------------------------------------------
+
+test('a failing (non-zero, no-output) PostToolUse hook is contained — the loop completes normally', async () => {
+  const hooks = { PostToolUse: [{ type: 'command', command: `${NODE} -e "process.exit(3)"` }] };
+  const mock = await startMockLLM();
+  mock.replyWith('<exec>echo hi</exec>');
+  mock.replyWith('All good.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { ev, cb } = collector();
+    const messages = [{ role: 'user', content: 'run it' }];
+    const { metrics } = await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+
+    assert.strictEqual(ev.tools.length, 1, 'the tool still executed');
+    assert.strictEqual(metrics.turns.length, 2, 'tool turn + final turn — the loop did not crash');
+    assert.ok(messages.some((m) => m.role === 'assistant' && m.content === 'All good.'));
+  } finally {
+    await mock.close();
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 5. A deny-listed hook command is never executed
+// ---------------------------------------------------------------------------
+
+test('a deny-listed PreToolUse hook command is skipped (never run) and does not block the tool', async () => {
+  const hooks = { PreToolUse: [{ type: 'command', matcher: '*', command: 'rm -rf /' }] };
+  const mock = await startMockLLM();
+  mock.replyWith('<exec>echo ALLOWED</exec>');
+  mock.replyWith('Done.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { ev, cb } = collector();
+    const messages = [{ role: 'user', content: 'go' }];
+    await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+
+    // The deny-listed hook is skipped, so the tool is NOT blocked and runs.
+    assert.strictEqual(ev.tools.length, 1, 'tool ran — a denied hook does not block');
+    const fed = messages.find((m) => m.role === 'user' && /Tool execution results/.test(m.content));
+    assert.match(fed.content, /ALLOWED/);
+    assert.ok(!fed.content.includes('BLOCKED by a PreToolUse hook'), 'no spurious block');
+  } finally {
+    await mock.close();
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 6. Stop hook fires once when the turn ends
+// ---------------------------------------------------------------------------
+
+test('Stop hook fires when the agent loop finishes the turn', async () => {
+  const dir = tmpdir();
+  const sentinel = path.join(dir, 'stopped.txt');
+  const prev = process.env.SEMALT_TEST_STOP_FILE;
+  process.env.SEMALT_TEST_STOP_FILE = sentinel;
+  const hooks = { Stop: [{
+    type: 'command',
+    command: `${NODE} -e "require('fs').writeFileSync(process.env.SEMALT_TEST_STOP_FILE,'x')"`,
+  }] };
+  const mock = await startMockLLM();
+  mock.replyWith('Final answer, no tools.');
+  try {
+    const { runner } = buildRunner(mock.base, hooks);
+    const { cb } = collector();
+    const messages = [{ role: 'user', content: 'just answer' }];
+    await runner.runAgentLoop(messages, 'test-model', 5, null, { callbacks: cb });
+    assert.ok(fs.existsSync(sentinel), 'the Stop hook ran at end of turn');
+  } finally {
+    await mock.close();
+    if (prev === undefined) delete process.env.SEMALT_TEST_STOP_FILE;
+    else process.env.SEMALT_TEST_STOP_FILE = prev;
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});

package/test/hooks-verify-sandbox.test.js

@@ -0,0 +1,232 @@
+'use strict';
+
+// Pre-Task 5.0a — verify + command-type hooks must run through the SAME OS
+// sandbox as agentExecShell. Two layers of coverage:
+//
+//   1. Fallback rules (deterministic, no real bwrap needed): the shared detection
+//      cache is primed to "unavailable" so we assert the fail-safe path —
+//      failIfUnavailable hard error / no-approver refuse / approver-yes run —
+//      identically to test/sandbox-agent.test.js but for verify/hooks.
+//   2. Kernel-level enforcement (REAL bwrap/sandbox-exec): a verify command and a
+//      command hook that write OUTSIDE the working dir are blocked by the OS.
+//      These SKIP gracefully when the primitive is absent on the runner.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+const fs = require('fs');
+const path = require('path');
+
+const { createVerifyRunner } = require('../lib/verify');
+const { createHookRunner } = require('../lib/hooks');
+const { detectSandbox, _resetSandboxDetection } = require('../lib/sandbox');
+
+const NODE = JSON.stringify(process.execPath);
+
+// Force the shared detection cache to "unavailable" so the fallback tests are
+// deterministic on ANY runner (mirrors test/sandbox-agent.test.js).
+function primeUnavailable() {
+  _resetSandboxDetection();
+  detectSandbox({ platform: 'linux', which: () => null, readFile: () => 'Linux version 6.0', force: true });
+}
+
+// ---------------------------------------------------------------------------
+// 1. Fallback rules — verify
+// ---------------------------------------------------------------------------
+
+test('verify: sandbox unavailable + auto + NO approver → REFUSED (never a silent unsandboxed run)', async () => {
+  primeUnavailable();
+  const runner = createVerifyRunner({
+    getConfig: () => ({ verify: { command: `${NODE} -e "process.exit(0)"` }, sandbox: { mode: 'auto' } }),
+  });
+  const res = await runner.run();
+  assert.strictEqual(res.ran, false, 'the command was never executed');
+  assert.strictEqual(res.passed, false, 'a refused verify cannot pass');
+  assert.match(res.output, /refused to run unsandboxed/i);
+  assert.match(res.fenced, /UNTRUSTED_EXTERNAL_CONTENT/);
+  _resetSandboxDetection();
+});
+
+test('verify: sandbox unavailable + failIfUnavailable → hard error (non-passing, names the gate)', async () => {
+  primeUnavailable();
+  const runner = createVerifyRunner({
+    getConfig: () => ({ verify: { command: `${NODE} -e "process.exit(0)"` }, sandbox: { mode: 'auto', failIfUnavailable: true } }),
+  });
+  const res = await runner.run();
+  assert.strictEqual(res.ran, false);
+  assert.strictEqual(res.passed, false);
+  assert.match(res.output, /failIfUnavailable/);
+  _resetSandboxDetection();
+});
+
+test('verify: sandbox unavailable + human approver says YES → runs unsandboxed and can pass', async () => {
+  primeUnavailable();
+  let asked = null;
+  const runner = createVerifyRunner({
+    getConfig: () => ({ verify: { command: `${NODE} -e "process.exit(0)"` }, sandbox: { mode: 'auto' } }),
+    onUnsandboxed: async (info) => { asked = info; return true; },
+  });
+  const res = await runner.run();
+  assert.ok(asked && /bwrap|bubblewrap|not found/i.test(asked.reason), 'approver receives the reason');
+  assert.strictEqual(res.ran, true);
+  assert.strictEqual(res.passed, true, 'exit 0 passes when the human approved an unsandboxed run');
+  _resetSandboxDetection();
+});
+
+test('verify: deny-list still fires BEFORE the sandbox layer (defense in depth)', async () => {
+  primeUnavailable(); // would refuse anyway — but the deny-list must win first
+  let sandboxCalls = 0;
+  const runner = createVerifyRunner({
+    getConfig: () => ({ verify: { command: 'rm -rf /' }, sandbox: { mode: 'auto' } }),
+    sandbox: () => { sandboxCalls++; return { run: true, useShell: true, file: 'x', args: [], sandbox: 'off' }; },
+  });
+  const res = await runner.run();
+  assert.ok(res.denied, 'deny-list label recorded');
+  assert.strictEqual(res.ran, false);
+  assert.strictEqual(sandboxCalls, 0, 'a deny-listed command never reaches the sandbox resolver');
+  _resetSandboxDetection();
+});
+
+// ---------------------------------------------------------------------------
+// 1. Fallback rules — command hooks
+// ---------------------------------------------------------------------------
+
+test('hook (command): sandbox unavailable + auto + NO approver → NOT run, contained (does not block)', async () => {
+  primeUnavailable();
+  const logs = [];
+  const runner = createHookRunner({
+    getConfig: () => ({ hooks: { PreToolUse: [{ type: 'command', command: `${NODE} -e "process.exit(1)"` }] }, sandbox: { mode: 'auto' } }),
+    log: (m) => logs.push(m),
+  });
+  const r = await runner.run('PreToolUse', { tool: 'shell' });
+  assert.strictEqual(r.blocked, false, 'a refused hook does not block the tool (contained like a timeout)');
+  assert.strictEqual(r.ran[0].ok, false);
+  assert.match(r.ran[0].error, /refused to run unsandboxed/i);
+  assert.ok(logs.some((l) => /not run/i.test(l)));
+  _resetSandboxDetection();
+});
+
+test('hook (command): deny-listed command never reaches the sandbox resolver', async () => {
+  primeUnavailable();
+  let sandboxCalls = 0;
+  const runner = createHookRunner({
+    getConfig: () => ({ hooks: { PreToolUse: [{ type: 'command', command: 'rm -rf /' }] }, sandbox: { mode: 'auto' } }),
+    sandbox: () => { sandboxCalls++; return { run: true, useShell: true, file: 'x', args: [], sandbox: 'off' }; },
+  });
+  const r = await runner.run('PreToolUse', { tool: 'shell' });
+  assert.strictEqual(sandboxCalls, 0, 'deny-list short-circuits before the sandbox');
+  assert.ok(r.ran[0].denied);
+  _resetSandboxDetection();
+});
+
+test('hook (prompt): unaffected by the sandbox (no shell, just injects text)', async () => {
+  primeUnavailable(); // even with an unavailable sandbox, a prompt hook still injects
+  const runner = createHookRunner({
+    getConfig: () => ({ hooks: { UserPromptSubmit: [{ type: 'prompt', prompt: 'Mind the style guide.' }] }, sandbox: { mode: 'auto' } }),
+  });
+  const r = await runner.run('UserPromptSubmit', { prompt: 'go' });
+  assert.strictEqual(r.feedback.length, 1, 'prompt hook injects regardless of sandbox availability');
+  assert.match(r.feedback[0], /Mind the style guide/);
+  _resetSandboxDetection();
+});
+
+// ---------------------------------------------------------------------------
+// 2. Kernel-level enforcement (REAL bwrap / sandbox-exec) — skip gracefully
+// ---------------------------------------------------------------------------
+
+function realDetect() {
+  _resetSandboxDetection();
+  return detectSandbox({ force: true });
+}
+const det = realDetect();
+const SKIP = det.available ? false : `OS sandbox tool unavailable on this runner (${det.reason || det.platform})`;
+
+// A target OUTSIDE the working dir AND outside the OS temp dir (both are writable
+// roots in the real wrap). The repo parent satisfies both — the sandbox must
+// block a write there. Cleaned up regardless of outcome.
+function outsideTarget(tag) {
+  return path.join(path.dirname(process.cwd()), `semalt-sbx-${tag}-${process.pid}.txt`);
+}
+
+// Quote-free shell redirects keep the command robust under the sh -c wrapper, so
+// a non-zero exit is unambiguously the SANDBOX denying the write — not a parsing
+// artifact. The target paths contain no spaces/quotes.
+test('verify (REAL jail): a command writing OUTSIDE the working dir is blocked by the kernel', { skip: SKIP }, async () => {
+  realDetect();
+  const escape = outsideTarget('verify-escape');
+  try { fs.unlinkSync(escape); } catch {}
+  const runner = createVerifyRunner({ getConfig: () => ({ verify: { command: `echo pwned > ${escape}` }, sandbox: { mode: 'auto' } }) });
+  try {
+    const res = await runner.run();
+    assert.strictEqual(res.passed, false, 'the out-of-CWD write must fail under the jail');
+    assert.ok(!fs.existsSync(escape), 'the out-of-jail file must NOT have been created');
+  } finally {
+    try { fs.unlinkSync(escape); } catch {}
+    _resetSandboxDetection();
+  }
+});
+
+test('hook (REAL jail): a command hook writing OUTSIDE the working dir is blocked by the kernel', { skip: SKIP }, async () => {
+  realDetect();
+  const escape = outsideTarget('hook-escape');
+  try { fs.unlinkSync(escape); } catch {}
+  const runner = createHookRunner({ getConfig: () => ({ hooks: { PostToolUse: [{ type: 'command', command: `echo pwned > ${escape}` }] }, sandbox: { mode: 'auto' } }) });
+  try {
+    const r = await runner.run('PostToolUse', { tool: 'shell', result: 'x' });
+    assert.strictEqual(r.ran[0].ok, false, 'the hook command must fail under the jail (out-of-CWD write blocked)');
+    assert.ok(!fs.existsSync(escape), 'the out-of-jail file must NOT have been created');
+  } finally {
+    try { fs.unlinkSync(escape); } catch {}
+    _resetSandboxDetection();
+  }
+});
+
+test('verify (REAL jail): a command writing to the OS temp dir (a writable root) still passes', { skip: SKIP }, async () => {
+  realDetect();
+  const os = require('os');
+  const inside = path.join(os.tmpdir(), `semalt-sbx-verify-ok-${process.pid}.txt`);
+  try { fs.unlinkSync(inside); } catch {}
+  const runner = createVerifyRunner({ getConfig: () => ({ verify: { command: `echo ok > ${inside}` }, sandbox: { mode: 'auto' } }) });
+  try {
+    const res = await runner.run();
+    assert.strictEqual(res.passed, true, 'a write to a writable root succeeds under the jail');
+    assert.ok(fs.existsSync(inside), 'the in-jail file was written');
+  } finally {
+    try { fs.unlinkSync(inside); } catch {}
+    _resetSandboxDetection();
+  }
+});
+
+// ---------------------------------------------------------------------------
+// 3. Binary network isolation reaches verify + hooks too (Task 4.4b).
+// ---------------------------------------------------------------------------
+//
+// sandbox.network: 'off' must apply through the SAME shared shim to verify and
+// command hooks — not just the agent's shell tool. Gated to bwrap, where the
+// no-network jail is observable by counting non-loopback interfaces. A node
+// one-liner exits 0 when it sees NO network (only loopback), non-zero otherwise.
+const NET_SKIP = (det.available && det.tool === 'bwrap') ? false
+  : `network-isolation kernel test needs bwrap (got ${det.tool || det.platform})`;
+const NONET_PROBE = `${NODE} -e 'const i=require("os").networkInterfaces();const n=Object.keys(i).filter(x=>x!=="lo"&&x!=="lo0");process.exit(n.length?5:0)'`;
+
+test('verify (REAL jail): sandbox.network off runs the verify command with NO network', { skip: NET_SKIP }, async () => {
+  realDetect();
+  // expected_exit_code 0 == "the probe saw no network" ⇒ passing proves the verify
+  // shell ran kernel-isolated from the network.
+  const runner = createVerifyRunner({ getConfig: () => ({ verify: { command: NONET_PROBE }, sandbox: { mode: 'auto', network: 'off' } }) });
+  try {
+    const res = await runner.run();
+    assert.strictEqual(res.ran, true);
+    assert.strictEqual(res.passed, true, 'the verify command observed no network (exit 0) under the no-network jail');
+    assert.strictEqual(res.exitCode, 0);
+  } finally { _resetSandboxDetection(); }
+});
+
+test('hook (REAL jail): sandbox.network off runs the command hook with NO network', { skip: NET_SKIP }, async () => {
+  realDetect();
+  const runner = createHookRunner({ getConfig: () => ({ hooks: { PostToolUse: [{ type: 'command', command: NONET_PROBE }] }, sandbox: { mode: 'auto', network: 'off' } }) });
+  try {
+    const r = await runner.run('PostToolUse', { tool: 'shell', result: 'x' });
+    assert.strictEqual(r.ran[0].ok, true, 'the hook command observed no network (exit 0) under the no-network jail');
+    assert.strictEqual(r.ran[0].exitCode, 0, 'no non-loopback interface inside the no-network jail');
+  } finally { _resetSandboxDetection(); }
+});