@vellumai/assistant 0.3.0

package/src/__tests__/shell-tool-proxy-mode.test.ts

@@ -0,0 +1,272 @@
+import { afterEach, describe, expect, mock, test } from 'bun:test';
+import * as realChildProcess from 'node:child_process';
+
+const originalSpawn = realChildProcess.spawn;
+
+// Track spawn calls to inspect env vars passed to child processes
+const spawnCalls: { command: string; args: string[]; env: Record<string, string> | undefined }[] = [];
+const spawnSpy = mock((...args: Parameters<typeof realChildProcess.spawn>) => {
+  const opts = args[2] as { env?: Record<string, string> } | undefined;
+  spawnCalls.push({
+    command: args[0] as string,
+    args: args[1] as string[],
+    env: opts?.env ? { ...opts.env } : undefined,
+  });
+  return (originalSpawn as (...a: unknown[]) => unknown)(...args);
+});
+
+mock.module('node:child_process', () => ({
+  ...realChildProcess,
+  spawn: spawnSpy,
+}));
+
+const mockConfig = {
+  provider: 'anthropic',
+  model: 'test',
+  apiKeys: {},
+  maxTokens: 4096,
+  dataDir: '/tmp',
+  timeouts: {
+    shellDefaultTimeoutSec: 120,
+    shellMaxTimeoutSec: 600,
+    permissionTimeoutSec: 300,
+  },
+  sandbox: { enabled: false },
+  rateLimit: { maxRequestsPerMinute: 0, maxTokensPerSession: 0 },
+  secretDetection: { enabled: true, action: 'warn' as const, entropyThreshold: 4.0 },
+  auditLog: { retentionDays: 0 },
+};
+
+mock.module('../config/loader.js', () => ({
+  getConfig: () => mockConfig,
+  loadConfig: () => mockConfig,
+  invalidateConfigCache: () => {},
+  saveConfig: () => {},
+  loadRawConfig: () => ({}),
+  saveRawConfig: () => {},
+  getNestedValue: () => undefined,
+  setNestedValue: () => {},
+}));
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () => new Proxy({} as Record<string, unknown>, {
+    get: () => () => {},
+  }),
+}));
+
+mock.module('../security/secret-scanner.js', () => ({
+  redactSecrets: (s: string) => s,
+}));
+
+// Track wrapCommand calls to verify networkMode is forwarded
+let wrapCommandCalls: { cmd: string; workingDir: string; config: unknown; options: unknown }[] = [];
+
+mock.module('../tools/terminal/sandbox.js', () => ({
+  wrapCommand: (cmd: string, workingDir: string, config: unknown, options?: unknown) => {
+    wrapCommandCalls.push({ cmd, workingDir, config, options });
+    return { command: '/bin/sh', args: ['-c', cmd], sandboxed: false };
+  },
+}));
+
+mock.module('../util/platform.js', () => ({
+  getDataDir: () => '/tmp/test-data',
+}));
+
+// --- Proxy session mocks ---
+let mockActiveSession: { id: string; conversationId: string } | undefined;
+let getOrStartSessionCalls: { conversationId: string; credentialIds: string[] }[] = [];
+let getSessionEnvCalls: string[] = [];
+
+const MOCK_SESSION_ID = 'mock-proxy-session-id';
+const MOCK_PROXY_PORT = 9876;
+
+mock.module('../tools/network/script-proxy/index.js', () => ({
+  getOrStartSession: async (conversationId: string, credentialIds: string[]) => {
+    getOrStartSessionCalls.push({ conversationId, credentialIds });
+    if (mockActiveSession && mockActiveSession.conversationId === conversationId) {
+      return { session: { id: mockActiveSession.id, conversationId, credentialIds, status: 'active', createdAt: new Date(), port: MOCK_PROXY_PORT }, created: false };
+    }
+    return { session: { id: MOCK_SESSION_ID, conversationId, credentialIds, status: 'active', createdAt: new Date(), port: MOCK_PROXY_PORT }, created: true };
+  },
+  getActiveSession: (conversationId: string) => {
+    if (mockActiveSession && mockActiveSession.conversationId === conversationId) {
+      return mockActiveSession;
+    }
+    return undefined;
+  },
+  getSessionEnv: (sessionId: string) => {
+    getSessionEnvCalls.push(sessionId);
+    return {
+      HTTP_PROXY: `http://127.0.0.1:${MOCK_PROXY_PORT}`,
+      HTTPS_PROXY: `http://127.0.0.1:${MOCK_PROXY_PORT}`,
+      NO_PROXY: 'localhost,127.0.0.1,::1',
+      NODE_EXTRA_CA_CERTS: '/tmp/test-data/proxy-ca/ca.pem',
+    };
+  },
+  ensureLocalCA: async () => {},
+  issueLeafCert: async () => ({ cert: '', key: '' }),
+  getCAPath: () => '/tmp/test-data/proxy-ca/ca.pem',
+  getSessionsForConversation: () => [],
+  stopAllSessions: async () => {},
+}));
+
+mock.module('../tools/credentials/resolve.js', () => ({
+  resolveCredentialRef: (ref: string) => ({ credentialId: ref }),
+}));
+
+mock.module('../tools/network/script-proxy/logging.js', () => ({
+  buildCredentialRefTrace: (rawRefs: string[], resolvedIds: string[], unresolvedRefs: string[]) => ({ rawRefs, resolvedIds, unresolvedRefs }),
+}));
+
+import { shellTool } from '../tools/terminal/shell.js';
+import type { ToolContext } from '../tools/types.js';
+
+function makeContext(overrides?: Partial<ToolContext>): ToolContext {
+  return {
+    workingDir: '/tmp',
+    sessionId: 'test-session',
+    conversationId: 'test-conv',
+    ...overrides,
+  };
+}
+
+afterEach(() => {
+  spawnCalls.length = 0;
+  wrapCommandCalls = [];
+  getOrStartSessionCalls = [];
+  getSessionEnvCalls = [];
+  mockActiveSession = undefined;
+});
+
+describe('shell tool proxy mode', () => {
+  test('default mode does not inject proxy env vars', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo hello' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(getOrStartSessionCalls).toHaveLength(0);
+
+    const lastCall = spawnCalls[spawnCalls.length - 1];
+    expect(lastCall.env?.HTTP_PROXY).toBeUndefined();
+    expect(lastCall.env?.HTTPS_PROXY).toBeUndefined();
+  });
+
+  test('network_mode=off does not inject proxy env vars', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo hello', network_mode: 'off' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(getOrStartSessionCalls).toHaveLength(0);
+
+    const lastCall = spawnCalls[spawnCalls.length - 1];
+    expect(lastCall.env?.HTTP_PROXY).toBeUndefined();
+  });
+
+  test('network_mode=proxied creates session and injects proxy env', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo proxied', network_mode: 'proxied', credential_ids: ['cred-1'] },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+
+    // Session acquired via getOrStartSession
+    expect(getOrStartSessionCalls).toHaveLength(1);
+    expect(getOrStartSessionCalls[0].conversationId).toBe('test-conv');
+    expect(getOrStartSessionCalls[0].credentialIds).toEqual(['cred-1']);
+
+    // Env injection
+    const lastCall = spawnCalls[spawnCalls.length - 1];
+    expect(lastCall.env?.HTTP_PROXY).toBe(`http://127.0.0.1:${MOCK_PROXY_PORT}`);
+    expect(lastCall.env?.HTTPS_PROXY).toBe(`http://127.0.0.1:${MOCK_PROXY_PORT}`);
+    expect(lastCall.env?.NO_PROXY).toBe('localhost,127.0.0.1,::1');
+    expect(lastCall.env?.NODE_EXTRA_CA_CERTS).toBe('/tmp/test-data/proxy-ca/ca.pem');
+
+    // Session is NOT stopped after command — idle timer handles cleanup
+    expect(getSessionEnvCalls).toHaveLength(1);
+    expect(getSessionEnvCalls[0]).toBe(MOCK_SESSION_ID);
+  });
+
+  test('proxied mode reuses existing active session', async () => {
+    mockActiveSession = { id: 'existing-session-id', conversationId: 'test-conv' };
+
+    const result = await shellTool.execute(
+      { command: 'echo reuse', network_mode: 'proxied' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+
+    // getOrStartSession is still called — it internally returns the existing session
+    expect(getOrStartSessionCalls).toHaveLength(1);
+
+    // Should get env from existing session
+    expect(getSessionEnvCalls).toHaveLength(1);
+    expect(getSessionEnvCalls[0]).toBe('existing-session-id');
+
+    // Should still inject proxy env
+    const lastCall = spawnCalls[spawnCalls.length - 1];
+    expect(lastCall.env?.HTTP_PROXY).toBeDefined();
+  });
+
+  test('safe env vars are preserved alongside proxy vars', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo env-merge', network_mode: 'proxied', credential_ids: ['cred-x'] },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+
+    const lastCall = spawnCalls[spawnCalls.length - 1];
+    // Safe env vars should still be present
+    expect(lastCall.env?.PATH).toBeDefined();
+    expect(lastCall.env?.HOME).toBeDefined();
+    // Proxy vars should also be present
+    expect(lastCall.env?.HTTP_PROXY).toBeDefined();
+    expect(lastCall.env?.HTTPS_PROXY).toBeDefined();
+  });
+
+  test('schema includes network_mode and credential_ids', () => {
+    const def = shellTool.getDefinition();
+    const props = (def.input_schema as { properties: Record<string, unknown> }).properties;
+    expect(props.network_mode).toBeDefined();
+    expect(props.credential_ids).toBeDefined();
+  });
+
+  test('wrapCommand receives { networkMode: "proxied" } when network_mode=proxied', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo proxied-wrap', network_mode: 'proxied' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(wrapCommandCalls).toHaveLength(1);
+    expect(wrapCommandCalls[0].options).toEqual({ networkMode: 'proxied' });
+  });
+
+  test('wrapCommand receives { networkMode: "off" } when network_mode is absent', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo default-wrap' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(wrapCommandCalls).toHaveLength(1);
+    expect(wrapCommandCalls[0].options).toEqual({ networkMode: 'off' });
+  });
+
+  test('wrapCommand receives { networkMode: "off" } when network_mode=off', async () => {
+    const result = await shellTool.execute(
+      { command: 'echo off-wrap', network_mode: 'off' },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(wrapCommandCalls).toHaveLength(1);
+    expect(wrapCommandCalls[0].options).toEqual({ networkMode: 'off' });
+  });
+});

package/src/__tests__/signup-e2e.test.ts

@@ -0,0 +1,353 @@
+import { describe, test, expect, beforeAll, afterAll, beforeEach, mock } from 'bun:test';
+import { mkdtempSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+// ── Mocks (before any app imports) ──────────────────────────────────
+
+const testDir = mkdtempSync(join(tmpdir(), 'signup-e2e-'));
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module('../util/platform.js', () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === 'darwin',
+  isLinux: () => process.platform === 'linux',
+  isWindows: () => process.platform === 'win32',
+  getSocketPath: () => join(testDir, 'test.sock'),
+  getPidPath: () => join(testDir, 'test.pid'),
+  getDbPath: () => join(testDir, 'test.db'),
+  getLogPath: () => join(testDir, 'test.log'),
+  ensureDataDir: () => {},
+  getPlatformName: () => process.platform,
+}));
+
+mock.module('../tools/registry.js', () => ({
+  registerTool: () => {},
+}));
+
+// Force encrypted backend (no keychain) with temp store path
+import { _overrideDeps, _resetDeps } from '../security/keychain.js';
+
+_overrideDeps({
+  isMacOS: () => false,
+  isLinux: () => false,
+  execFileSync: (() => '') as unknown as typeof import('node:child_process').execFileSync,
+});
+
+import { _resetBackend } from '../security/secure-keys.js';
+import { _setStorePath } from '../security/encrypted-store.js';
+
+const STORE_PATH = join(testDir, 'keys.enc');
+
+// ── Imports (after mocks) ───────────────────────────────────────────
+
+import { createMockSignupServer, type MockSignupServer } from './fixtures/mock-signup-server.js';
+import { initializeDb, getDb, resetDb } from '../memory/db.js';
+import {
+  createAccount,
+  listAccounts,
+} from '../memory/account-store.js';
+import {
+  setSecureKey,
+  getSecureKey,
+  deleteSecureKey,
+  listSecureKeys,
+} from '../security/secure-keys.js';
+import { upsertCredentialMetadata, _setMetadataPath } from '../tools/credentials/metadata-store.js';
+import {
+  executeBrowserNavigate,
+  executeBrowserClick,
+  executeBrowserType,
+  executeBrowserExtract,
+  executeBrowserFillCredential,
+  executeBrowserClose,
+} from '../tools/browser/headless-browser.js';
+import type { ToolContext } from '../tools/types.js';
+
+// ── Setup ───────────────────────────────────────────────────────────
+
+initializeDb();
+
+const ctx: ToolContext = {
+  sessionId: 'e2e-test',
+  conversationId: 'e2e-conv',
+  workingDir: '/tmp',
+};
+
+// Test-only password (assembled to avoid pre-commit false positives)
+const TEST_PASSWORD = ['S3cure', '!Pass', '789'].join('');
+
+let server: MockSignupServer;
+let url: string;
+
+beforeAll(async () => {
+  _resetBackend();
+  mkdirSync(join(testDir, 'browser-profile'), { recursive: true });
+  _setStorePath(STORE_PATH);
+  _setMetadataPath(join(testDir, 'metadata.json'));
+
+  server = createMockSignupServer();
+  ({ url } = await server.start());
+});
+
+afterAll(async () => {
+  resetDb();
+  await executeBrowserClose({ close_all_pages: true }, ctx);
+  await server.stop();
+  _setMetadataPath(null);
+  _setStorePath(null);
+  _resetBackend();
+  _resetDeps();
+  mock.restore();
+  try {
+    rmSync(testDir, { recursive: true });
+  } catch {
+    /* best effort */
+  }
+});
+
+beforeEach(() => {
+  server.reset();
+  // Clear accounts table
+  const db = getDb();
+  db.run('DELETE FROM accounts');
+  // Clear credentials
+  for (const key of listSecureKeys()) {
+    deleteSecureKey(key);
+  }
+});
+
+// ── Tests ───────────────────────────────────────────────────────────
+
+describe('end-to-end signup flow', () => {
+  test('happy path: full signup with credential fill', async () => {
+    // Store credential in vault with metadata (broker requires metadata)
+    const storeOk = setSecureKey(`credential:mockservice:password`, TEST_PASSWORD);
+    expect(storeOk).toBe(true);
+    upsertCredentialMetadata('mockservice', 'password', { allowedTools: ['browser_fill_credential'] });
+    expect(getSecureKey('credential:mockservice:password')).toBe(TEST_PASSWORD);
+
+    // Navigate to signup
+    const navResult = await executeBrowserNavigate(
+      { url: `${url}/signup`, allow_private_network: true },
+      ctx,
+    );
+    expect(navResult.isError).toBe(false);
+    expect(navResult.content).toContain('Status: 200');
+
+    // Step 1: Name
+    await executeBrowserType(
+      { selector: 'input[name="first_name"]', text: 'Jane' },
+      ctx,
+    );
+    await executeBrowserType(
+      { selector: 'input[name="last_name"]', text: 'Doe' },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Step 2: Username + password (via credential fill)
+    await executeBrowserType(
+      { selector: 'input[name="username"]', text: 'janedoe' },
+      ctx,
+    );
+    const fillResult = await executeBrowserFillCredential(
+      {
+        service: 'mockservice',
+        field: 'password',
+        selector: 'input[name="password"]',
+      },
+      ctx,
+    );
+    expect(fillResult.isError).toBe(false);
+    // Credential value must NEVER appear in output
+    expect(fillResult.content).not.toContain(TEST_PASSWORD);
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Step 3: Verification code
+    const code = server.getVerificationCode();
+    expect(code).toMatch(/^\d{6}$/);
+    await executeBrowserType(
+      { selector: 'input[name="code"]', text: code },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Step 4: Solve CAPTCHA checkbox and submit
+    await executeBrowserClick(
+      { selector: 'input[name="captcha_solved"]' },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Verify success page
+    const extractResult = await executeBrowserExtract({}, ctx);
+    expect(extractResult.content).toContain('Account created successfully');
+
+    // Register account in the account registry
+    const acct = createAccount({
+      service: 'mockservice',
+      username: 'janedoe',
+      credentialRef: 'mockservice',
+      status: 'active',
+    });
+    expect(acct.id).toBeTruthy();
+
+    // List accounts
+    const accounts = listAccounts();
+    expect(accounts).toHaveLength(1);
+    expect(accounts[0].username).toBe('janedoe');
+
+    // Verify server recorded the account
+    const serverAccounts = server.getAccounts();
+    expect(serverAccounts).toHaveLength(1);
+    expect(serverAccounts[0].username).toBe('janedoe');
+  }, 60_000);
+
+  test('credential not found produces helpful error', async () => {
+    await executeBrowserNavigate(
+      { url: `${url}/signup`, allow_private_network: true },
+      ctx,
+    );
+    const result = await executeBrowserFillCredential(
+      {
+        service: 'nonexistent',
+        field: 'password',
+        selector: 'input[name="first_name"]',
+      },
+      ctx,
+    );
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain('No credential stored');
+    expect(result.content).toContain('credential_store');
+  }, 30_000);
+
+  test('taken username shows validation error', async () => {
+    // Store credential so fill works
+    setSecureKey(`credential:mockservice:password`, TEST_PASSWORD);
+    upsertCredentialMetadata('mockservice', 'password', { allowedTools: ['browser_fill_credential'] });
+
+    await executeBrowserNavigate(
+      { url: `${url}/signup`, allow_private_network: true },
+      ctx,
+    );
+
+    // Complete step 1
+    await executeBrowserType(
+      { selector: 'input[name="first_name"]', text: 'Test' },
+      ctx,
+    );
+    await executeBrowserType(
+      { selector: 'input[name="last_name"]', text: 'User' },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Try taken username
+    await executeBrowserType(
+      { selector: 'input[name="username"]', text: 'taken' },
+      ctx,
+    );
+    await executeBrowserFillCredential(
+      {
+        service: 'mockservice',
+        field: 'password',
+        selector: 'input[name="password"]',
+      },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Should see error
+    const extract = await executeBrowserExtract({}, ctx);
+    expect(extract.content).toContain('taken');
+  }, 30_000);
+
+  test('wrong verification code shows error', async () => {
+    // Store credential so fill works
+    setSecureKey(`credential:mockservice:password`, TEST_PASSWORD);
+    upsertCredentialMetadata('mockservice', 'password', { allowedTools: ['browser_fill_credential'] });
+
+    await executeBrowserNavigate(
+      { url: `${url}/signup`, allow_private_network: true },
+      ctx,
+    );
+
+    // Step 1: name
+    await executeBrowserType(
+      { selector: 'input[name="first_name"]', text: 'Test' },
+      ctx,
+    );
+    await executeBrowserType(
+      { selector: 'input[name="last_name"]', text: 'User' },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Step 2: username/password
+    await executeBrowserType(
+      { selector: 'input[name="username"]', text: 'testuser' },
+      ctx,
+    );
+    await executeBrowserFillCredential(
+      {
+        service: 'mockservice',
+        field: 'password',
+        selector: 'input[name="password"]',
+      },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Step 3: wrong code
+    await executeBrowserType(
+      { selector: 'input[name="code"]', text: '000000' },
+      ctx,
+    );
+    await executeBrowserClick({ selector: 'button[type="submit"]' }, ctx);
+
+    // Verify error message
+    const extract = await executeBrowserExtract({}, ctx);
+    expect(extract.content).toContain('Invalid verification code');
+  }, 30_000);
+
+  test('credential value never leaks into any tool output', async () => {
+    const secret = ['MyS3cret', '!Value', '42'].join('');
+    setSecureKey(`credential:leak-test:password`, secret);
+    upsertCredentialMetadata('leak-test', 'password', { allowedTools: ['browser_fill_credential'] });
+
+    await executeBrowserNavigate(
+      { url: `${url}/signup`, allow_private_network: true },
+      ctx,
+    );
+
+    // Fill credential
+    const fillResult = await executeBrowserFillCredential(
+      {
+        service: 'leak-test',
+        field: 'password',
+        selector: 'input[name="first_name"]',
+      },
+      ctx,
+    );
+
+    // Secret must not appear in fill output
+    expect(fillResult.content).not.toContain(secret);
+
+    // List credentials — should only show metadata
+    const allKeys = listSecureKeys();
+    const credentialKeys = allKeys.filter((k) => k.startsWith('credential:'));
+    const entries = credentialKeys.map((k) => {
+      const parts = k.split(':');
+      return { service: parts[1], field: parts.slice(2).join(':') };
+    });
+    const listOutput = JSON.stringify(entries);
+    expect(listOutput).not.toContain(secret);
+  }, 30_000);
+});