@vellumai/assistant 0.3.0

package/src/__tests__/compaction.benchmark.test.ts

@@ -0,0 +1,176 @@
+/**
+ * Context Window Compaction Benchmark
+ *
+ * Measures compaction cost with a mock provider:
+ * - compaction latency under threshold pressure
+ * - no-op fast path for below-threshold histories
+ * - token reduction ratio after compaction
+ * - summary call count within expected range
+ * - severe pressure overriding cooldown
+ */
+import { describe, expect, mock, test } from 'bun:test';
+
+import { DEFAULT_CONFIG } from '../config/defaults.js';
+import { ContextWindowManager } from '../context/window-manager.js';
+import { estimatePromptTokens } from '../context/token-estimator.js';
+import type { Message, Provider } from '../providers/types.js';
+
+mock.module('../util/logger.js', () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+function makeSummaryProvider(counter: { calls: number }): Provider {
+  return {
+    name: 'mock',
+    async sendMessage() {
+      counter.calls += 1;
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `## Goals\n- Preserve state\n## Constraints\n- Keep PRs small\n## Decisions\n- Call ${counter.calls}`,
+          },
+        ],
+        model: 'mock-model',
+        usage: { inputTokens: 420, outputTokens: 85 },
+        stopReason: 'end_turn',
+      };
+    },
+  };
+}
+
+function makeLongMessages(turns: number): Message[] {
+  const rows: Message[] = [];
+  for (let i = 0; i < turns; i++) {
+    rows.push({
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          text: `[U${i}] User message with enough content to estimate tokens. Topic ${i % 9}.`,
+        },
+      ],
+    });
+    rows.push({
+      role: 'assistant',
+      content: [
+        {
+          type: 'text',
+          text: `[A${i}] Assistant response with relevant content. Result ${i % 7}.`,
+        },
+      ],
+    });
+  }
+  return rows;
+}
+
+function makeConfig() {
+  return {
+    ...DEFAULT_CONFIG.contextWindow,
+    maxInputTokens: 6000,
+    targetInputTokens: 3200,
+    compactThreshold: 0.6,
+    preserveRecentUserTurns: 8,
+    chunkTokens: 1200,
+  };
+}
+
+describe('Compaction benchmark', () => {
+  test('compaction with mock provider completes under 500ms', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 90 turns = 180 messages, well above 60% of 6000 = 3600 threshold
+    const messages = makeLongMessages(90);
+    const before = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(before).toBeGreaterThan(config.maxInputTokens * config.compactThreshold);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(true);
+    expect(elapsed).toBeLessThan(500);
+  });
+
+  test('below-threshold check returns in under 50ms (no-op)', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    // 3 turns = 6 messages, well below threshold
+    const messages = makeLongMessages(3);
+
+    const start = performance.now();
+    const result = await manager.maybeCompact(messages);
+    const elapsed = performance.now() - start;
+
+    expect(result.compacted).toBe(false);
+    expect(result.reason).toBe('below compaction threshold');
+    expect(elapsed).toBeLessThan(50);
+    expect(counter.calls).toBe(0);
+  });
+
+  test('token reduction ratio exceeds 30% after compaction', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    const reductionRatio =
+      (result.previousEstimatedInputTokens - result.estimatedInputTokens) /
+      result.previousEstimatedInputTokens;
+    expect(reductionRatio).toBeGreaterThan(0.3);
+  });
+
+  test('summary calls fall within 2-6 range', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    const config = makeConfig();
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const result = await manager.maybeCompact(messages);
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThanOrEqual(2);
+    expect(result.summaryCalls).toBeLessThanOrEqual(6);
+    expect(result.summaryCalls).toBe(counter.calls);
+  });
+
+  test('severe pressure triggers compaction even during cooldown', async () => {
+    const counter = { calls: 0 };
+    const provider = makeSummaryProvider(counter);
+    // Use a tighter maxInputTokens so 90 turns exceeds the 95% severe threshold
+    const config = {
+      ...makeConfig(),
+      maxInputTokens: 4000,
+      targetInputTokens: 2000,
+    };
+    const manager = new ContextWindowManager(provider, 'system prompt', config);
+
+    const messages = makeLongMessages(90);
+    const estimated = estimatePromptTokens(messages, 'system prompt', {
+      providerName: 'mock',
+    });
+    expect(estimated).toBeGreaterThan(config.maxInputTokens * 0.95);
+
+    // Simulate being within cooldown by setting lastCompactedAt to now
+    const result = await manager.maybeCompact(messages, undefined, {
+      lastCompactedAt: Date.now(),
+    });
+
+    expect(result.compacted).toBe(true);
+    expect(result.summaryCalls).toBeGreaterThan(0);
+  });
+});

package/src/__tests__/computer-use-session-compaction.test.ts

@@ -0,0 +1,132 @@
+import { describe, test, expect } from 'bun:test';
+import { ComputerUseSession } from '../daemon/computer-use-session.js';
+import type { Message } from '../providers/types.js';
+
+/**
+ * Helper to create a user message with a tool_result block containing
+ * an AX tree wrapped in markers.
+ */
+function toolResultMsg(content: string): Message {
+  return {
+    role: 'user',
+    content: [
+      {
+        type: 'tool_result',
+        tool_use_id: 'test-id',
+        content,
+      },
+    ],
+  };
+}
+
+describe('ComputerUseSession.escapeAxTreeContent', () => {
+  test('escapes a literal closing tag in the content', () => {
+    const input = 'some text </ax-tree> more text';
+    const escaped = ComputerUseSession.escapeAxTreeContent(input);
+    expect(escaped).toBe('some text &lt;/ax-tree&gt; more text');
+  });
+
+  test('escapes multiple occurrences', () => {
+    const input = '</ax-tree> hello </ax-tree>';
+    const escaped = ComputerUseSession.escapeAxTreeContent(input);
+    expect(escaped).toBe('&lt;/ax-tree&gt; hello &lt;/ax-tree&gt;');
+  });
+
+  test('is case-insensitive', () => {
+    const input = '</AX-TREE> and </Ax-Tree>';
+    const escaped = ComputerUseSession.escapeAxTreeContent(input);
+    expect(escaped).toBe('&lt;/ax-tree&gt; and &lt;/ax-tree&gt;');
+  });
+
+  test('leaves content without closing tags unchanged', () => {
+    const input = 'Window "My App" [1]\n  Button "OK" [2]';
+    expect(ComputerUseSession.escapeAxTreeContent(input)).toBe(input);
+  });
+});
+
+describe('ComputerUseSession.compactHistory', () => {
+  test('[experimental] strips old AX trees and keeps the most recent ones', () => {
+    const messages: Message[] = [
+      { role: 'assistant', content: [{ type: 'text', text: 'thinking...' }] },
+      toolResultMsg('<ax-tree>CURRENT SCREEN STATE:\nWindow "App" [1]</ax-tree>'),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 1' }] },
+      toolResultMsg('<ax-tree>CURRENT SCREEN STATE:\nWindow "App" [2]</ax-tree>'),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 2' }] },
+      toolResultMsg('<ax-tree>CURRENT SCREEN STATE:\nWindow "App" [3]</ax-tree>'),
+    ];
+
+    const compacted = ComputerUseSession.compactHistory(messages);
+
+    // First AX tree (index 1) should be stripped
+    const firstToolResult = compacted[1].content[0];
+    expect(firstToolResult.type).toBe('tool_result');
+    if (firstToolResult.type === 'tool_result') {
+      expect(firstToolResult.content).toContain('<ax_tree_omitted />');
+      expect(firstToolResult.content).not.toContain('<ax-tree>');
+    }
+
+    // Last two AX trees should be preserved
+    const secondToolResult = compacted[3].content[0];
+    if (secondToolResult.type === 'tool_result') {
+      expect(secondToolResult.content).toContain('<ax-tree>');
+    }
+    const thirdToolResult = compacted[5].content[0];
+    if (thirdToolResult.type === 'tool_result') {
+      expect(thirdToolResult.content).toContain('<ax-tree>');
+    }
+  });
+
+  test('[experimental] handles AX tree content containing literal </ax-tree> (escaped)', () => {
+    // Simulate content where the AX tree text includes an escaped closing tag,
+    // e.g. user is viewing XML source code with "</ax-tree>" in it.
+    const escapedContent =
+      '<ax-tree>CURRENT SCREEN STATE:\nTextArea "editor" [1]\n  ' +
+      'Line: &lt;/ax-tree&gt; some xml\n</ax-tree>';
+
+    const messages: Message[] = [
+      { role: 'assistant', content: [{ type: 'text', text: 'action 0' }] },
+      toolResultMsg(escapedContent),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 1' }] },
+      toolResultMsg(escapedContent),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 2' }] },
+      toolResultMsg('<ax-tree>CURRENT SCREEN STATE:\nWindow "App" [3]</ax-tree>'),
+    ];
+
+    const compacted = ComputerUseSession.compactHistory(messages);
+
+    // The first message with escaped content should be fully stripped
+    const firstToolResult = compacted[1].content[0];
+    if (firstToolResult.type === 'tool_result') {
+      expect(firstToolResult.content).not.toContain('<ax-tree>');
+      expect(firstToolResult.content).toContain('<ax_tree_omitted />');
+    }
+  });
+
+  test('regex fails on unescaped </ax-tree> inside content (demonstrating the bug)', () => {
+    // This test demonstrates what happens WITHOUT escaping: the regex
+    // only partially removes the AX tree block.
+    const unescapedContent =
+      '<ax-tree>CURRENT SCREEN STATE:\nTextArea "editor" [1]\n  ' +
+      'Line: </ax-tree> some xml leftover\n</ax-tree>';
+
+    const messages: Message[] = [
+      { role: 'assistant', content: [{ type: 'text', text: 'action 0' }] },
+      toolResultMsg(unescapedContent),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 1' }] },
+      toolResultMsg(unescapedContent),
+      { role: 'assistant', content: [{ type: 'text', text: 'action 2' }] },
+      toolResultMsg('<ax-tree>CURRENT SCREEN STATE:\nWindow "App" [3]</ax-tree>'),
+    ];
+
+    const compacted = ComputerUseSession.compactHistory(messages);
+
+    // Without escaping, the first tool result has leftover content after
+    // the regex only matched up to the FIRST </ax-tree>.
+    const firstToolResult = compacted[1].content[0];
+    if (firstToolResult.type === 'tool_result') {
+      // The non-greedy regex stops at the first </ax-tree>, leaving
+      // " some xml leftover\n</ax-tree>" behind.
+      expect(firstToolResult.content).toContain('some xml leftover');
+    }
+  });
+});

package/src/__tests__/computer-use-session-lifecycle.test.ts

@@ -0,0 +1,293 @@
+import { describe, test, expect, mock } from 'bun:test';
+
+// Mock config before importing modules that depend on it.
+// The permissions mode must be 'legacy' so computer-use tools
+// (which have no trust rules in test) don't trigger approval prompts.
+mock.module('../config/loader.js', () => ({
+  getConfig: () => ({
+    provider: 'mock-provider',
+    permissions: { mode: 'legacy' },
+    apiKeys: {},
+    sandbox: { enabled: false },
+    timeouts: { toolExecutionTimeoutSec: 30, permissionTimeoutSec: 5 },
+    skills: { load: { extraDirs: [] } },
+    secretDetection: { enabled: false },
+    contextWindow: {
+      enabled: true,
+      maxInputTokens: 180000,
+      targetInputTokens: 110000,
+      compactThreshold: 0.8,
+      preserveRecentUserTurns: 8,
+      summaryMaxTokens: 1200,
+      chunkTokens: 12000,
+    },
+  }),
+  invalidateConfigCache: () => {},
+}));
+
+import { ComputerUseSession } from '../daemon/computer-use-session.js';
+import type { Provider, ProviderResponse } from '../providers/types.js';
+import type { CuObservation, ServerMessage } from '../daemon/ipc-protocol.js';
+
+function createProvider(responses: ProviderResponse[]): { provider: Provider; getCalls: () => number } {
+  let calls = 0;
+  const provider: Provider = {
+    name: 'mock',
+    async sendMessage() {
+      const response = responses[calls] ?? responses[responses.length - 1];
+      calls++;
+      return response;
+    },
+  };
+  return { provider, getCalls: () => calls };
+}
+
+describe('ComputerUseSession lifecycle', () => {
+  test('stops provider loop immediately after terminal computer_use_done tool', async () => {
+    const { provider, getCalls } = createProvider([
+      {
+        content: [{
+          type: 'tool_use',
+          id: 'tu-1',
+          name: 'computer_use_done',
+          input: { summary: 'Task finished' },
+        }],
+        model: 'mock-model',
+        usage: { inputTokens: 10, outputTokens: 5 },
+        stopReason: 'tool_use',
+      },
+      {
+        content: [{ type: 'text', text: 'This should never be requested' }],
+        model: 'mock-model',
+        usage: { inputTokens: 10, outputTokens: 5 },
+        stopReason: 'end_turn',
+      },
+    ]);
+
+    const sentMessages: ServerMessage[] = [];
+    let terminalCalls = 0;
+
+    const session = new ComputerUseSession(
+      'cu-test-1',
+      'test task',
+      1440,
+      900,
+      provider,
+      (msg) => { sentMessages.push(msg); },
+      'computer_use',
+      () => { terminalCalls++; },
+    );
+
+    const observation: CuObservation = {
+      type: 'cu_observation',
+      sessionId: 'cu-test-1',
+      axTree: 'Window "Test" [1]',
+    };
+
+    await session.handleObservation(observation);
+
+    // If computer_use_done does not abort the loop, we'd see an extra provider call.
+    expect(getCalls()).toBe(1);
+    expect(session.getState()).toBe('complete');
+    expect(terminalCalls).toBe(1);
+
+    const completes = sentMessages.filter(
+      (msg): msg is Extract<ServerMessage, { type: 'cu_complete' }> => msg.type === 'cu_complete',
+    );
+    expect(completes).toHaveLength(1);
+    expect(completes[0].summary).toBe('Task finished');
+  });
+
+  test('notifies terminal callback only once on repeated abort calls', () => {
+    const { provider } = createProvider([
+      {
+        content: [{ type: 'text', text: 'unused' }],
+        model: 'mock-model',
+        usage: { inputTokens: 1, outputTokens: 1 },
+        stopReason: 'end_turn',
+      },
+    ]);
+
+    let terminalCalls = 0;
+    const session = new ComputerUseSession(
+      'cu-test-2',
+      'test task',
+      1440,
+      900,
+      provider,
+      () => {},
+      'computer_use',
+      () => { terminalCalls++; },
+    );
+
+    session.abort();
+    session.abort();
+
+    expect(terminalCalls).toBe(1);
+    expect(session.getState()).toBe('error');
+  });
+
+  test('CU session passes exactly 12 computer_use_* tools to the agent loop', async () => {
+    let capturedTools: string[] = [];
+    const provider: Provider = {
+      name: 'mock',
+      async sendMessage(_msgs, tools) {
+        capturedTools = (tools ?? []).map((t) => t.name);
+        return {
+          content: [{
+            type: 'tool_use',
+            id: 'tu-capture',
+            name: 'computer_use_done',
+            input: { summary: 'Done' },
+          }],
+          model: 'mock-model',
+          usage: { inputTokens: 10, outputTokens: 5 },
+          stopReason: 'tool_use',
+        };
+      },
+    };
+
+    const session = new ComputerUseSession(
+      'cu-tool-capture',
+      'capture tools',
+      1440,
+      900,
+      provider,
+      () => {},
+      'computer_use',
+    );
+
+    await session.handleObservation({
+      type: 'cu_observation',
+      sessionId: 'cu-tool-capture',
+      axTree: 'Window "Test" [1]',
+    });
+
+    const cuTools = capturedTools.filter((n) => n.startsWith('computer_use_'));
+    expect(cuTools).toHaveLength(12);
+
+    // Assert exact set of expected CU tool names
+    const expectedCuTools = [
+      'computer_use_click',
+      'computer_use_double_click',
+      'computer_use_right_click',
+      'computer_use_type_text',
+      'computer_use_key',
+      'computer_use_scroll',
+      'computer_use_drag',
+      'computer_use_wait',
+      'computer_use_open_app',
+      'computer_use_run_applescript',
+      'computer_use_done',
+      'computer_use_respond',
+    ];
+    for (const name of expectedCuTools) {
+      expect(cuTools).toContain(name);
+    }
+  });
+
+  test('computer_use_respond is a terminal tool that completes the session', async () => {
+    const { provider } = createProvider([
+      {
+        content: [{
+          type: 'tool_use',
+          id: 'tu-respond',
+          name: 'computer_use_respond',
+          input: { answer: 'The meeting is at 3pm', reasoning: 'Found in calendar' },
+        }],
+        model: 'mock-model',
+        usage: { inputTokens: 10, outputTokens: 5 },
+        stopReason: 'tool_use',
+      },
+    ]);
+
+    const sentMessages: ServerMessage[] = [];
+    const session = new ComputerUseSession(
+      'cu-respond-test',
+      'check my schedule',
+      1440,
+      900,
+      provider,
+      (msg) => { sentMessages.push(msg); },
+      'computer_use',
+    );
+
+    await session.handleObservation({
+      type: 'cu_observation',
+      sessionId: 'cu-respond-test',
+      axTree: 'Window "Calendar" [1]',
+    });
+
+    expect(session.getState()).toBe('complete');
+    const completes = sentMessages.filter(
+      (msg): msg is Extract<ServerMessage, { type: 'cu_complete' }> => msg.type === 'cu_complete',
+    );
+    expect(completes).toHaveLength(1);
+    expect(completes[0].summary).toBe('The meeting is at 3pm');
+    expect(completes[0].isResponse).toBe(true);
+  });
+
+  test('default construction preactivates computer-use skill and provides 12 CU tools', async () => {
+    let capturedTools: string[] = [];
+    const provider: Provider = {
+      name: 'mock',
+      async sendMessage(_msgs, tools) {
+        capturedTools = (tools ?? []).map((t) => t.name);
+        return {
+          content: [{
+            type: 'tool_use',
+            id: 'tu-default',
+            name: 'computer_use_done',
+            input: { summary: 'Done' },
+          }],
+          model: 'mock-model',
+          usage: { inputTokens: 10, outputTokens: 5 },
+          stopReason: 'tool_use',
+        };
+      },
+    };
+
+    // No preactivatedSkillIds passed — defaults to ['computer-use'] via skill projection
+    const session = new ComputerUseSession(
+      'cu-default-projection',
+      'test default projection',
+      1440, 900,
+      provider,
+      () => {},
+      'computer_use',
+      undefined,
+    );
+
+    await session.handleObservation({
+      type: 'cu_observation',
+      sessionId: 'cu-default-projection',
+      axTree: 'Window "Test" [1]',
+    });
+
+    const cuTools = capturedTools.filter((n) => n.startsWith('computer_use_'));
+    expect(cuTools).toHaveLength(12);
+  });
+
+  test('constructor accepts preactivatedSkillIds parameter', () => {
+    const { provider } = createProvider([{
+      content: [{ type: 'text', text: 'unused' }],
+      model: 'mock-model',
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: 'end_turn',
+    }]);
+
+    // Should not throw
+    const session = new ComputerUseSession(
+      'cu-preactivated',
+      'test preactivated',
+      1440, 900,
+      provider,
+      () => {},
+      'computer_use',
+      undefined,
+      ['computer-use'],
+    );
+
+    expect(session).toBeDefined();
+  });
+});