@bluecopa/harness 0.1.0-snapshot.99 → 1.0.0

package/tests/integration/structured-messages-agent.spec.ts

@@ -0,0 +1,265 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { createAgent } from '../../src/agent/create-agent';
+import { LocalToolProvider } from '../../src/providers/local-tool-provider';
+import type { AgentMessage, AgentAction, AgentStreamEvent } from '../../src/agent/types';
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
+  tempDirs.length = 0;
+});
+
+describe('structured messages in agent run()', () => {
+  it('single tool call produces assistant with toolCalls and tool with toolResults', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-struct-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      async nextAction() {
+        turn += 1;
+        if (turn === 1) {
+          return {
+            type: 'tool',
+            name: 'Write',
+            args: { path: 'test.txt', content: 'structured' },
+          };
+        }
+        return { type: 'final', content: 'done' };
+      },
+    });
+
+    const result = await agent.run('write a file');
+
+    // Find the assistant message with toolCalls
+    const assistantMsg = result.messages.find(
+      (m) => m.role === 'assistant' && m.toolCalls && m.toolCalls.length > 0
+    );
+    expect(assistantMsg).toBeDefined();
+    expect(assistantMsg!.toolCalls).toHaveLength(1);
+    expect(assistantMsg!.toolCalls![0]!.toolName).toBe('Write');
+    expect(assistantMsg!.toolCalls![0]!.toolCallId).toBeTruthy();
+    expect(assistantMsg!.toolCalls![0]!.args).toEqual({ path: 'test.txt', content: 'structured' });
+
+    // Find the tool message with toolResults
+    const toolMsg = result.messages.find(
+      (m) => m.role === 'tool' && m.toolResults && m.toolResults.length > 0
+    );
+    expect(toolMsg).toBeDefined();
+    expect(toolMsg!.toolResults).toHaveLength(1);
+    expect(toolMsg!.toolResults![0]!.toolName).toBe('Write');
+    expect(toolMsg!.toolResults![0]!.isError).toBe(false);
+    // content string still present for display
+    expect(toolMsg!.content).toContain('Write(test.txt)');
+  });
+
+  it('tool batch produces assistant with multiple toolCalls', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-batch-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      async nextAction() {
+        turn += 1;
+        if (turn === 1) {
+          return {
+            type: 'tool_batch',
+            calls: [
+              { type: 'tool', name: 'Write', args: { path: 'a.txt', content: 'aaa' } },
+              { type: 'tool', name: 'Write', args: { path: 'b.txt', content: 'bbb' } },
+            ],
+          };
+        }
+        return { type: 'final', content: 'done' };
+      },
+    });
+
+    const result = await agent.run('write two files');
+
+    // Assistant message should have 2 toolCalls
+    const assistantMsg = result.messages.find(
+      (m) => m.role === 'assistant' && m.toolCalls && m.toolCalls.length === 2
+    );
+    expect(assistantMsg).toBeDefined();
+    expect(assistantMsg!.toolCalls![0]!.toolName).toBe('Write');
+    expect(assistantMsg!.toolCalls![1]!.toolName).toBe('Write');
+    // Each should have a unique toolCallId
+    expect(assistantMsg!.toolCalls![0]!.toolCallId).not.toBe(
+      assistantMsg!.toolCalls![1]!.toolCallId
+    );
+
+    // Should have 2 tool result messages
+    const toolMsgs = result.messages.filter(
+      (m) => m.role === 'tool' && m.toolResults && m.toolResults.length > 0
+    );
+    expect(toolMsgs).toHaveLength(2);
+    expect(toolMsgs[0]!.toolResults![0]!.toolName).toBe('Write');
+    expect(toolMsgs[1]!.toolResults![0]!.toolName).toBe('Write');
+  });
+
+  it('failed tool call sets isError on toolResults', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-err-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      async nextAction() {
+        turn += 1;
+        if (turn === 1) {
+          // Read a file that doesn't exist
+          return {
+            type: 'tool',
+            name: 'Read',
+            args: { path: 'nonexistent.txt' },
+          };
+        }
+        return { type: 'final', content: 'done' };
+      },
+    });
+
+    const result = await agent.run('read missing file');
+
+    const toolMsg = result.messages.find(
+      (m) => m.role === 'tool' && m.toolResults && m.toolResults.length > 0
+    );
+    expect(toolMsg).toBeDefined();
+    expect(toolMsg!.toolResults![0]!.isError).toBe(true);
+    expect(toolMsg!.toolResults![0]!.result).toContain('ERROR');
+  });
+
+  it('toolCallId from loop action is preserved through to messages', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-id-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+    const knownId = 'toolu_test_abc123';
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      async nextAction() {
+        turn += 1;
+        if (turn === 1) {
+          return {
+            type: 'tool',
+            name: 'Write',
+            args: { path: 'id-test.txt', content: 'hi' },
+            toolCallId: knownId,
+          };
+        }
+        return { type: 'final', content: 'done' };
+      },
+    });
+
+    const result = await agent.run('test toolCallId passthrough');
+
+    const assistantMsg = result.messages.find(
+      (m) => m.role === 'assistant' && m.toolCalls?.some((tc) => tc.toolCallId === knownId)
+    );
+    expect(assistantMsg).toBeDefined();
+
+    const toolMsg = result.messages.find(
+      (m) => m.role === 'tool' && m.toolResults?.some((tr) => tr.toolCallId === knownId)
+    );
+    expect(toolMsg).toBeDefined();
+  });
+});
+
+describe('structured messages in agent stream()', () => {
+  it('streaming yields tool_start events and builds structured messages', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-stream-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+    const toolCallId = 'stream-call-1';
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      loop: {
+        async nextAction() {
+          turn += 1;
+          return { type: 'final', content: 'done' };
+        },
+        async *streamAction(messages: AgentMessage[]): AsyncGenerator<AgentStreamEvent> {
+          turn += 1;
+          if (turn === 1) {
+            yield { type: 'text_delta', text: 'Creating file...' };
+            yield {
+              type: 'tool_start',
+              name: 'Write',
+              args: { path: 'stream.txt', content: 'streamed' },
+              toolCallId,
+            };
+          }
+          // No tools on second turn → final
+        },
+      },
+    });
+
+    const events: AgentStreamEvent[] = [];
+    for await (const event of agent.stream('stream test')) {
+      events.push(event);
+    }
+
+    // Should have a tool_start event with toolCallId
+    const toolStart = events.find(
+      (e) => e.type === 'tool_start' && e.name === 'Write'
+    );
+    expect(toolStart).toBeDefined();
+
+    // Should have a done event
+    const done = events.find((e) => e.type === 'done');
+    expect(done).toBeDefined();
+  });
+
+  it('streaming non-streaming fallback builds structured messages', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-fallback-'));
+    tempDirs.push(dir);
+
+    const provider = new LocalToolProvider(dir);
+
+    let turn = 0;
+    const agent = createAgent({
+      toolProvider: provider,
+      loop: {
+        async nextAction(): Promise<AgentAction> {
+          turn += 1;
+          if (turn === 1) {
+            return {
+              type: 'tool',
+              name: 'Write',
+              args: { path: 'fallback.txt', content: 'via-nextAction' },
+              toolCallId: 'fallback-id',
+            };
+          }
+          return { type: 'final', content: 'complete' };
+        },
+        // No streamAction — forces fallback path
+      },
+    });
+
+    const events: AgentStreamEvent[] = [];
+    for await (const event of agent.stream('fallback test')) {
+      events.push(event);
+    }
+
+    expect(events.some((e) => e.type === 'tool_start' && e.name === 'Write')).toBe(true);
+    expect(events.some((e) => e.type === 'tool_end' && e.name === 'Write')).toBe(true);
+    expect(events.some((e) => e.type === 'done')).toBe(true);
+  });
+});

package/tests/integration/subagent-isolation.spec.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest';
+
+import { createSubagentManager } from '../../src/subagents/subagent-manager';
+import { LocalToolProvider } from '../../src/providers/local-tool-provider';
+
+describe('subagent isolation', () => {
+  it('runs child tasks with isolated runtime state', async () => {
+    let callCount = 0;
+
+    const manager = createSubagentManager(() => ({
+      toolProvider: new LocalToolProvider(),
+      async nextAction() {
+        callCount += 1;
+        return { type: 'final', content: `child-finished-${callCount}` };
+      }
+    }));
+
+    const first = await manager.runIsolated('child task 1');
+    const second = await manager.runIsolated('child task 2');
+
+    expect(first.output).toBe('child-finished-1');
+    expect(second.output).toBe('child-finished-2');
+  });
+});

package/tests/security/skill-sandbox-isolation.spec.ts

@@ -0,0 +1,51 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import type { SandboxProvider } from '../../src/interfaces/sandbox-provider';
+import { SkillManager } from '../../src/skills/skill-manager';
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
+  tempDirs.length = 0;
+});
+
+class NoopSandbox implements SandboxProvider {
+  async exec() {
+    return { exitCode: 0, stdout: '', stderr: '' };
+  }
+
+  async readSandboxFile() {
+    return { data: new Uint8Array() };
+  }
+
+  async writeSandboxFile() {}
+}
+
+describe('skill sandbox isolation', () => {
+  it('rejects unsafe skill paths containing traversal sequences', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'harness-skill-security-'));
+    tempDirs.push(dir);
+
+    const indexPath = join(dir, 'skills.json');
+    await writeFile(
+      indexPath,
+      JSON.stringify([
+        {
+          name: 'unsafe',
+          description: 'unsafe path',
+          path: '../outside/SKILL.md'
+        }
+      ]),
+      'utf8'
+    );
+
+    const manager = new SkillManager(new NoopSandbox());
+    await manager.discover(indexPath);
+
+    await expect(manager.invoke('unsafe')).rejects.toThrow('unsafe skill path');
+  });
+});

package/tests/unit/create-tools-schema-parity.spec.ts

@@ -0,0 +1,22 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { describe, expect, it } from 'vitest';
+
+import { createToolRegistry, extractedToolNames, loadExtractedTools } from '../../src/agent/create-tools';
+
+describe('createTools parity', () => {
+  it('loads extracted tools and preserves canonical names', () => {
+    const expectedNames = ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'];
+    expect(extractedToolNames()).toEqual(expectedNames);
+
+    const registry = createToolRegistry();
+    expect([...registry.keys()]).toEqual(expectedNames);
+  });
+
+  it('matches extracted snapshot exactly', () => {
+    const path = resolve(import.meta.dirname, '../../src/extracted/tools.json');
+    const raw = JSON.parse(readFileSync(path, 'utf8')) as ReturnType<typeof loadExtractedTools>;
+    const loaded = loadExtractedTools();
+    expect(loaded).toEqual(raw);
+  });
+});

package/tests/unit/extracted-manifest.spec.ts

@@ -0,0 +1,41 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { describe, expect, it } from 'vitest';
+
+interface ExtractedManifest {
+  schemaVersion: string;
+  sourceCommit: string;
+  extractedAt: string;
+  checksums: {
+    tools: string;
+    systemPrompt: string;
+    compactionPrompt: string;
+  };
+}
+
+const root = resolve(__dirname, '..', '..');
+
+function sha256(filePath: string): string {
+  const content = readFileSync(filePath);
+  return createHash('sha256').update(content).digest('hex');
+}
+
+describe('extracted artifact manifest', () => {
+  it('contains required metadata and valid checksums', () => {
+    const manifestPath = resolve(root, 'src/extracted/manifest.json');
+    const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as ExtractedManifest;
+
+    expect(manifest.schemaVersion).toMatch(/^\d+\.\d+\.\d+$/);
+    expect(manifest.sourceCommit.length).toBeGreaterThan(0);
+    expect(() => new Date(manifest.extractedAt)).not.toThrow();
+
+    const expectedTools = sha256(resolve(root, 'src/extracted/tools.json'));
+    const expectedSystem = sha256(resolve(root, 'src/extracted/prompts/system.md'));
+    const expectedCompaction = sha256(resolve(root, 'src/extracted/prompts/compaction.md'));
+
+    expect(manifest.checksums.tools).toBe(expectedTools);
+    expect(manifest.checksums.systemPrompt).toBe(expectedSystem);
+    expect(manifest.checksums.compactionPrompt).toBe(expectedCompaction);
+  });
+});

package/tests/unit/interfaces-contract.spec.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from 'vitest';
+
+import type { HookCallback, HookContext } from '../../src/interfaces/hooks';
+import type { SessionStore } from '../../src/interfaces/session-store';
+import type { SandboxProvider } from '../../src/interfaces/sandbox-provider';
+import type { ToolProvider } from '../../src/interfaces/tool-provider';
+
+describe('interfaces contract smoke checks', () => {
+  it('tool provider capabilities expose required flags', () => {
+    const provider: ToolProvider = {
+      async bash() {
+        return { success: true, output: '' };
+      },
+      async readFile() {
+        return { success: true, output: '' };
+      },
+      async writeFile() {
+        return { success: true, output: '' };
+      },
+      async editFile() {
+        return { success: true, output: '' };
+      },
+      async glob() {
+        return { success: true, output: '' };
+      },
+      async grep() {
+        return { success: true, output: '' };
+      },
+      capabilities() {
+        return {
+          bash: true,
+          fileSystem: true,
+          webFetch: false,
+          webSearch: false,
+          codeExecution: false,
+          sandboxed: false
+        };
+      }
+    };
+
+    const caps = provider.capabilities();
+    expect(Object.keys(caps).sort()).toEqual([
+      'bash',
+      'codeExecution',
+      'fileSystem',
+      'sandboxed',
+      'webFetch',
+      'webSearch'
+    ]);
+  });
+
+  it('sandbox provider contract supports exec and file io', async () => {
+    const sandbox: SandboxProvider = {
+      async exec() {
+        return { exitCode: 0, stdout: 'ok', stderr: '' };
+      },
+      async readSandboxFile() {
+        return {
+          data: Buffer.from('content', 'utf8'),
+          mimeType: 'text/plain',
+          filename: 'x'
+        };
+      },
+      async writeSandboxFile() {}
+    };
+
+    const result = await sandbox.exec('echo ok');
+    expect(result.exitCode).toBe(0);
+    const file = await sandbox.readSandboxFile('x');
+    expect(Buffer.from(file.data).toString('utf8')).toBe('content');
+  });
+
+  it('session store contract returns null for missing sessions', async () => {
+    const store: SessionStore = {
+      async save() {},
+      async get() {
+        return null;
+      },
+      async list() {
+        return [];
+      },
+      async delete() {}
+    };
+
+    expect(await store.get('missing')).toBeNull();
+    expect(await store.list()).toEqual([]);
+  });
+
+  it('hook callback can deny execution with reason', async () => {
+    const hook: HookCallback = async (context: HookContext) => {
+      if (context.event === 'PreToolUse' && context.toolName === 'Bash') {
+        return { allow: false, reason: 'blocked by policy' };
+      }
+      return { allow: true };
+    };
+
+    const result = await hook({ event: 'PreToolUse', toolName: 'Bash' });
+    expect(result?.allow).toBe(false);
+    expect(result?.reason).toBe('blocked by policy');
+  });
+});

package/tests/unit/structured-messages.spec.ts

@@ -0,0 +1,176 @@
+import { describe, expect, it } from 'vitest';
+
+import type { AgentMessage, ToolCallInfo, ToolResultInfo } from '../../src/agent/types';
+import { LosslessContextStore } from '../../src/loop/context-store';
+
+describe('AgentMessage structured fields', () => {
+  it('assistant message carries toolCalls alongside content', () => {
+    const toolCalls: ToolCallInfo[] = [
+      { toolCallId: 'call-1', toolName: 'Write', args: { path: '/tmp/a.txt', content: 'hello' } },
+      { toolCallId: 'call-2', toolName: 'Bash', args: { command: 'echo ok' } },
+    ];
+
+    const msg: AgentMessage = {
+      role: 'assistant',
+      content: 'I will create a file and run a command.',
+      toolCalls,
+    };
+
+    expect(msg.toolCalls).toHaveLength(2);
+    expect(msg.toolCalls![0]!.toolCallId).toBe('call-1');
+    expect(msg.toolCalls![1]!.toolName).toBe('Bash');
+    // content preserved for display/logging
+    expect(msg.content).toContain('create a file');
+  });
+
+  it('tool message carries toolResults alongside content', () => {
+    const toolResults: ToolResultInfo[] = [
+      { toolCallId: 'call-1', toolName: 'Write', result: 'ok' },
+    ];
+
+    const msg: AgentMessage = {
+      role: 'tool',
+      content: 'Write(/tmp/a.txt): ok',
+      toolResults,
+    };
+
+    expect(msg.toolResults).toHaveLength(1);
+    expect(msg.toolResults![0]!.toolCallId).toBe('call-1');
+    expect(msg.toolResults![0]!.isError).toBeUndefined();
+    // content preserved for display
+    expect(msg.content).toContain('Write');
+  });
+
+  it('tool result with isError flag', () => {
+    const msg: AgentMessage = {
+      role: 'tool',
+      content: 'Bash: ERROR: command not found',
+      toolResults: [{
+        toolCallId: 'call-x',
+        toolName: 'Bash',
+        result: 'command not found',
+        isError: true,
+      }],
+    };
+
+    expect(msg.toolResults![0]!.isError).toBe(true);
+  });
+
+  it('messages without structured fields remain backward-compatible', () => {
+    const plain: AgentMessage = { role: 'user', content: 'hello' };
+    expect(plain.toolCalls).toBeUndefined();
+    expect(plain.toolResults).toBeUndefined();
+  });
+});
+
+describe('LosslessContextStore preserves structured fields', () => {
+  it('getView returns toolCalls and toolResults when under budget', () => {
+    const store = new LosslessContextStore({ maxTokenBudget: 100_000 });
+
+    const messages: AgentMessage[] = [
+      { role: 'user', content: 'write a file' },
+      {
+        role: 'assistant',
+        content: 'Writing file.',
+        toolCalls: [{ toolCallId: 'c1', toolName: 'Write', args: { path: '/tmp/f.txt', content: 'data' } }],
+      },
+      {
+        role: 'tool',
+        content: 'Write(/tmp/f.txt): ok',
+        toolResults: [{ toolCallId: 'c1', toolName: 'Write', result: 'ok' }],
+      },
+      { role: 'assistant', content: 'Done.' },
+    ];
+
+    store.ingest(messages);
+    const view = store.getView();
+
+    expect(view).toHaveLength(4);
+    expect(view[1]!.toolCalls).toHaveLength(1);
+    expect(view[1]!.toolCalls![0]!.toolCallId).toBe('c1');
+    expect(view[2]!.toolResults).toHaveLength(1);
+    expect(view[2]!.toolResults![0]!.toolCallId).toBe('c1');
+  });
+
+  it('cold zone trimming preserves toolResults on tool messages', () => {
+    // Use a very small budget to force trimming
+    const store = new LosslessContextStore({
+      maxTokenBudget: 200,
+      trimThreshold: 0.5,
+      stubThreshold: 50,
+    });
+
+    // Build enough messages to push the early ones into cold zone
+    const messages: AgentMessage[] = [
+      { role: 'user', content: 'initial request' },
+      {
+        role: 'assistant',
+        content: 'step 1',
+        toolCalls: [{ toolCallId: 'old-1', toolName: 'Read', args: { path: '/a' } }],
+      },
+      {
+        role: 'tool',
+        content: 'Read(/a): short',
+        toolResults: [{ toolCallId: 'old-1', toolName: 'Read', result: 'short' }],
+      },
+    ];
+
+    // Add hot-zone messages to push earlier messages into cold zone
+    for (let i = 0; i < 10; i++) {
+      messages.push({ role: 'user', content: `follow up message ${i} with some filler text to consume budget` });
+      messages.push({ role: 'assistant', content: `response ${i} with additional filler text to fill the budget` });
+    }
+
+    store.ingest(messages);
+    const view = store.getView();
+
+    // Find the tool message if it survived cold zone trimming
+    const toolMsg = view.find(m => m.role === 'tool' && m.toolResults);
+    if (toolMsg) {
+      // The key assertion: toolResults survived trimming (spread preserves them)
+      expect(toolMsg.toolResults).toBeDefined();
+      expect(toolMsg.toolResults![0]!.toolCallId).toBe('old-1');
+    }
+    // Either way, verify no tool message lost its toolResults while keeping content
+    for (const m of view) {
+      if (m.role === 'tool' && m.content.startsWith('Read(')) {
+        expect(m.toolResults).toBeDefined();
+      }
+    }
+  });
+
+  it('assistant toolCalls survive in cold zone (pushed as-is)', () => {
+    const store = new LosslessContextStore({
+      maxTokenBudget: 200,
+      trimThreshold: 0.5,
+    });
+
+    const messages: AgentMessage[] = [
+      { role: 'user', content: 'do something' },
+      {
+        role: 'assistant',
+        content: 'calling tool',
+        toolCalls: [{ toolCallId: 'tc-1', toolName: 'Bash', args: { command: 'ls' } }],
+      },
+      {
+        role: 'tool',
+        content: 'Bash("ls"): file1 file2',
+        toolResults: [{ toolCallId: 'tc-1', toolName: 'Bash', result: 'file1 file2' }],
+      },
+    ];
+
+    for (let i = 0; i < 10; i++) {
+      messages.push({ role: 'user', content: `later message ${i} padding text to push budget` });
+      messages.push({ role: 'assistant', content: `later reply ${i} with extra padding text here` });
+    }
+
+    store.ingest(messages);
+    const view = store.getView();
+
+    // Find the assistant message with toolCalls if it survived
+    const assistantWithCalls = view.find(m => m.role === 'assistant' && m.toolCalls);
+    if (assistantWithCalls) {
+      expect(assistantWithCalls.toolCalls![0]!.toolCallId).toBe('tc-1');
+    }
+  });
+});