@bluecopa/harness 1.0.0 → 2.0.0

package/tests/integration/agent-skill-default-from-sandbox.spec.ts

@@ -1,66 +0,0 @@
-import { mkdtemp, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import { createAgent } from '../../src/agent/create-agent';
-import type { SandboxFileBlob, SandboxProvider } from '../../src/interfaces/sandbox-provider';
-import { LocalToolProvider } from '../../src/providers/local-tool-provider';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
-  tempDirs.length = 0;
-});
-
-class NoopSandbox implements SandboxProvider {
-  async exec() {
-    return { exitCode: 0, stdout: '', stderr: '' };
-  }
-
-  async readSandboxFile(): Promise<SandboxFileBlob> {
-    return { data: new Uint8Array() };
-  }
-
-  async writeSandboxFile(): Promise<void> {}
-}
-
-describe('agent skill defaults from sandbox provider', () => {
-  it('auto-enables SkillManager behavior when sandboxProvider is provided', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-agent-skill-default-'));
-    tempDirs.push(dir);
-
-    const provider = new LocalToolProvider(dir);
-    const skillPath = join(dir, 'sample-skill.md');
-    const indexPath = join(dir, 'skills-index.json');
-
-    await writeFile(
-      skillPath,
-      ['---', 'name: docx-skill', 'description: creates docx files', '---', '', 'Use this skill for docx creation.'].join('\n'),
-      'utf8'
-    );
-    await writeFile(
-      indexPath,
-      JSON.stringify([{ name: 'docx-skill', description: 'creates docx files', path: skillPath }]),
-      'utf8'
-    );
-
-    const agent = createAgent({
-      toolProvider: provider,
-      sandboxProvider: new NoopSandbox(),
-      skillIndexPath: indexPath,
-      async nextAction(messages) {
-        return {
-          type: 'final',
-          content: messages[0]?.content ?? ''
-        };
-      }
-    });
-
-    const result = await agent.run('please use docx-skill');
-    expect(result.output).toContain('Skill selected: docx-skill');
-    expect(result.output).toContain('Use this skill for docx creation.');
-  });
-});
-

package/tests/integration/concurrency-single-turn.spec.ts

@@ -1,35 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { SingleFlightStepExecutor } from '../../src/agent/step-executor';
-
-describe('single-flight step executor', () => {
-  it('serializes concurrent tasks in submission order', async () => {
-    const executor = new SingleFlightStepExecutor();
-    const calls: string[] = [];
-
-    const a = executor.run(async () => {
-      calls.push('a-start');
-      await new Promise((resolve) => setTimeout(resolve, 10));
-      calls.push('a-end');
-      return 'a';
-    });
-
-    const b = executor.run(async () => {
-      calls.push('b-start');
-      await new Promise((resolve) => setTimeout(resolve, 1));
-      calls.push('b-end');
-      return 'b';
-    });
-
-    const c = executor.run(async () => {
-      calls.push('c-start');
-      calls.push('c-end');
-      return 'c';
-    });
-
-    const result = await Promise.all([a, b, c]);
-
-    expect(result).toEqual(['a', 'b', 'c']);
-    expect(calls).toEqual(['a-start', 'a-end', 'b-start', 'b-end', 'c-start', 'c-end']);
-  });
-});

package/tests/integration/otel-metrics-emission.spec.ts

@@ -1,62 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { createAgent } from '../../src/agent/create-agent';
-import { HookRunner } from '../../src/hooks/hook-runner';
-import { withHooks } from '../../src/hooks/tool-wrappers';
-import { HarnessTelemetry } from '../../src/observability/otel';
-import { PermissionManager } from '../../src/permissions/permission-manager';
-import { LocalToolProvider } from '../../src/providers/local-tool-provider';
-
-describe('otel metrics emission', () => {
-  it('emits step and tool metrics when telemetry is enabled', async () => {
-    const telemetry = new HarnessTelemetry(true);
-    const provider = withHooks(
-      new LocalToolProvider(),
-      new HookRunner(),
-      new PermissionManager('allow_all'),
-      telemetry
-    );
-
-    let turn = 0;
-    const agent = createAgent({
-      toolProvider: provider,
-      telemetry,
-      async nextAction() {
-        turn += 1;
-        if (turn === 1) {
-          return { type: 'tool', name: 'Bash', args: { command: 'node -e "console.log(7)"' } };
-        }
-        return { type: 'final', content: 'done' };
-      }
-    });
-
-    await agent.run('metrics please');
-
-    const metrics = telemetry.getMetrics();
-    expect(metrics.some((m) => m.name === 'agent_steps_total')).toBe(true);
-    expect(metrics.some((m) => m.name === 'tool_calls_total')).toBe(true);
-    expect(metrics.some((m) => m.name === 'tool_call_duration_ms')).toBe(true);
-  });
-
-  it('emits nothing when telemetry is disabled', async () => {
-    const telemetry = new HarnessTelemetry(false);
-    const provider = withHooks(
-      new LocalToolProvider(),
-      new HookRunner(),
-      new PermissionManager('allow_all'),
-      telemetry
-    );
-
-    const agent = createAgent({
-      toolProvider: provider,
-      telemetry,
-      async nextAction() {
-        return { type: 'final', content: 'done' };
-      }
-    });
-
-    await agent.run('disabled please');
-    expect(telemetry.getSpans()).toEqual([]);
-    expect(telemetry.getMetrics()).toEqual([]);
-  });
-});

package/tests/integration/otel-trace-propagation.spec.ts

@@ -1,48 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { createAgent } from '../../src/agent/create-agent';
-import { HookRunner } from '../../src/hooks/hook-runner';
-import { withHooks } from '../../src/hooks/tool-wrappers';
-import { PermissionManager } from '../../src/permissions/permission-manager';
-import { LocalToolProvider } from '../../src/providers/local-tool-provider';
-import { HarnessTelemetry } from '../../src/observability/otel';
-
-describe('otel trace propagation', () => {
-  it('emits parent-child spans for run, step, and tool calls', async () => {
-    const telemetry = new HarnessTelemetry(true);
-    const wrappedProvider = withHooks(
-      new LocalToolProvider(),
-      new HookRunner(),
-      new PermissionManager('allow_all'),
-      telemetry
-    );
-
-    let turn = 0;
-    const agent = createAgent({
-      toolProvider: wrappedProvider,
-      telemetry,
-      async nextAction() {
-        turn += 1;
-        if (turn === 1) {
-          return { type: 'tool', name: 'Bash', args: { command: 'node -e "console.log(1)"' } };
-        }
-        return { type: 'final', content: 'done' };
-      }
-    });
-
-    await agent.run('trace please');
-
-    const spans = telemetry.getSpans();
-    const run = spans.find((span) => span.name === 'agent.run');
-    const step = spans.find((span) => span.name === 'agent.step');
-    const tool = spans.find((span) => span.name === 'tool.call');
-
-    expect(run).toBeDefined();
-    expect(step).toBeDefined();
-    expect(tool).toBeDefined();
-    expect(step?.parentSpanId).toBe(run?.spanId);
-    expect(tool?.parentSpanId).toBe(run?.spanId);
-    expect(run?.traceId).toBe(step?.traceId);
-    expect(step?.traceId).toBe(tool?.traceId);
-  });
-});

package/tests/integration/parity-benchmark.spec.ts

@@ -1,45 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { createAgent } from '../../src/agent/create-agent';
-import { LocalToolProvider } from '../../src/providers/local-tool-provider';
-import { AgentEvaluator } from '../../src/optimization/agent-evaluator';
-import { OptimizationRunner } from '../../src/optimization/optimization-runner';
-
-describe('parity benchmark foundation', () => {
-  it('evaluates a candidate and produces scored trace output', async () => {
-    const agent = createAgent({
-      toolProvider: new LocalToolProvider(),
-      async nextAction() {
-        return { type: 'final', content: '42' };
-      }
-    });
-
-    const evaluator = new AgentEvaluator((prompt) => agent.run(prompt));
-    const result = await evaluator.evaluate({ prompt: 'return 42', expectedContains: '42' });
-
-    expect(result.score).toBe(1);
-    expect(result.trace.events.length).toBe(2);
-    expect(result.output).toBe('42');
-  });
-
-  it('selects the best candidate by average score', async () => {
-    const runner = new OptimizationRunner();
-
-    const result = await runner.run(
-      [
-        {
-          id: 'good',
-          run: async () => ({ output: '42', steps: 1, messages: [] })
-        },
-        {
-          id: 'bad',
-          run: async () => ({ output: '0', steps: 1, messages: [] })
-        }
-      ],
-      [{ prompt: 'return 42', expectedContains: '42' }]
-    );
-
-    expect(result.bestCandidateId).toBe('good');
-    expect(result.bestScore).toBe(1);
-  });
-});

package/tests/integration/provider-local-smoke.spec.ts

@@ -1,63 +0,0 @@
-import { mkdtemp, readFile as readFsFile, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import { CompositeToolProvider } from '../../src/providers/composite-tool-provider';
-import { LocalToolProvider } from '../../src/providers/local-tool-provider';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(
-    tempDirs.map(async (dir) => {
-      await rm(dir, { recursive: true, force: true });
-    })
-  );
-  tempDirs.length = 0;
-});
-
-describe('local provider smoke', () => {
-  it('supports write, read, edit, glob, grep, and bash', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-local-'));
-    tempDirs.push(dir);
-
-    const provider = new LocalToolProvider(dir);
-
-    const writeResult = await provider.writeFile('src/app.ts', 'const x = 1;\n');
-    expect(writeResult.success).toBe(true);
-
-    const readResult = await provider.readFile('src/app.ts');
-    expect(readResult.output).toContain('const x = 1;');
-
-    const editResult = await provider.editFile('src/app.ts', 'const x = 1;', 'const x = 2;');
-    expect(editResult.success).toBe(true);
-
-    const globResult = await provider.glob('src/*.ts');
-    expect(globResult.output).toContain('src/app.ts');
-
-    const grepResult = await provider.grep('x = 2', 'src');
-    expect(grepResult.output).toContain('src/app.ts:1:const x = 2;');
-
-    const bashResult = await provider.bash('node -e "console.log(42)"');
-    expect(bashResult.success).toBe(true);
-    expect(bashResult.output).toContain('42');
-
-    const fileContent = await readFsFile(join(dir, 'src/app.ts'), 'utf8');
-    expect(fileContent).toContain('const x = 2;');
-  });
-
-  it('composite provider routes capabilities correctly', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-composite-'));
-    tempDirs.push(dir);
-
-    const local = new LocalToolProvider(dir);
-    const composite = new CompositeToolProvider([local]);
-
-    expect(composite.capabilities().bash).toBe(true);
-
-    await composite.writeFile('file.txt', 'hello');
-    const out = await composite.readFile('file.txt');
-    expect(out.output).toBe('hello');
-  });
-});

package/tests/integration/session-resume.spec.ts

@@ -1,30 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import { FileSessionStore } from '../../src/sessions/file-session-store';
-import { SessionManager } from '../../src/sessions/session-manager';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
-  tempDirs.length = 0;
-});
-
-describe('session resume', () => {
-  it('creates, checkpoints, and resumes a session', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-session-'));
-    tempDirs.push(dir);
-
-    const manager = new SessionManager(new FileSessionStore(dir));
-    const session = await manager.create('build harness');
-    await manager.appendMessage(session.id, { role: 'assistant', content: 'starting' });
-
-    const resumed = await manager.resume(session.id);
-    expect(resumed).not.toBeNull();
-    expect(resumed?.messages.length).toBe(2);
-    expect(resumed?.messages[1]?.content).toBe('starting');
-  });
-});

package/tests/integration/skill-install-rollback.spec.ts

@@ -1,64 +0,0 @@
-import { mkdtemp, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import type { SandboxProvider } from '../../src/interfaces/sandbox-provider';
-import { SkillManager } from '../../src/skills/skill-manager';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
-  tempDirs.length = 0;
-});
-
-class FailingSandbox implements SandboxProvider {
-  async exec() {
-    return { exitCode: 1, stdout: '', stderr: 'install failed' };
-  }
-
-  async readSandboxFile() {
-    return { data: new Uint8Array() };
-  }
-
-  async writeSandboxFile() {}
-}
-
-describe('skill install rollback/degraded state', () => {
-  it('marks skill as degraded on dependency install failure', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-skill-install-'));
-    tempDirs.push(dir);
-
-    const skillPath = join(dir, 'skill.md');
-    const indexPath = join(dir, 'skills.json');
-
-    await writeFile(
-      skillPath,
-      [
-        '---',
-        'name: failing',
-        'description: failing skill',
-        'python_deps: totally-broken-package',
-        '---',
-        '',
-        'instructions'
-      ].join('\n'),
-      'utf8'
-    );
-    await writeFile(
-      indexPath,
-      JSON.stringify([{ name: 'failing', description: 'failing skill', path: skillPath }]),
-      'utf8'
-    );
-
-    const manager = new SkillManager(new FailingSandbox());
-    await manager.discover(indexPath);
-
-    const invoked = await manager.invoke('failing');
-    expect(invoked.instructions).toContain('instructions');
-
-    await expect(manager.installDependencies('failing')).rejects.toThrow('install failed');
-    expect(manager.getInstallState('failing')).toBe('degraded');
-  });
-});

package/tests/integration/skill-sandbox-file-blob.spec.ts

@@ -1,54 +0,0 @@
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import type { SandboxFileBlob, SandboxProvider } from '../../src/interfaces/sandbox-provider';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
-  tempDirs.length = 0;
-});
-
-class BlobSandbox implements SandboxProvider {
-  constructor(private readonly root: string) {}
-
-  async exec() {
-    return { exitCode: 0, stdout: '', stderr: '' };
-  }
-
-  async readSandboxFile(path: string): Promise<SandboxFileBlob> {
-    const bytes = await readFile(join(this.root, path));
-    return {
-      data: new Uint8Array(bytes),
-      mimeType: 'application/octet-stream',
-      filename: path.split('/').pop() ?? path,
-    };
-  }
-
-  async writeSandboxFile(path: string, content: SandboxFileBlob): Promise<void> {
-    await writeFile(join(this.root, path), Buffer.from(content.data));
-  }
-}
-
-describe('skill sandbox file blob contract', () => {
-  it('roundtrips binary artifact bytes with mime metadata', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-skill-blob-'));
-    tempDirs.push(dir);
-
-    const sandbox = new BlobSandbox(dir);
-    const payload = new Uint8Array([0, 255, 10, 20, 30, 40, 50, 60]);
-    await sandbox.writeSandboxFile('artifact.bin', {
-      data: payload,
-      mimeType: 'application/octet-stream',
-      filename: 'artifact.bin'
-    });
-
-    const result = await sandbox.readSandboxFile('artifact.bin');
-    expect(result.mimeType).toBe('application/octet-stream');
-    expect(result.filename).toBe('artifact.bin');
-    expect(Array.from(result.data)).toEqual(Array.from(payload));
-  });
-});

package/tests/integration/skills-progressive-disclosure.spec.ts

@@ -1,61 +0,0 @@
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, describe, expect, it } from 'vitest';
-
-import type { SandboxProvider } from '../../src/interfaces/sandbox-provider';
-import { SkillManager } from '../../src/skills/skill-manager';
-
-const tempDirs: string[] = [];
-
-afterEach(async () => {
-  await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
-  tempDirs.length = 0;
-});
-
-class MockSandbox implements SandboxProvider {
-  async exec() {
-    return { exitCode: 0, stdout: '', stderr: '' };
-  }
-
-  async readSandboxFile(path: string) {
-    const raw = await readFile(path, 'utf8');
-    return {
-      data: Buffer.from(raw, 'utf8')
-    };
-  }
-
-  async writeSandboxFile(path: string, content: { data: Uint8Array }) {
-    await writeFile(path, Buffer.from(content.data).toString('utf8'), 'utf8');
-  }
-}
-
-describe('skill progressive disclosure', () => {
-  it('keeps summary available and loads instructions only on invoke', async () => {
-    const dir = await mkdtemp(join(tmpdir(), 'harness-skills-'));
-    tempDirs.push(dir);
-
-    const skillPath = join(dir, 'skill.md');
-    const indexPath = join(dir, 'skills.json');
-
-    await writeFile(
-      skillPath,
-      ['---', 'name: sample', 'description: sample skill', '---', '', 'full instruction text'].join('\n'),
-      'utf8'
-    );
-    await writeFile(
-      indexPath,
-      JSON.stringify([{ name: 'sample', description: 'sample skill', path: skillPath }]),
-      'utf8'
-    );
-
-    const manager = new SkillManager(new MockSandbox());
-    const summaries = await manager.discover(indexPath);
-
-    expect(summaries[0]?.name).toBe('sample');
-    expect(manager.getSkillSummaryForPrompt()[0]?.description).toBe('sample skill');
-
-    const invoked = await manager.invoke('sample');
-    expect(invoked.instructions).toContain('full instruction text');
-  });
-});

package/tests/integration/streaming-compaction-boundary.spec.ts

@@ -1,43 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { LLMCompactionStrategy } from '../../src/context/llm-compaction-strategy';
-import { prepareStep } from '../../src/context/prepare-step';
-import { TokenTracker } from '../../src/context/token-tracker';
-
-describe('streaming compaction boundary', () => {
-  it('defers compaction while streaming and compacts before next non-streaming step', async () => {
-    const tokenTracker = new TokenTracker({ estimator: (text) => text.length });
-    const strategy = new LLMCompactionStrategy(0.5);
-
-    const messages = [
-      { role: 'user' as const, content: 'hello there' },
-      { role: 'assistant' as const, content: 'response one' },
-      { role: 'user' as const, content: 'please continue with extra detail' }
-    ];
-
-    const streaming = await prepareStep({
-      messages,
-      maxTokens: 10,
-      isStreaming: true,
-      state: {},
-      tokenTracker,
-      compactionStrategy: strategy
-    });
-
-    expect(streaming.state.pendingCompaction).toBe(true);
-    expect(streaming.messages).toEqual(messages);
-
-    const nextStep = await prepareStep({
-      messages: streaming.messages,
-      maxTokens: 10,
-      isStreaming: false,
-      state: streaming.state,
-      tokenTracker,
-      compactionStrategy: strategy
-    });
-
-    expect(nextStep.state.pendingCompaction).toBe(false);
-    expect(nextStep.messages.length).toBeLessThan(messages.length + 1);
-    expect(nextStep.messages[0]?.content).toContain('Compaction summary');
-  });
-});