@bluecopa/harness 1.0.0 → 2.0.0

package/src/loop/lcm-tool-loop.ts

@@ -1,163 +0,0 @@
-import type { AgentAction, AgentLoop, AgentMessage, AgentStreamEvent } from '../agent/types';
-import type { ToolProvider } from '../interfaces/tool-provider';
-import type { ActivityEntry, BridgeConfig } from './bridge';
-import { LosslessContextStore, type ContextStoreConfig } from './context-store';
-import { SandboxBridge } from './bridge';
-
-// ── REPL script marker ──
-// The inner loop signals a REPL script by returning a Bash action whose
-// command starts with this prefix.
-const REPL_MARKER = '##REPL##\n';
-
-export interface LCMToolLoopConfig {
-  /** Inner loop that drives LLM decisions. Default: must be provided. */
-  innerLoop: AgentLoop;
-  /** Tool provider for bridge file I/O and bash execution. */
-  toolProvider: ToolProvider;
-  /** Context store tuning. */
-  contextStore?: ContextStoreConfig;
-  /** Enable REPL mode. Default: true. Set false for context-management-only mode. */
-  enableRepl?: boolean;
-  /** Bridge directory inside the sandbox. Default: /var/run/bridge */
-  bridgeDir?: string;
-  /** Bridge poll interval in ms. Default: 200 */
-  bridgePollIntervalMs?: number;
-  /** Harness callback: fulfill LLM requests from the REPL. */
-  onLlmRequest?: BridgeConfig['onLlmRequest'];
-  /** Harness callback: fulfill web fetch requests from the REPL. */
-  onWebFetchRequest?: BridgeConfig['onWebFetchRequest'];
-  /** Harness callback: fulfill web search requests from the REPL. */
-  onWebSearchRequest?: BridgeConfig['onWebSearchRequest'];
-  /** Harness callback: fulfill ask-user requests from the REPL. */
-  onAskUserRequest?: BridgeConfig['onAskUserRequest'];
-  /** Harness callback: fulfill tell-user requests from the REPL. */
-  onTellUserRequest?: BridgeConfig['onTellUserRequest'];
-  /** Harness callback: receive real-time activity log entries from the REPL. */
-  onActivity?: (entry: ActivityEntry) => void;
-}
-
-/**
- * LCM Tool Loop — Lossless Context Management with optional REPL orchestration.
- *
- * Composes over an inner AgentLoop (e.g. VercelAgentLoop). Two modes:
- *
- * **Standard mode** (default): The inner loop decides one action at a time.
- * LCM provides lossless context trimming so the LLM always sees a coherent,
- * budget-fitting view of the full conversation.
- *
- * **REPL mode**: When the inner loop returns a Bash action with the REPL marker,
- * the loop writes a Python script into the sandbox, injects the bridge module,
- * runs the script, and polls for sub-requests (LLM, web_fetch, ask_user, etc.)
- * that the harness fulfills. The user sees real-time activity via onActivity.
- */
-export class LCMToolLoop implements AgentLoop {
-  private readonly store: LosslessContextStore;
-  private readonly inner: AgentLoop;
-  private readonly tp: ToolProvider;
-  private readonly enableRepl: boolean;
-  private readonly bridgeDir: string;
-  private readonly config: LCMToolLoopConfig;
-  private bridgeSetupDone = false;
-
-  constructor(config: LCMToolLoopConfig) {
-    this.config = config;
-    this.inner = config.innerLoop;
-    this.tp = config.toolProvider;
-    this.enableRepl = config.enableRepl ?? true;
-    this.bridgeDir = config.bridgeDir ?? '/var/run/bridge';
-    this.store = new LosslessContextStore(config.contextStore);
-  }
-
-  async nextAction(messages: AgentMessage[]): Promise<AgentAction> {
-    // 1. Ingest full history into lossless store
-    this.store.ingest(messages);
-
-    // 2. Get trimmed view
-    const view = this.store.getView();
-    const stats = this.store.stats();
-    this.log(`raw=${stats.raw} view=${stats.view} tokens≈${stats.tokensView}`);
-
-    // 3. Delegate to inner loop with the trimmed view
-    const action = await this.inner.nextAction(view);
-
-    // 4. If this is a REPL script, enter REPL mode
-    if (this.enableRepl && this.isReplScript(action)) {
-      return this.executeReplWithBridge(action);
-    }
-
-    // 5. Standard passthrough
-    return action;
-  }
-
-  async *streamAction(messages: AgentMessage[]): AsyncGenerator<AgentStreamEvent> {
-    // Context management — same as nextAction()
-    this.store.ingest(messages);
-    const view = this.store.getView();
-
-    // Delegate streaming to inner loop
-    if (this.inner.streamAction) {
-      yield* this.inner.streamAction(view);
-    }
-    // No fallback to nextAction() here — the agent's stream() handles that
-  }
-
-  /** Expose store stats for external logging. */
-  getStats() {
-    return this.store.stats();
-  }
-
-  // ── REPL detection ──
-
-  private isReplScript(action: AgentAction): boolean {
-    if (action.type !== 'tool' || action.name !== 'Bash') return false;
-    const cmd = String(action.args.command ?? '');
-    return cmd.startsWith(REPL_MARKER);
-  }
-
-  // ── REPL execution ──
-
-  private async executeReplWithBridge(action: AgentAction): Promise<AgentAction> {
-    if (action.type !== 'tool') return action;
-
-    const script = String(action.args.command ?? '').slice(REPL_MARKER.length);
-    this.log('entering REPL mode');
-
-    // Ensure bridge is set up — spread only defined optional fields to
-    // satisfy exactOptionalPropertyTypes.
-    const bridgeCfg: BridgeConfig = { toolProvider: this.tp };
-    bridgeCfg.bridgeDir = this.bridgeDir;
-    if (this.config.bridgePollIntervalMs != null) bridgeCfg.pollIntervalMs = this.config.bridgePollIntervalMs;
-    if (this.config.onLlmRequest) bridgeCfg.onLlmRequest = this.config.onLlmRequest;
-    if (this.config.onWebFetchRequest) bridgeCfg.onWebFetchRequest = this.config.onWebFetchRequest;
-    if (this.config.onWebSearchRequest) bridgeCfg.onWebSearchRequest = this.config.onWebSearchRequest;
-    if (this.config.onAskUserRequest) bridgeCfg.onAskUserRequest = this.config.onAskUserRequest;
-    if (this.config.onTellUserRequest) bridgeCfg.onTellUserRequest = this.config.onTellUserRequest;
-    if (this.config.onActivity) bridgeCfg.onActivity = this.config.onActivity;
-    const bridge = new SandboxBridge(bridgeCfg);
-
-    if (!this.bridgeSetupDone) {
-      await bridge.setup();
-      this.bridgeSetupDone = true;
-    }
-
-    // Write the REPL script into the sandbox
-    const scriptPath = `${this.bridgeDir}/repl_script.py`;
-    await this.tp.writeFile(scriptPath, script);
-
-    // Run and poll
-    const output = await bridge.pollUntilComplete(scriptPath);
-    this.log('REPL mode complete');
-
-    // Return the REPL output as a final action
-    return {
-      type: 'final',
-      content: output || 'REPL script completed with no output.',
-    };
-  }
-
-  private log(msg: string): void {
-    if (typeof process !== 'undefined' && process.stderr) {
-      process.stderr.write(`\x1b[2m◇ ${msg}\x1b[0m\n`);
-    }
-  }
-}

package/src/loop/vercel-agent-loop.ts

@@ -1,279 +0,0 @@
-import { generateText, streamText, stepCountIs, tool, type Tool } from 'ai';
-import { anthropic } from '@ai-sdk/anthropic';
-import { z } from 'zod';
-
-import type { AgentAction, AgentMessage, AgentLoop, AgentStreamEvent, ToolCallAction, ToolBatchAction } from '../agent/types';
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type AnyTool = Tool<any, any>;
-
-// ── Native tool definitions (no execute — harness runs them) ──
-
-/** Built-in tool schemas. Merge with custom tools: `{ ...builtinTools, ...myTools }` */
-export const builtinTools = {
-  Bash: tool({
-    description: 'Run a shell command',
-    inputSchema: z.object({
-      command: z.string().describe('The shell command to run'),
-      cwd: z.string().optional().describe('Working directory'),
-      timeout: z.number().optional().describe('Timeout in milliseconds'),
-    }),
-  }),
-  Read: tool({
-    description: 'Read a file from the filesystem',
-    inputSchema: z.object({
-      path: z.string().describe('Absolute path to the file'),
-    }),
-  }),
-  Write: tool({
-    description: 'Write content to a file (creates or overwrites)',
-    inputSchema: z.object({
-      path: z.string().describe('Absolute path to the file'),
-      content: z.string().describe('Content to write'),
-    }),
-  }),
-  Edit: tool({
-    description: 'Replace text in a file (exact match)',
-    inputSchema: z.object({
-      path: z.string().describe('Absolute path to the file'),
-      old_text: z.string().describe('Exact text to find'),
-      new_text: z.string().describe('Replacement text'),
-    }),
-  }),
-  Glob: tool({
-    description: 'Find files matching a glob pattern',
-    inputSchema: z.object({
-      pattern: z.string().describe('Glob pattern (e.g. **/*.ts)'),
-    }),
-  }),
-  Grep: tool({
-    description: 'Search file contents with a regex pattern',
-    inputSchema: z.object({
-      pattern: z.string().describe('Regex pattern to search for'),
-      path: z.string().optional().describe('Directory or file to search in'),
-    }),
-  }),
-  WebFetch: tool({
-    description: 'Fetch content from a URL',
-    inputSchema: z.object({
-      url: z.string().describe('URL to fetch'),
-      selector: z.string().optional().describe('CSS selector to extract'),
-      maxContentLength: z.number().optional().describe('Max content length'),
-    }),
-  }),
-  WebSearch: tool({
-    description: 'Search the web',
-    inputSchema: z.object({
-      query: z.string().describe('Search query'),
-    }),
-  }),
-  AskUser: tool({
-    description: 'Ask the user a question and wait for their response',
-    inputSchema: z.object({
-      question: z.string().describe('The question to ask'),
-      options: z.array(z.string()).optional().describe('Optional choices'),
-    }),
-  }),
-  TellUser: tool({
-    description: 'Display a message to the user (no response expected)',
-    inputSchema: z.object({
-      message: z.string().describe('The message to display'),
-    }),
-  }),
-  DownloadRawFile: tool({
-    description: 'Download a file from the sandbox to the local machine',
-    inputSchema: z.object({
-      path: z.string().describe('Sandbox path to download'),
-    }),
-  }),
-};
-
-// ── Convert AgentMessage[] to Vercel AI SDK ModelMessage[] ──
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type ModelMessage = any;
-
-function toModelMessages(messages: AgentMessage[]): ModelMessage[] {
-  const out: ModelMessage[] = [];
-
-  for (const msg of messages) {
-    if (msg.role === 'system') {
-      out.push({ role: 'system', content: msg.content });
-      continue;
-    }
-
-    if (msg.role === 'user') {
-      out.push({ role: 'user', content: msg.content });
-      continue;
-    }
-
-    if (msg.role === 'assistant') {
-      if (msg.toolCalls && msg.toolCalls.length > 0) {
-        // Structured assistant message with tool calls
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const parts: any[] = [];
-        if (msg.content) {
-          parts.push({ type: 'text', text: msg.content });
-        }
-        for (const tc of msg.toolCalls) {
-          parts.push({
-            type: 'tool-call',
-            toolCallId: tc.toolCallId,
-            toolName: tc.toolName,
-            input: tc.args,
-          });
-        }
-        out.push({ role: 'assistant', content: parts });
-      } else {
-        out.push({ role: 'assistant', content: msg.content });
-      }
-      continue;
-    }
-
-    if (msg.role === 'tool') {
-      if (msg.toolResults && msg.toolResults.length > 0) {
-        // Structured tool result message
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const parts: any[] = msg.toolResults.map(tr => ({
-          type: 'tool-result',
-          toolCallId: tr.toolCallId,
-          toolName: tr.toolName,
-          output: tr.isError
-            ? { type: 'error-text', value: tr.result }
-            : { type: 'text', value: tr.result },
-        }));
-        out.push({ role: 'tool', content: parts });
-      } else {
-        // Fallback: no structured fields — wrap as user message
-        out.push({ role: 'user', content: `[Tool result]: ${msg.content}` });
-      }
-      continue;
-    }
-  }
-
-  return out;
-}
-
-export interface VercelAgentLoopConfig {
-  model?: string;
-  systemPrompt?: string;
-  apiKey?: string;
-  /** Custom tool definitions. If provided, replaces built-in agentTools for LLM calls. */
-  tools?: Record<string, AnyTool>;
-}
-
-export class VercelAgentLoop implements AgentLoop {
-  private readonly model: string;
-  private readonly systemPrompt: string;
-  private readonly tools: Record<string, AnyTool>;
-  private readonly validToolNames: Set<string>;
-
-  constructor(config: VercelAgentLoopConfig = {}) {
-    this.model = config.model ?? process.env.HARNESS_MODEL ?? 'claude-sonnet-4-5';
-    this.tools = config.tools ?? builtinTools;
-    this.validToolNames = new Set(Object.keys(this.tools));
-    this.systemPrompt =
-      config.systemPrompt ??
-      [
-        'You are an agent that accomplishes tasks using tools.',
-        'You may call multiple independent tools in a single turn.',
-        'Use tools when shell or filesystem access is required.',
-        'For file-generation tasks, prefer one-shot execution with a single Bash command (heredoc/script + run + verify) instead of repeated Write rewrites.',
-        'Avoid rewriting the same file multiple times unless a previous run returned an error that requires a fix.',
-        'When the task is fully complete, respond with a brief text summary (no tool call).',
-      ].join(' ');
-
-    if (config.apiKey) {
-      process.env.ANTHROPIC_API_KEY = config.apiKey;
-    }
-  }
-
-  async nextAction(messages: AgentMessage[]): Promise<AgentAction> {
-    if (!process.env.ANTHROPIC_API_KEY) {
-      throw new Error('ANTHROPIC_API_KEY is required for default VercelAgentLoop');
-    }
-
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const result = await (generateText as any)({
-      model: anthropic(this.model),
-      tools: this.tools,
-      toolChoice: 'auto',
-      system: this.systemPrompt,
-      messages: toModelMessages(messages),
-      stopWhen: stepCountIs(1),
-    });
-
-    // If the model made tool calls, extract them
-    if (result.toolCalls && result.toolCalls.length > 0) {
-      const validCalls: ToolCallAction[] = [];
-      for (const call of result.toolCalls) {
-        const name = call.toolName;
-        if (this.validToolNames.has(name)) {
-          const toolCallId = (call as { toolCallId?: string }).toolCallId;
-          validCalls.push({
-            type: 'tool',
-            name,
-            args: (call as { input: Record<string, unknown> }).input,
-            ...(toolCallId != null ? { toolCallId } : {}),
-          });
-        }
-      }
-
-      if (validCalls.length === 0) {
-        return { type: 'final', content: `Unknown tool: ${result.toolCalls[0]!.toolName}` };
-      }
-
-      // Single call → backward-compatible ToolCallAction
-      if (validCalls.length === 1) {
-        return validCalls[0]!;
-      }
-
-      // Multiple calls → ToolBatchAction
-      return { type: 'tool_batch', calls: validCalls } satisfies ToolBatchAction;
-    }
-
-    // No tool call — model responded with text (task complete)
-    const text = result.text?.trim();
-    return { type: 'final', content: text || 'Done.' };
-  }
-
-  async *streamAction(messages: AgentMessage[]): AsyncGenerator<AgentStreamEvent> {
-    if (!process.env.ANTHROPIC_API_KEY) {
-      throw new Error('ANTHROPIC_API_KEY is required for default VercelAgentLoop');
-    }
-
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const result = (streamText as any)({
-      model: anthropic(this.model),
-      tools: this.tools,
-      toolChoice: 'auto',
-      system: this.systemPrompt,
-      messages: toModelMessages(messages),
-      stopWhen: stepCountIs(1),
-    });
-
-    const toolArgs = new Map<string, string>();
-
-    for await (const part of result.fullStream) {
-      if (part.type === 'text-delta') {
-        yield { type: 'text_delta', text: part.text };
-      }
-      if (part.type === 'tool-input-start') {
-        toolArgs.set(part.id, '');
-      }
-      if (part.type === 'tool-input-delta') {
-        toolArgs.set(part.id, (toolArgs.get(part.id) ?? '') + part.delta);
-      }
-      if (part.type === 'tool-call') {
-        const name = part.toolName;
-        if (this.validToolNames.has(name)) {
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          const p = part as any;
-          const args: Record<string, unknown> = p.args ?? p.input ?? {};
-          const toolCallId: string | undefined = p.toolCallId;
-          yield { type: 'tool_start', name, args, ...(toolCallId != null ? { toolCallId } : {}) };
-        }
-      }
-    }
-  }
-}

package/src/observability/context.ts

@@ -1,17 +0,0 @@
-import type { SpanRecord } from './otel';
-
-export interface CorrelationFields {
-  trace_id: string;
-  span_id: string;
-  run_id?: string | undefined;
-  session_id?: string | undefined;
-}
-
-export function correlationFromSpan(span: SpanRecord, runId?: string, sessionId?: string): CorrelationFields {
-  return {
-    trace_id: span.traceId,
-    span_id: span.spanId,
-    run_id: runId,
-    session_id: sessionId
-  };
-}

package/src/observability/metrics.ts

@@ -1,27 +0,0 @@
-import { HarnessTelemetry } from './otel';
-
-export function recordToolCallDuration(
-  telemetry: HarnessTelemetry | undefined,
-  toolName: string,
-  durationMs: number,
-  success: boolean
-): void {
-  if (!telemetry?.isEnabled()) return;
-  telemetry.counter('tool_calls_total', 1, { tool: toolName, success });
-  telemetry.histogram('tool_call_duration_ms', durationMs, { tool: toolName, success });
-}
-
-export function recordAgentStep(telemetry: HarnessTelemetry | undefined): void {
-  if (!telemetry?.isEnabled()) return;
-  telemetry.counter('agent_steps_total', 1);
-}
-
-export function recordCompaction(telemetry: HarnessTelemetry | undefined): void {
-  if (!telemetry?.isEnabled()) return;
-  telemetry.counter('compactions_total', 1);
-}
-
-export function recordAgentError(telemetry: HarnessTelemetry | undefined): void {
-  if (!telemetry?.isEnabled()) return;
-  telemetry.counter('agent_errors_total', 1);
-}

package/src/observability/otel.ts

@@ -1,105 +0,0 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { randomUUID } from 'node:crypto';
-
-export interface SpanRecord {
-  traceId: string;
-  spanId: string;
-  parentSpanId?: string | undefined;
-  name: string;
-  attributes: Record<string, string | number | boolean>;
-  startTime: number;
-  endTime: number;
-}
-
-export interface MetricRecord {
-  name: string;
-  value: number;
-  type: 'counter' | 'histogram';
-  attributes: Record<string, string | number | boolean>;
-}
-
-export interface SpanHandle {
-  traceId: string;
-  spanId: string;
-  end(attributes?: Record<string, string | number | boolean>): void;
-}
-
-export class HarnessTelemetry {
-  private readonly context = new AsyncLocalStorage<{ traceId: string; spanId: string }>();
-  private readonly spans: SpanRecord[] = [];
-  private readonly metrics: MetricRecord[] = [];
-
-  constructor(private readonly enabled = true) {}
-
-  isEnabled(): boolean {
-    return this.enabled;
-  }
-
-  startSpan(name: string, attributes: Record<string, string | number | boolean> = {}): SpanHandle {
-    const now = Date.now();
-    const parent = this.context.getStore();
-    const traceId = parent?.traceId ?? randomUUID().replace(/-/g, '');
-    const spanId = randomUUID().replace(/-/g, '').slice(0, 16);
-
-    const record: SpanRecord = {
-      traceId,
-      spanId,
-      parentSpanId: parent?.spanId,
-      name,
-      attributes: { ...attributes },
-      startTime: now,
-      endTime: now
-    };
-
-    this.spans.push(record);
-
-    return {
-      traceId,
-      spanId,
-      end: (extra = {}) => {
-        record.endTime = Date.now();
-        record.attributes = { ...record.attributes, ...extra };
-      }
-    };
-  }
-
-  async withSpan<T>(
-    name: string,
-    attributes: Record<string, string | number | boolean>,
-    fn: () => Promise<T>
-  ): Promise<T> {
-    if (!this.enabled) {
-      return fn();
-    }
-
-    const span = this.startSpan(name, attributes);
-    return this.context.run({ traceId: span.traceId, spanId: span.spanId }, async () => {
-      try {
-        const result = await fn();
-        span.end({ success: true });
-        return result;
-      } catch (error) {
-        span.end({ success: false, error: error instanceof Error ? error.message : 'unknown' });
-        throw error;
-      }
-    });
-  }
-
-  counter(name: string, value = 1, attributes: Record<string, string | number | boolean> = {}): void {
-    if (!this.enabled) return;
-    this.metrics.push({ name, value, type: 'counter', attributes });
-  }
-
-  histogram(name: string, value: number, attributes: Record<string, string | number | boolean> = {}): void {
-    if (!this.enabled) return;
-    this.metrics.push({ name, value, type: 'histogram', attributes });
-  }
-
-  getSpans(): SpanRecord[] {
-    return [...this.spans];
-  }
-
-  getMetrics(): MetricRecord[] {
-    return [...this.metrics];
-  }
-}

package/src/observability/tracing.ts

@@ -1,13 +0,0 @@
-import { HarnessTelemetry } from './otel';
-
-export async function traceStep<T>(
-  telemetry: HarnessTelemetry | undefined,
-  name: string,
-  attributes: Record<string, string | number | boolean>,
-  fn: () => Promise<T>
-): Promise<T> {
-  if (!telemetry?.isEnabled()) {
-    return fn();
-  }
-  return telemetry.withSpan(name, attributes, fn);
-}

package/src/optimization/agent-evaluator.ts

@@ -1,40 +0,0 @@
-import { randomUUID } from 'node:crypto';
-
-import type { AgentRunResult } from '../agent/create-agent';
-import { TraceCollector } from './trace-collector';
-
-export interface EvaluationTask {
-  prompt: string;
-  expectedContains?: string;
-}
-
-export interface EvaluationResult {
-  score: number;
-  output: string;
-  trace: ReturnType<TraceCollector['toExecutionTrace']>;
-}
-
-export type AgentRunner = (prompt: string) => Promise<AgentRunResult>;
-
-export class AgentEvaluator {
-  constructor(private readonly runAgent: AgentRunner) {}
-
-  async evaluate(task: EvaluationTask): Promise<EvaluationResult> {
-    const trace = new TraceCollector(randomUUID());
-    trace.add('task.start', { prompt: task.prompt });
-
-    const result = await this.runAgent(task.prompt);
-    trace.add('task.finish', { output: result.output, steps: result.steps });
-
-    let score = 1;
-    if (task.expectedContains && !result.output.includes(task.expectedContains)) {
-      score = 0;
-    }
-
-    return {
-      score,
-      output: result.output,
-      trace: trace.toExecutionTrace()
-    };
-  }
-}

package/src/optimization/config-serializer.ts

@@ -1,16 +0,0 @@
-import { createHash } from 'node:crypto';
-
-export class ConfigSerializer {
-  static serialize(config: Record<string, unknown>): string {
-    return JSON.stringify(config, Object.keys(config).sort(), 2);
-  }
-
-  static deserialize(serialized: string): Record<string, unknown> {
-    return JSON.parse(serialized) as Record<string, unknown>;
-  }
-
-  static hash(config: Record<string, unknown>): string {
-    const serialized = ConfigSerializer.serialize(config);
-    return createHash('sha256').update(serialized).digest('hex');
-  }
-}