agent-relay 2.0.29 → 2.0.32

package/packages/benchmark/dist/types.d.ts

@@ -0,0 +1,178 @@
+/**
+ * Benchmark Types
+ *
+ * Type definitions for the agent swarm performance benchmark system.
+ */
+/**
+ * Configuration type for benchmark runs
+ */
+export type ConfigurationType = 'single' | 'subagent' | 'swarm';
+/**
+ * Task complexity level
+ */
+export type TaskComplexity = 'low' | 'medium' | 'high';
+/**
+ * A benchmark task definition
+ */
+export interface Task {
+    /** Unique task identifier */
+    id: string;
+    /** Human-readable task description */
+    description: string;
+    /** Files the task operates on */
+    files: string[];
+    /** Success criteria for the task */
+    expectedOutcome: string;
+    /** Optional Harbor-style success criteria key for compatibility */
+    success_criteria?: string;
+    /** Task complexity level */
+    complexity: TaskComplexity;
+    /** Optional timeout in milliseconds (default: 300000 = 5 min) */
+    timeoutMs?: number;
+    /** Optional tags for categorization */
+    tags?: string[];
+}
+/**
+ * Result of a single benchmark run
+ */
+export interface RunResult {
+    /** Task identifier */
+    taskId: string;
+    /** Configuration used for this run */
+    configuration: ConfigurationType;
+    /** Total time from start to completion in milliseconds */
+    totalTimeMs: number;
+    /** Time to first agent action in milliseconds */
+    timeToFirstActionMs: number;
+    /** Total number of inter-agent messages */
+    messageCount: number;
+    /** Average message latency in milliseconds */
+    avgLatencyMs: number;
+    /** P50 latency in milliseconds */
+    latencyP50Ms: number;
+    /** P99 latency in milliseconds */
+    latencyP99Ms: number;
+    /** Number of coordination rounds */
+    coordinationRounds: number;
+    /** Number of agents used */
+    agentCount: number;
+    /** Total tokens consumed (if available) */
+    totalTokensUsed: number;
+    /** Peak memory usage in MB */
+    peakMemoryMb: number;
+    /** Whether the task completed successfully */
+    success: boolean;
+    /** Completion rate (0-1) for partial success */
+    completionRate: number;
+    /** Error messages if any */
+    errors: string[];
+    /** Timestamp when the run started */
+    startedAt: number;
+    /** Timestamp when the run completed */
+    completedAt: number;
+}
+/**
+ * Comparison result across all configurations
+ */
+export interface ComparisonResult {
+    /** Task identifier */
+    taskId: string;
+    /** Results for each configuration */
+    results: Map<ConfigurationType, RunResult>;
+    /** The winning configuration based on scoring */
+    winner: ConfigurationType;
+    /** Score breakdown for each configuration */
+    scores: Map<ConfigurationType, ScoreBreakdown>;
+}
+/**
+ * Score breakdown for a configuration
+ */
+export interface ScoreBreakdown {
+    /** Total score (0-100) */
+    total: number;
+    /** Success component (0-50) */
+    successScore: number;
+    /** Time efficiency component (0-30) */
+    timeScore: number;
+    /** Resource efficiency component (0-20) */
+    efficiencyScore: number;
+}
+/**
+ * Benchmark configuration options
+ */
+export interface BenchmarkConfig {
+    /** Which configurations to run */
+    configurations: ConfigurationType[];
+    /** CLI to use for agents (default: 'claude') */
+    cli: string;
+    /** Working directory for tasks */
+    cwd?: string;
+    /** Suppress console output */
+    quiet: boolean;
+    /** Cool-down time between runs in milliseconds */
+    cooldownMs: number;
+    /** Maximum concurrent agents for swarm */
+    maxSwarmSize: number;
+    /** Custom socket path for relay */
+    socketPath?: string;
+}
+/**
+ * Default benchmark configuration
+ */
+export declare const DEFAULT_BENCHMARK_CONFIG: BenchmarkConfig;
+/**
+ * Metrics collected during a run
+ */
+export interface RunMetrics {
+    /** Number of messages sent */
+    messages: number;
+    /** Message latencies in milliseconds */
+    latencies: number[];
+    /** Run start timestamp */
+    startTime: number;
+    /** Spawned agent names */
+    spawnedAgents: string[];
+    /** Error events */
+    errors: string[];
+}
+/**
+ * Task dataset definition
+ */
+export interface TaskDataset {
+    /** Dataset name */
+    name: string;
+    /** Dataset description */
+    description?: string;
+    /** Version identifier */
+    version?: string;
+    /** Tasks in the dataset */
+    tasks: Task[];
+}
+/**
+ * Harbor-compatible evaluation input
+ */
+export interface HarborTaskInput {
+    id: string;
+    description: string;
+    files?: string[];
+    success_criteria?: string;
+    complexity?: TaskComplexity;
+    agents_required?: number;
+    [key: string]: unknown;
+}
+/**
+ * Harbor-compatible evaluation output
+ */
+export interface HarborEvaluationOutput {
+    task_id: string;
+    configurations: Record<ConfigurationType, RunResult>;
+    winner: ConfigurationType;
+    scores: Record<ConfigurationType, ScoreBreakdown>;
+    metadata: {
+        benchmark_version: string;
+        started_at: number;
+        completed_at: number;
+        total_duration_ms: number;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/packages/benchmark/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,eAAe,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4BAA4B;IAC5B,UAAU,EAAE,cAAc,CAAC;IAC3B,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,aAAa,EAAE,iBAAiB,CAAC;IAGjC,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAG5B,2CAA2C;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,8CAA8C;IAC9C,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAG3B,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IAGrB,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,EAAE,CAAC;IAGjB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,OAAO,EAAE,GAAG,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IAC3C,iDAAiD;IACjD,MAAM,EAAE,iBAAiB,CAAC;IAC1B,6CAA6C;IAC7C,MAAM,EAAE,GAAG,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kCAAkC;IAClC,cAAc,EAAE,iBAAiB,EAAE,CAAC;IACpC,gDAAgD;IAChD,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,KAAK,EAAE,OAAO,CAAC;IACf,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAMtC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,mBAAmB;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IACrD,MAAM,EAAE,iBAAiB,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;IAClD,QAAQ,EAAE;QACR,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH"}

package/packages/benchmark/dist/types.js

@@ -0,0 +1,16 @@
+/**
+ * Benchmark Types
+ *
+ * Type definitions for the agent swarm performance benchmark system.
+ */
+/**
+ * Default benchmark configuration
+ */
+export const DEFAULT_BENCHMARK_CONFIG = {
+    configurations: ['single', 'subagent', 'swarm'],
+    cli: 'claude',
+    quiet: false,
+    cooldownMs: 5000,
+    maxSwarmSize: 10,
+};
+//# sourceMappingURL=types.js.map

package/packages/benchmark/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAoIH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC;IAC/C,GAAG,EAAE,QAAQ;IACb,KAAK,EAAE,KAAK;IACZ,UAAU,EAAE,IAAI;IAChB,YAAY,EAAE,EAAE;CACjB,CAAC"}

package/packages/benchmark/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "@agent-relay/benchmark",
+  "version": "2.0.32",
+  "description": "Performance benchmarking for agent swarms, sub-agents, and single agents using Harbor",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./runners": {
+      "types": "./dist/runners/index.d.ts",
+      "import": "./dist/runners/index.js"
+    },
+    "./harbor": {
+      "types": "./dist/harbor.d.ts",
+      "import": "./dist/harbor.js"
+    }
+  },
+  "bin": {
+    "relay-benchmark": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "datasets",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "benchmark": "node dist/cli.js",
+    "benchmark:single": "node dist/cli.js --config single",
+    "benchmark:subagent": "node dist/cli.js --config subagent",
+    "benchmark:swarm": "node dist/cli.js --config swarm",
+    "benchmark:compare": "node dist/cli.js --config all"
+  },
+  "keywords": [
+    "agent-relay",
+    "benchmark",
+    "performance",
+    "swarm",
+    "harbor",
+    "ai-agents"
+  ],
+  "author": "Agent Workforce",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/AgentWorkforce/relay.git",
+    "directory": "packages/benchmark"
+  },
+  "dependencies": {
+    "@agent-relay/sdk": "2.0.32",
+    "@agent-relay/protocol": "2.0.32",
+    "@agent-relay/spawner": "2.0.32",
+    "commander": "^12.1.0",
+    "yaml": "^2.3.4"
+  },
+  "devDependencies": {
+    "@agent-relay/daemon": "2.0.29",
+    "@types/node": "^22.19.3",
+    "typescript": "^5.9.3",
+    "vitest": "^3.0.0"
+  },
+  "peerDependencies": {
+    "@agent-relay/daemon": ">=2.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@agent-relay/daemon": {
+      "optional": true
+    }
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/packages/benchmark/src/benchmark.ts

@@ -0,0 +1,298 @@
+/**
+ * Comparison Benchmark
+ *
+ * Main orchestrator for running comparison benchmarks across configurations.
+ */
+
+import type {
+  ConfigurationType,
+  Task,
+  RunResult,
+  ComparisonResult,
+  ScoreBreakdown,
+  BenchmarkConfig,
+} from './types.js';
+import { DEFAULT_BENCHMARK_CONFIG } from './types.js';
+import {
+  ConfigurationRunner,
+  SingleAgentRunner,
+  SubAgentRunner,
+  SwarmRunner,
+} from './runners/index.js';
+
+/**
+ * Main benchmark orchestrator
+ */
+export class ComparisonBenchmark {
+  private config: BenchmarkConfig;
+  private runners: Map<ConfigurationType, ConfigurationRunner>;
+
+  constructor(config: Partial<BenchmarkConfig> = {}) {
+    this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
+
+    // Initialize runners for configured configurations
+    this.runners = new Map();
+    for (const configType of this.config.configurations) {
+      this.runners.set(configType, this.createRunner(configType));
+    }
+  }
+
+  /**
+   * Create a runner for a configuration type
+   */
+  private createRunner(type: ConfigurationType): ConfigurationRunner {
+    switch (type) {
+      case 'single':
+        return new SingleAgentRunner(this.config);
+      case 'subagent':
+        return new SubAgentRunner(this.config);
+      case 'swarm':
+        return new SwarmRunner(this.config);
+      default:
+        throw new Error(`Unknown configuration type: ${type}`);
+    }
+  }
+
+  /**
+   * Run a comparison across all configured configurations
+   */
+  async runComparison(task: Task): Promise<ComparisonResult> {
+    const results = new Map<ConfigurationType, RunResult>();
+    const scores = new Map<ConfigurationType, ScoreBreakdown>();
+
+    for (const [configType, runner] of this.runners) {
+      if (!this.config.quiet) {
+        console.log(`\n=== Running ${configType} configuration ===`);
+        console.log(`Task: ${task.id}`);
+      }
+
+      try {
+        await runner.setup();
+        const result = await runner.run(task);
+        await runner.teardown();
+
+        results.set(configType, result);
+        scores.set(configType, this.calculateScore(result));
+
+        if (!this.config.quiet) {
+          this.printRunResult(result);
+        }
+      } catch (err) {
+        console.error(`Error running ${configType}:`, (err as Error).message);
+
+        // Create failed result
+        const failedResult: RunResult = {
+          taskId: task.id,
+          configuration: configType,
+          totalTimeMs: 0,
+          timeToFirstActionMs: 0,
+          messageCount: 0,
+          avgLatencyMs: 0,
+          latencyP50Ms: 0,
+          latencyP99Ms: 0,
+          coordinationRounds: 0,
+          agentCount: 0,
+          totalTokensUsed: 0,
+          peakMemoryMb: 0,
+          success: false,
+          completionRate: 0,
+          errors: [(err as Error).message],
+          startedAt: Date.now(),
+          completedAt: Date.now(),
+        };
+        results.set(configType, failedResult);
+        scores.set(configType, { total: 0, successScore: 0, timeScore: 0, efficiencyScore: 0 });
+      }
+
+      // Cool-down between runs
+      if (this.config.cooldownMs > 0) {
+        await new Promise((r) => setTimeout(r, this.config.cooldownMs));
+      }
+    }
+
+    const winner = this.determineWinner(results, scores);
+
+    return {
+      taskId: task.id,
+      results,
+      winner,
+      scores,
+    };
+  }
+
+  /**
+   * Run a single configuration
+   */
+  async runSingle(
+    task: Task,
+    configType: ConfigurationType
+  ): Promise<RunResult> {
+    const runner = this.runners.get(configType);
+    if (!runner) {
+      throw new Error(`Configuration ${configType} not enabled`);
+    }
+
+    await runner.setup();
+    const result = await runner.run(task);
+    await runner.teardown();
+
+    return result;
+  }
+
+  /**
+   * Calculate score breakdown for a result
+   */
+  private calculateScore(result: RunResult): ScoreBreakdown {
+    const maxTimeMs = 300000; // 5 minutes baseline
+
+    // Success component (0-50 points)
+    const successScore = result.success ? 50 : result.completionRate * 25;
+
+    // Time component (0-30 points) - faster is better
+    const timeScore = result.success
+      ? 30 * Math.max(0, 1 - result.totalTimeMs / maxTimeMs)
+      : 0;
+
+    // Efficiency component (0-20 points) - fewer agents is better for same result
+    const efficiencyScore = result.success
+      ? 20 / Math.max(1, result.agentCount)
+      : 0;
+
+    return {
+      total: successScore + timeScore + efficiencyScore,
+      successScore,
+      timeScore,
+      efficiencyScore,
+    };
+  }
+
+  /**
+   * Determine the winning configuration
+   */
+  private determineWinner(
+    results: Map<ConfigurationType, RunResult>,
+    scores: Map<ConfigurationType, ScoreBreakdown>
+  ): ConfigurationType {
+    let best: ConfigurationType = 'single';
+    let bestScore = -1;
+
+    for (const [configType, score] of scores) {
+      if (score.total > bestScore) {
+        bestScore = score.total;
+        best = configType;
+      }
+    }
+
+    return best;
+  }
+
+  /**
+   * Print a single run result
+   */
+  private printRunResult(result: RunResult): void {
+    console.log(`\nResult for ${result.configuration}:`);
+    console.log(`  Success: ${result.success ? '✓' : '✗'}`);
+    console.log(`  Time: ${(result.totalTimeMs / 1000).toFixed(1)}s`);
+    console.log(`  Agents: ${result.agentCount}`);
+    console.log(`  Messages: ${result.messageCount}`);
+    if (result.errors.length > 0) {
+      console.log(`  Errors: ${result.errors.join(', ')}`);
+    }
+  }
+
+  /**
+   * Print comparison table
+   */
+  printComparison(comparison: ComparisonResult): void {
+    console.log('\n' + '='.repeat(60));
+    console.log('COMPARISON RESULTS');
+    console.log('='.repeat(60));
+    console.log(`Task: ${comparison.taskId}`);
+    console.log(`Winner: ${comparison.winner.toUpperCase()}`);
+    console.log('');
+
+    // Build table data
+    const configs = Array.from(comparison.results.keys());
+    const headers = ['Metric', ...configs.map((c) => c.charAt(0).toUpperCase() + c.slice(1))];
+
+    const rows = [
+      [
+        'Success',
+        ...configs.map((c) =>
+          comparison.results.get(c)?.success ? '✓' : '✗'
+        ),
+      ],
+      [
+        'Time (s)',
+        ...configs.map((c) =>
+          ((comparison.results.get(c)?.totalTimeMs || 0) / 1000).toFixed(1)
+        ),
+      ],
+      [
+        'Agents',
+        ...configs.map((c) =>
+          String(comparison.results.get(c)?.agentCount || 0)
+        ),
+      ],
+      [
+        'Messages',
+        ...configs.map((c) =>
+          String(comparison.results.get(c)?.messageCount || 0)
+        ),
+      ],
+      [
+        'Avg Latency (ms)',
+        ...configs.map((c) =>
+          (comparison.results.get(c)?.avgLatencyMs || 0).toFixed(0)
+        ),
+      ],
+      [
+        'Completion %',
+        ...configs.map((c) =>
+          ((comparison.results.get(c)?.completionRate || 0) * 100).toFixed(0) + '%'
+        ),
+      ],
+      [
+        'Score',
+        ...configs.map((c) =>
+          (comparison.scores.get(c)?.total || 0).toFixed(1)
+        ),
+      ],
+    ];
+
+    // Print table
+    const colWidths = headers.map((h, i) =>
+      Math.max(h.length, ...rows.map((r) => String(r[i]).length))
+    );
+
+    const separator = colWidths.map((w) => '-'.repeat(w + 2)).join('+');
+
+    console.log(separator);
+    console.log(
+      '|' +
+        headers.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('|') +
+        '|'
+    );
+    console.log(separator);
+
+    for (const row of rows) {
+      console.log(
+        '|' +
+          row.map((cell, i) => ` ${String(cell).padEnd(colWidths[i])} `).join('|') +
+          '|'
+      );
+    }
+    console.log(separator);
+  }
+}
+
+/**
+ * Quick helper to run a comparison benchmark
+ */
+export async function runComparison(
+  task: Task,
+  config?: Partial<BenchmarkConfig>
+): Promise<ComparisonResult> {
+  const benchmark = new ComparisonBenchmark(config);
+  return benchmark.runComparison(task);
+}