agent-relay 2.3.2 → 2.3.5

package/packages/benchmark/dist/types.d.ts

@@ -1,178 +0,0 @@
-/**
- * Benchmark Types
- *
- * Type definitions for the agent swarm performance benchmark system.
- */
-/**
- * Configuration type for benchmark runs
- */
-export type ConfigurationType = 'single' | 'subagent' | 'swarm';
-/**
- * Task complexity level
- */
-export type TaskComplexity = 'low' | 'medium' | 'high';
-/**
- * A benchmark task definition
- */
-export interface Task {
-    /** Unique task identifier */
-    id: string;
-    /** Human-readable task description */
-    description: string;
-    /** Files the task operates on */
-    files: string[];
-    /** Success criteria for the task */
-    expectedOutcome: string;
-    /** Optional Harbor-style success criteria key for compatibility */
-    success_criteria?: string;
-    /** Task complexity level */
-    complexity: TaskComplexity;
-    /** Optional timeout in milliseconds (default: 300000 = 5 min) */
-    timeoutMs?: number;
-    /** Optional tags for categorization */
-    tags?: string[];
-}
-/**
- * Result of a single benchmark run
- */
-export interface RunResult {
-    /** Task identifier */
-    taskId: string;
-    /** Configuration used for this run */
-    configuration: ConfigurationType;
-    /** Total time from start to completion in milliseconds */
-    totalTimeMs: number;
-    /** Time to first agent action in milliseconds */
-    timeToFirstActionMs: number;
-    /** Total number of inter-agent messages */
-    messageCount: number;
-    /** Average message latency in milliseconds */
-    avgLatencyMs: number;
-    /** P50 latency in milliseconds */
-    latencyP50Ms: number;
-    /** P99 latency in milliseconds */
-    latencyP99Ms: number;
-    /** Number of coordination rounds */
-    coordinationRounds: number;
-    /** Number of agents used */
-    agentCount: number;
-    /** Total tokens consumed (if available) */
-    totalTokensUsed: number;
-    /** Peak memory usage in MB */
-    peakMemoryMb: number;
-    /** Whether the task completed successfully */
-    success: boolean;
-    /** Completion rate (0-1) for partial success */
-    completionRate: number;
-    /** Error messages if any */
-    errors: string[];
-    /** Timestamp when the run started */
-    startedAt: number;
-    /** Timestamp when the run completed */
-    completedAt: number;
-}
-/**
- * Comparison result across all configurations
- */
-export interface ComparisonResult {
-    /** Task identifier */
-    taskId: string;
-    /** Results for each configuration */
-    results: Map<ConfigurationType, RunResult>;
-    /** The winning configuration based on scoring */
-    winner: ConfigurationType;
-    /** Score breakdown for each configuration */
-    scores: Map<ConfigurationType, ScoreBreakdown>;
-}
-/**
- * Score breakdown for a configuration
- */
-export interface ScoreBreakdown {
-    /** Total score (0-100) */
-    total: number;
-    /** Success component (0-50) */
-    successScore: number;
-    /** Time efficiency component (0-30) */
-    timeScore: number;
-    /** Resource efficiency component (0-20) */
-    efficiencyScore: number;
-}
-/**
- * Benchmark configuration options
- */
-export interface BenchmarkConfig {
-    /** Which configurations to run */
-    configurations: ConfigurationType[];
-    /** CLI to use for agents (default: 'claude') */
-    cli: string;
-    /** Working directory for tasks */
-    cwd?: string;
-    /** Suppress console output */
-    quiet: boolean;
-    /** Cool-down time between runs in milliseconds */
-    cooldownMs: number;
-    /** Maximum concurrent agents for swarm */
-    maxSwarmSize: number;
-    /** Custom socket path for relay */
-    socketPath?: string;
-}
-/**
- * Default benchmark configuration
- */
-export declare const DEFAULT_BENCHMARK_CONFIG: BenchmarkConfig;
-/**
- * Metrics collected during a run
- */
-export interface RunMetrics {
-    /** Number of messages sent */
-    messages: number;
-    /** Message latencies in milliseconds */
-    latencies: number[];
-    /** Run start timestamp */
-    startTime: number;
-    /** Spawned agent names */
-    spawnedAgents: string[];
-    /** Error events */
-    errors: string[];
-}
-/**
- * Task dataset definition
- */
-export interface TaskDataset {
-    /** Dataset name */
-    name: string;
-    /** Dataset description */
-    description?: string;
-    /** Version identifier */
-    version?: string;
-    /** Tasks in the dataset */
-    tasks: Task[];
-}
-/**
- * Harbor-compatible evaluation input
- */
-export interface HarborTaskInput {
-    id: string;
-    description: string;
-    files?: string[];
-    success_criteria?: string;
-    complexity?: TaskComplexity;
-    agents_required?: number;
-    [key: string]: unknown;
-}
-/**
- * Harbor-compatible evaluation output
- */
-export interface HarborEvaluationOutput {
-    task_id: string;
-    configurations: Record<ConfigurationType, RunResult>;
-    winner: ConfigurationType;
-    scores: Record<ConfigurationType, ScoreBreakdown>;
-    metadata: {
-        benchmark_version: string;
-        started_at: number;
-        completed_at: number;
-        total_duration_ms: number;
-    };
-}
-//# sourceMappingURL=types.d.ts.map

package/packages/benchmark/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEvD;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,eAAe,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4BAA4B;IAC5B,UAAU,EAAE,cAAc,CAAC;IAC3B,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,aAAa,EAAE,iBAAiB,CAAC;IAGjC,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAG5B,2CAA2C;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,8CAA8C;IAC9C,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAG3B,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IAGrB,8CAA8C;IAC9C,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,EAAE,CAAC;IAGjB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,OAAO,EAAE,GAAG,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IAC3C,iDAAiD;IACjD,MAAM,EAAE,iBAAiB,CAAC;IAC1B,6CAA6C;IAC7C,MAAM,EAAE,GAAG,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kCAAkC;IAClC,cAAc,EAAE,iBAAiB,EAAE,CAAC;IACpC,gDAAgD;IAChD,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,KAAK,EAAE,OAAO,CAAC;IACf,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAMtC,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,mBAAmB;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IACrD,MAAM,EAAE,iBAAiB,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;IAClD,QAAQ,EAAE;QACR,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH"}

package/packages/benchmark/dist/types.js

@@ -1,16 +0,0 @@
-/**
- * Benchmark Types
- *
- * Type definitions for the agent swarm performance benchmark system.
- */
-/**
- * Default benchmark configuration
- */
-export const DEFAULT_BENCHMARK_CONFIG = {
-    configurations: ['single', 'subagent', 'swarm'],
-    cli: 'claude',
-    quiet: false,
-    cooldownMs: 5000,
-    maxSwarmSize: 10,
-};
-//# sourceMappingURL=types.js.map

package/packages/benchmark/dist/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAoIH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC;IAC/C,GAAG,EAAE,QAAQ;IACb,KAAK,EAAE,KAAK;IACZ,UAAU,EAAE,IAAI;IAChB,YAAY,EAAE,EAAE;CACjB,CAAC"}

package/packages/benchmark/package.json

@@ -1,80 +0,0 @@
-{
-  "name": "@agent-relay/benchmark",
-  "version": "2.3.2",
-  "description": "Performance benchmarking for agent swarms, sub-agents, and single agents using Harbor",
-  "type": "module",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js"
-    },
-    "./runners": {
-      "types": "./dist/runners/index.d.ts",
-      "import": "./dist/runners/index.js"
-    },
-    "./harbor": {
-      "types": "./dist/harbor.d.ts",
-      "import": "./dist/harbor.js"
-    }
-  },
-  "bin": {
-    "relay-benchmark": "dist/cli.js"
-  },
-  "files": [
-    "dist",
-    "datasets",
-    "README.md"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "clean": "rm -rf dist",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "benchmark": "node dist/cli.js",
-    "benchmark:single": "node dist/cli.js --config single",
-    "benchmark:subagent": "node dist/cli.js --config subagent",
-    "benchmark:swarm": "node dist/cli.js --config swarm",
-    "benchmark:compare": "node dist/cli.js --config all"
-  },
-  "keywords": [
-    "agent-relay",
-    "benchmark",
-    "performance",
-    "swarm",
-    "harbor",
-    "ai-agents"
-  ],
-  "author": "Agent Workforce",
-  "license": "Apache-2.0",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/AgentWorkforce/relay.git",
-    "directory": "packages/benchmark"
-  },
-  "dependencies": {
-    "@agent-relay/sdk": "2.3.2",
-    "@agent-relay/protocol": "2.3.2",
-    "@agent-relay/spawner": "2.3.2",
-    "commander": "^12.1.0",
-    "yaml": "^2.3.4"
-  },
-  "devDependencies": {
-    "@agent-relay/daemon": "2.3.2",
-    "@types/node": "^22.19.3",
-    "typescript": "^5.9.3",
-    "vitest": "^3.0.0"
-  },
-  "peerDependencies": {
-    "@agent-relay/daemon": ">=2.0.0"
-  },
-  "peerDependenciesMeta": {
-    "@agent-relay/daemon": {
-      "optional": true
-    }
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  }
-}

package/packages/benchmark/src/benchmark.ts

@@ -1,298 +0,0 @@
-/**
- * Comparison Benchmark
- *
- * Main orchestrator for running comparison benchmarks across configurations.
- */
-
-import type {
-  ConfigurationType,
-  Task,
-  RunResult,
-  ComparisonResult,
-  ScoreBreakdown,
-  BenchmarkConfig,
-} from './types.js';
-import { DEFAULT_BENCHMARK_CONFIG } from './types.js';
-import {
-  ConfigurationRunner,
-  SingleAgentRunner,
-  SubAgentRunner,
-  SwarmRunner,
-} from './runners/index.js';
-
-/**
- * Main benchmark orchestrator
- */
-export class ComparisonBenchmark {
-  private config: BenchmarkConfig;
-  private runners: Map<ConfigurationType, ConfigurationRunner>;
-
-  constructor(config: Partial<BenchmarkConfig> = {}) {
-    this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
-
-    // Initialize runners for configured configurations
-    this.runners = new Map();
-    for (const configType of this.config.configurations) {
-      this.runners.set(configType, this.createRunner(configType));
-    }
-  }
-
-  /**
-   * Create a runner for a configuration type
-   */
-  private createRunner(type: ConfigurationType): ConfigurationRunner {
-    switch (type) {
-      case 'single':
-        return new SingleAgentRunner(this.config);
-      case 'subagent':
-        return new SubAgentRunner(this.config);
-      case 'swarm':
-        return new SwarmRunner(this.config);
-      default:
-        throw new Error(`Unknown configuration type: ${type}`);
-    }
-  }
-
-  /**
-   * Run a comparison across all configured configurations
-   */
-  async runComparison(task: Task): Promise<ComparisonResult> {
-    const results = new Map<ConfigurationType, RunResult>();
-    const scores = new Map<ConfigurationType, ScoreBreakdown>();
-
-    for (const [configType, runner] of this.runners) {
-      if (!this.config.quiet) {
-        console.log(`\n=== Running ${configType} configuration ===`);
-        console.log(`Task: ${task.id}`);
-      }
-
-      try {
-        await runner.setup();
-        const result = await runner.run(task);
-        await runner.teardown();
-
-        results.set(configType, result);
-        scores.set(configType, this.calculateScore(result));
-
-        if (!this.config.quiet) {
-          this.printRunResult(result);
-        }
-      } catch (err) {
-        console.error(`Error running ${configType}:`, (err as Error).message);
-
-        // Create failed result
-        const failedResult: RunResult = {
-          taskId: task.id,
-          configuration: configType,
-          totalTimeMs: 0,
-          timeToFirstActionMs: 0,
-          messageCount: 0,
-          avgLatencyMs: 0,
-          latencyP50Ms: 0,
-          latencyP99Ms: 0,
-          coordinationRounds: 0,
-          agentCount: 0,
-          totalTokensUsed: 0,
-          peakMemoryMb: 0,
-          success: false,
-          completionRate: 0,
-          errors: [(err as Error).message],
-          startedAt: Date.now(),
-          completedAt: Date.now(),
-        };
-        results.set(configType, failedResult);
-        scores.set(configType, { total: 0, successScore: 0, timeScore: 0, efficiencyScore: 0 });
-      }
-
-      // Cool-down between runs
-      if (this.config.cooldownMs > 0) {
-        await new Promise((r) => setTimeout(r, this.config.cooldownMs));
-      }
-    }
-
-    const winner = this.determineWinner(results, scores);
-
-    return {
-      taskId: task.id,
-      results,
-      winner,
-      scores,
-    };
-  }
-
-  /**
-   * Run a single configuration
-   */
-  async runSingle(
-    task: Task,
-    configType: ConfigurationType
-  ): Promise<RunResult> {
-    const runner = this.runners.get(configType);
-    if (!runner) {
-      throw new Error(`Configuration ${configType} not enabled`);
-    }
-
-    await runner.setup();
-    const result = await runner.run(task);
-    await runner.teardown();
-
-    return result;
-  }
-
-  /**
-   * Calculate score breakdown for a result
-   */
-  private calculateScore(result: RunResult): ScoreBreakdown {
-    const maxTimeMs = 300000; // 5 minutes baseline
-
-    // Success component (0-50 points)
-    const successScore = result.success ? 50 : result.completionRate * 25;
-
-    // Time component (0-30 points) - faster is better
-    const timeScore = result.success
-      ? 30 * Math.max(0, 1 - result.totalTimeMs / maxTimeMs)
-      : 0;
-
-    // Efficiency component (0-20 points) - fewer agents is better for same result
-    const efficiencyScore = result.success
-      ? 20 / Math.max(1, result.agentCount)
-      : 0;
-
-    return {
-      total: successScore + timeScore + efficiencyScore,
-      successScore,
-      timeScore,
-      efficiencyScore,
-    };
-  }
-
-  /**
-   * Determine the winning configuration
-   */
-  private determineWinner(
-    results: Map<ConfigurationType, RunResult>,
-    scores: Map<ConfigurationType, ScoreBreakdown>
-  ): ConfigurationType {
-    let best: ConfigurationType = 'single';
-    let bestScore = -1;
-
-    for (const [configType, score] of scores) {
-      if (score.total > bestScore) {
-        bestScore = score.total;
-        best = configType;
-      }
-    }
-
-    return best;
-  }
-
-  /**
-   * Print a single run result
-   */
-  private printRunResult(result: RunResult): void {
-    console.log(`\nResult for ${result.configuration}:`);
-    console.log(`  Success: ${result.success ? '✓' : '✗'}`);
-    console.log(`  Time: ${(result.totalTimeMs / 1000).toFixed(1)}s`);
-    console.log(`  Agents: ${result.agentCount}`);
-    console.log(`  Messages: ${result.messageCount}`);
-    if (result.errors.length > 0) {
-      console.log(`  Errors: ${result.errors.join(', ')}`);
-    }
-  }
-
-  /**
-   * Print comparison table
-   */
-  printComparison(comparison: ComparisonResult): void {
-    console.log('\n' + '='.repeat(60));
-    console.log('COMPARISON RESULTS');
-    console.log('='.repeat(60));
-    console.log(`Task: ${comparison.taskId}`);
-    console.log(`Winner: ${comparison.winner.toUpperCase()}`);
-    console.log('');
-
-    // Build table data
-    const configs = Array.from(comparison.results.keys());
-    const headers = ['Metric', ...configs.map((c) => c.charAt(0).toUpperCase() + c.slice(1))];
-
-    const rows = [
-      [
-        'Success',
-        ...configs.map((c) =>
-          comparison.results.get(c)?.success ? '✓' : '✗'
-        ),
-      ],
-      [
-        'Time (s)',
-        ...configs.map((c) =>
-          ((comparison.results.get(c)?.totalTimeMs || 0) / 1000).toFixed(1)
-        ),
-      ],
-      [
-        'Agents',
-        ...configs.map((c) =>
-          String(comparison.results.get(c)?.agentCount || 0)
-        ),
-      ],
-      [
-        'Messages',
-        ...configs.map((c) =>
-          String(comparison.results.get(c)?.messageCount || 0)
-        ),
-      ],
-      [
-        'Avg Latency (ms)',
-        ...configs.map((c) =>
-          (comparison.results.get(c)?.avgLatencyMs || 0).toFixed(0)
-        ),
-      ],
-      [
-        'Completion %',
-        ...configs.map((c) =>
-          ((comparison.results.get(c)?.completionRate || 0) * 100).toFixed(0) + '%'
-        ),
-      ],
-      [
-        'Score',
-        ...configs.map((c) =>
-          (comparison.scores.get(c)?.total || 0).toFixed(1)
-        ),
-      ],
-    ];
-
-    // Print table
-    const colWidths = headers.map((h, i) =>
-      Math.max(h.length, ...rows.map((r) => String(r[i]).length))
-    );
-
-    const separator = colWidths.map((w) => '-'.repeat(w + 2)).join('+');
-
-    console.log(separator);
-    console.log(
-      '|' +
-        headers.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('|') +
-        '|'
-    );
-    console.log(separator);
-
-    for (const row of rows) {
-      console.log(
-        '|' +
-          row.map((cell, i) => ` ${String(cell).padEnd(colWidths[i])} `).join('|') +
-          '|'
-      );
-    }
-    console.log(separator);
-  }
-}
-
-/**
- * Quick helper to run a comparison benchmark
- */
-export async function runComparison(
-  task: Task,
-  config?: Partial<BenchmarkConfig>
-): Promise<ComparisonResult> {
-  const benchmark = new ComparisonBenchmark(config);
-  return benchmark.runComparison(task);
-}