@theone1345/smartrelay 0.2.0

package/config/runners/openrouter.yaml

@@ -0,0 +1,83 @@
+runners:
+  # ==========================================
+  # OpenRouter Cloud Models
+  # ==========================================
+  openrouter-claude-3.7-sonnet:
+    type: "openrouter"
+    model: "anthropic/claude-3.7-sonnet"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 3.00
+    cost_per_million_output_tokens: 15.00
+    default_params:
+      max_tokens: 4096
+      temperature: 0.5
+
+  openrouter-deepseek-r1:
+    type: "openrouter"
+    model: "deepseek/deepseek-r1"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.55
+    cost_per_million_output_tokens: 2.19
+    default_params:
+      max_tokens: 4096
+      temperature: 0.6
+
+  openrouter-deepseek-v3:
+    type: "openrouter"
+    model: "deepseek/deepseek-chat"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.14
+    cost_per_million_output_tokens: 0.28
+    default_params:
+      max_tokens: 2048
+      temperature: 0.7
+
+  openrouter-qwen-2.5-coder:
+    type: "openrouter"
+    model: "qwen/qwen-2.5-coder-32b-instruct"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.12
+    cost_per_million_output_tokens: 0.20
+    default_params:
+      max_tokens: 4096
+      temperature: 0.2
+
+  openrouter-llama-3.3-70b:
+    type: "openrouter"
+    model: "meta-llama/llama-3.3-70b-instruct"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.12
+    cost_per_million_output_tokens: 0.30
+    default_params:
+      max_tokens: 4096
+      temperature: 0.7
+
+  openrouter-gemini-2.0-flash:
+    type: "openrouter"
+    model: "google/gemini-2.0-flash-001"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.10
+    cost_per_million_output_tokens: 0.40
+    default_params:
+      max_tokens: 4096
+      temperature: 0.7
+
+  openrouter-claude-sonnet-4.5:
+    type: "openrouter"
+    model: "anthropic/claude-sonnet-4.5"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 3.00
+    cost_per_million_output_tokens: 15.00
+    default_params:
+      max_tokens: 2048
+      temperature: 0.7
+
+  openrouter-qwen-coder-free:
+    type: "openrouter"
+    model: "qwen/qwen-2.5-coder-32b-instruct:free"
+    api_key_env: "OPENROUTER_API_KEY"
+    cost_per_million_input_tokens: 0.00
+    cost_per_million_output_tokens: 0.00
+    default_params:
+      max_tokens: 2048
+      temperature: 0.2

package/config.yaml

@@ -0,0 +1,16 @@
+server:
+  name: "mcp-delegation-server"
+  default_runner: "nemotron-3-super-120b-a12b"
+  max_concurrency: 5
+  default_timeout_seconds: 60.0
+
+# =========================================================================
+# Modular Runner Configurations (Organized by Provider & Purpose)
+# =========================================================================
+includes:
+  - "config/runners/agents.yaml"       # Specialized Role-Based Agents
+  - "config/runners/nvidia.yaml"       # NVIDIA NIM / API Catalog Models
+  - "config/runners/openrouter.yaml"   # OpenRouter Cloud Models
+  - "config/runners/ollama.yaml"       # Local Offline Models ($0.00)
+  - "config/runners/anthropic.yaml"    # Anthropic Direct API
+  - "config/runners/openai.yaml"       # OpenAI Direct API

package/dist/benchmark/engine.d.ts

@@ -0,0 +1,49 @@
+/** Benchmark engine for concurrent multi-runner execution and evaluation. */
+import type { RunnerRegistry } from '../runners/registry.js';
+/** Performance and evaluation scores for a single runner in a benchmark. */
+export interface RunnerBenchmarkSummary {
+    runner_id: string;
+    model: string;
+    success: boolean;
+    latency_ms: number;
+    input_tokens: number;
+    output_tokens: number;
+    estimated_cost_usd: number;
+    scores: Record<string, Record<string, unknown>>;
+    overall_score: number;
+    output_preview: string;
+    full_output: string;
+    error_message: string | null;
+}
+/** Consolidated benchmark report across all participating runners. */
+export interface BenchmarkResult {
+    task: string;
+    timestamp: string;
+    total_duration_ms: number;
+    winner_runner_id: string | null;
+    runners_evaluated: number;
+    successful_runs: number;
+    failed_runs: number;
+    summaries: RunnerBenchmarkSummary[];
+    comparison_table_markdown: string;
+}
+/** Serialize a benchmark report the way `BenchmarkResult.to_json()` did. */
+export declare function benchmarkResultToJson(result: BenchmarkResult, indent?: number | null): string;
+export interface RunBenchmarkOptions {
+    task: string;
+    runnerIds: string[];
+    params?: Record<string, unknown> | null;
+    evalCriteria?: Record<string, unknown> | null;
+    referenceAnswer?: string | null;
+    judgeRunnerId?: string | null;
+}
+/** Coordinates concurrent runner execution, applies scorers, and aggregates results. */
+export declare class BenchmarkEngine {
+    readonly registry: RunnerRegistry;
+    readonly maxConcurrency: number;
+    constructor(registry: RunnerRegistry, maxConcurrency?: number);
+    /** Fan a task out across multiple runners concurrently and evaluate their outputs. */
+    runBenchmark(options: RunBenchmarkOptions): Promise<BenchmarkResult>;
+    /** Construct a GitHub-flavored Markdown comparison table. */
+    private buildMarkdownTable;
+}

package/dist/benchmark/engine.js

@@ -0,0 +1,147 @@
+/** Benchmark engine for concurrent multi-runner execution and evaluation. */
+import { CostScorer, ExactMatchScorer, LatencyScorer, LLMJudgeScorer, } from './scorers.js';
+import { makeRunnerResult } from '../runners/base.js';
+import { errorMessage, round, Semaphore, startTimer, truncateByCodePoint } from '../util.js';
+/** Serialize a benchmark report the way `BenchmarkResult.to_json()` did. */
+export function benchmarkResultToJson(result, indent = 2) {
+    return JSON.stringify(result, null, indent ?? undefined);
+}
+/** Coordinates concurrent runner execution, applies scorers, and aggregates results. */
+export class BenchmarkEngine {
+    registry;
+    maxConcurrency;
+    constructor(registry, maxConcurrency = 5) {
+        this.registry = registry;
+        this.maxConcurrency = maxConcurrency;
+    }
+    /** Fan a task out across multiple runners concurrently and evaluate their outputs. */
+    async runBenchmark(options) {
+        const { task, runnerIds, params, evalCriteria, referenceAnswer, judgeRunnerId } = options;
+        const elapsed = startTimer();
+        const timestamp = new Date().toISOString();
+        const criteria = evalCriteria ?? {};
+        // Unknown runner IDs are silently skipped, as in the Python original.
+        const runnersToExecute = [];
+        for (const runnerId of runnerIds) {
+            const runner = this.registry.get(runnerId);
+            if (runner)
+                runnersToExecute.push(runner);
+        }
+        // 1. Execute all runners concurrently, bounded by max_concurrency.
+        const semaphore = new Semaphore(this.maxConcurrency);
+        const rawResults = await Promise.all(runnersToExecute.map((runner) => semaphore.run(async () => {
+            try {
+                return await runner.execute(task, params);
+            }
+            catch (error) {
+                return makeRunnerResult({
+                    runner_id: runner.id,
+                    model: runner.model,
+                    task,
+                    success: false,
+                    error_message: `Execution failed: ${errorMessage(error)}`,
+                });
+            }
+        })));
+        // 2. Build scorers.
+        const scorers = [new LatencyScorer(), new CostScorer()];
+        if (referenceAnswer)
+            scorers.push(new ExactMatchScorer());
+        if (judgeRunnerId) {
+            const judgeRunner = this.registry.get(judgeRunnerId);
+            if (judgeRunner) {
+                const rubric = typeof criteria['rubric'] === 'string' ? criteria['rubric'] : null;
+                scorers.push(new LLMJudgeScorer(judgeRunner, rubric));
+            }
+        }
+        // 3. Score every result.
+        const summaries = [];
+        for (const result of rawResults) {
+            const scores = {};
+            const qualityScores = [];
+            for (const scorer of scorers) {
+                try {
+                    const scoreResult = await scorer.score({
+                        task,
+                        result,
+                        referenceAnswer,
+                        allResults: rawResults,
+                    });
+                    scores[scorer.name] = scoreResult;
+                    if (scorer instanceof LLMJudgeScorer) {
+                        qualityScores.push(scoreResult.score);
+                    }
+                    else if (scorer instanceof ExactMatchScorer && referenceAnswer) {
+                        qualityScores.push(scoreResult.score);
+                    }
+                }
+                catch (error) {
+                    scores[scorer.name] = {
+                        scorer_name: scorer.name,
+                        score: 0,
+                        error: errorMessage(error),
+                    };
+                }
+            }
+            // Overall score: mean of the quality scores, else 100 on success and 0 on failure.
+            let overall;
+            if (qualityScores.length > 0) {
+                overall = round(qualityScores.reduce((sum, value) => sum + value, 0) / qualityScores.length, 1);
+            }
+            else {
+                overall = result.success ? 100 : 0;
+            }
+            summaries.push({
+                runner_id: result.runner_id,
+                model: result.model,
+                success: result.success,
+                latency_ms: result.latency_ms,
+                input_tokens: result.input_tokens,
+                output_tokens: result.output_tokens,
+                estimated_cost_usd: result.estimated_cost_usd,
+                scores,
+                overall_score: overall,
+                output_preview: truncateByCodePoint(result.output, 150),
+                full_output: result.output,
+                error_message: result.error_message,
+            });
+        }
+        // 4. Rank: successes first, then higher score, then lower latency.
+        summaries.sort((a, b) => Number(b.success) - Number(a.success) ||
+            b.overall_score - a.overall_score ||
+            a.latency_ms - b.latency_ms);
+        const winner = summaries[0];
+        const winnerId = winner && winner.success ? winner.runner_id : null;
+        const successfulCount = summaries.filter((summary) => summary.success).length;
+        return {
+            task,
+            timestamp,
+            total_duration_ms: elapsed(),
+            winner_runner_id: winnerId,
+            runners_evaluated: summaries.length,
+            successful_runs: successfulCount,
+            failed_runs: summaries.length - successfulCount,
+            summaries,
+            comparison_table_markdown: this.buildMarkdownTable(summaries, winnerId),
+        };
+    }
+    /** Construct a GitHub-flavored Markdown comparison table. */
+    buildMarkdownTable(summaries, winnerId) {
+        const lines = [
+            '| Rank | Runner ID | Model | Status | Latency | Tokens (In/Out) | Est. Cost | Quality Score |',
+            '| :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |',
+        ];
+        summaries.forEach((summary, index) => {
+            const trophy = summary.runner_id === winnerId ? ' 🏆 (Winner)' : '';
+            const status = summary.success ? '✅ Success' : '❌ Failed';
+            const tokens = `${summary.input_tokens} / ${summary.output_tokens}`;
+            const cost = `$${summary.estimated_cost_usd.toFixed(5)}`;
+            const latency = `${summary.latency_ms.toFixed(0)} ms`;
+            const score = summary.success ? `${summary.overall_score.toFixed(1)}/100` : 'N/A';
+            lines.push(`| ${index + 1} | **\`${summary.runner_id}\`**${trophy} | \`${summary.model}\` | ${status} | ` +
+                `${latency} | ${tokens} | ${cost} | ${score} |`);
+        });
+        return lines.join('\n');
+    }
+}
+//# sourceMappingURL=engine.js.map

package/dist/benchmark/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/benchmark/engine.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAEL,UAAU,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GAEf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,gBAAgB,EAAsC,MAAM,oBAAoB,CAAC;AAE1F,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AA+B7F,4EAA4E;AAC5E,MAAM,UAAU,qBAAqB,CAAC,MAAuB,EAAE,MAAM,GAAkB,CAAC;IACtF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC,CAAC;AAC3D,CAAC;AAWD,wFAAwF;AACxF,MAAM,OAAO,eAAe;IAEf,QAAQ;IACR,cAAc;IAFzB,YACW,QAAwB,EACxB,cAAc,GAAW,CAAC;wBAD1B,QAAQ;8BACR,cAAc;IACtB,CAAC;IAEJ,sFAAsF;IACtF,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC7C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAE1F,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,YAAY,IAAI,EAAE,CAAC;QAEpC,sEAAsE;QACtE,MAAM,gBAAgB,GAAiB,EAAE,CAAC;QAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,MAAM;gBAAE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAED,mEAAmE;QACnE,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC9B,SAAS,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,OAAO,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,gBAAgB,CAAC;oBACtB,SAAS,EAAE,MAAM,CAAC,EAAE;oBACpB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,IAAI;oBACJ,OAAO,EAAE,KAAK;oBACd,aAAa,EAAE,qBAAqB,YAAY,CAAC,KAAK,CAAC,EAAE;iBAC1D,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,oBAAoB;QACpB,MAAM,OAAO,GAAiB,CAAC,IAAI,aAAa,EAAE,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACtE,IAAI,eAAe;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC;QAE1D,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACrD,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAClF,OAAO,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,MAAM,SAAS,GAA6B,EAAE,CAAC;QAE/C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,MAAM,GAA4C,EAAE,CAAC;YAC3D,MAAM,aAAa,GAAa,EAAE,CAAC;YAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,WAAW,GAAgB,MAAM,MAAM,CAAC,KAAK,CAAC;wBAClD,IAAI;wBACJ,MAAM;wBACN,eAAe;wBACf,UAAU,EAAE,UAAU;qBACvB,CAAC,CAAC;oBACH,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,WAAiD,CAAC;oBAExE,IAAI,MAAM,YAAY,cAAc,EAAE,CAAC;wBACrC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;oBACxC,CAAC;yBAAM,IAAI,MAAM,YAAY,gBAAgB,IAAI,eAAe,EAAE,CAAC;wBACjE,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG;wBACpB,WAAW,EAAE,MAAM,CAAC,IAAI;wBACxB,KAAK,EAAE,CAAC;wBACR,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC;qBAC3B,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,mFAAmF;YACnF,IAAI,OAAe,CAAC;YACpB,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAClG,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,CAAC;YAED,SAAS,CAAC,IAAI,CAAC;gBACb,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;gBAC7C,MAAM;gBACN,aAAa,EAAE,OAAO;gBACtB,cAAc,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;gBACvD,WAAW,EAAE,MAAM,CAAC,MAAM;gBAC1B,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC,CAAC,CAAC;QACL,CAAC;QAED,mEAAmE;QACnE,SAAS,CAAC,IAAI,CACZ,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;YACrC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa;YACjC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAC9B,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QACpE,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,SAAS;YACT,iBAAiB,EAAE,OAAO,EAAE;YAC5B,gBAAgB,EAAE,QAAQ;YAC1B,iBAAiB,EAAE,SAAS,CAAC,MAAM;YACnC,eAAe,EAAE,eAAe;YAChC,WAAW,EAAE,SAAS,CAAC,MAAM,GAAG,eAAe;YAC/C,SAAS;YACT,yBAAyB,EAAE,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC;SACxE,CAAC;IACJ,CAAC;IAED,6DAA6D;IACrD,kBAAkB,CAAC,SAAmC,EAAE,QAAuB;QACrF,MAAM,KAAK,GAAG;YACZ,+FAA+F;YAC/F,gEAAgE;SACjE,CAAC;QAEF,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1D,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC;YACpE,MAAM,IAAI,GAAG,IAAI,OAAO,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,OAAO,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;YACtD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;YAElF,KAAK,CAAC,IAAI,CACR,KAAK,KAAK,GAAG,CAAC,UAAU,OAAO,CAAC,SAAS,OAAO,MAAM,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK;gBAC5F,GAAG,OAAO,MAAM,MAAM,MAAM,IAAI,MAAM,KAAK,IAAI,CAClD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF"}

package/dist/benchmark/scorers.d.ts

@@ -0,0 +1,59 @@
+/** Pluggable scorers for benchmarking LLM outputs. */
+import type { BaseRunner, RunnerResult } from '../runners/base.js';
+/**
+ * Evaluation score and metrics returned by a scorer.
+ *
+ * snake_case and in declaration order: this is serialized straight onto the wire.
+ */
+export interface ScoreResult {
+    scorer_name: string;
+    /** Normalized score (typically 0-100) or a raw metric value. */
+    score: number;
+    /** "points", "ms", "usd", "percent", "tokens/sec". */
+    unit: string;
+    details: Record<string, unknown>;
+    reasoning: string | null;
+}
+/** Context passed to every scorer. */
+export interface ScoreContext {
+    task: string;
+    result: RunnerResult;
+    referenceAnswer?: string | null;
+    allResults?: RunnerResult[] | null;
+}
+/** Abstract base class for benchmark scorers. */
+export declare abstract class BaseScorer {
+    abstract get name(): string;
+    /** Compute an evaluation score for a runner's result. */
+    abstract score(context: ScoreContext): Promise<ScoreResult>;
+}
+/** Scores runners on response latency and generation throughput. */
+export declare class LatencyScorer extends BaseScorer {
+    get name(): string;
+    score({ result }: ScoreContext): Promise<ScoreResult>;
+}
+/** Scores runners on financial token cost in USD. */
+export declare class CostScorer extends BaseScorer {
+    get name(): string;
+    score({ result }: ScoreContext): Promise<ScoreResult>;
+}
+/**
+ * Scores output against a reference answer via exact match and token overlap.
+ *
+ * The word split uses `\p{L}\p{N}_` rather than JS's ASCII-only `\w` so the
+ * overlap ratio matches Python's Unicode-aware `re.findall(r"\w+", ...)`.
+ */
+export declare class ExactMatchScorer extends BaseScorer {
+    private static readonly WORD_PATTERN;
+    get name(): string;
+    score({ result, referenceAnswer }: ScoreContext): Promise<ScoreResult>;
+}
+/** Uses a designated LLM runner as a judge to score outputs against a rubric. */
+export declare class LLMJudgeScorer extends BaseScorer {
+    readonly judgeRunner: BaseRunner;
+    static readonly DEFAULT_RUBRIC: string;
+    readonly rubric: string;
+    constructor(judgeRunner: BaseRunner, rubric?: string | null);
+    get name(): string;
+    score({ task, result, referenceAnswer }: ScoreContext): Promise<ScoreResult>;
+}

package/dist/benchmark/scorers.js

@@ -0,0 +1,241 @@
+/** Pluggable scorers for benchmarking LLM outputs. */
+import { errorMessage, round } from '../util.js';
+function makeScore(init) {
+    return {
+        scorer_name: init.scorer_name,
+        score: init.score,
+        unit: init.unit ?? 'points',
+        details: init.details ?? {},
+        reasoning: init.reasoning ?? null,
+    };
+}
+/** Abstract base class for benchmark scorers. */
+export class BaseScorer {
+}
+/** Scores runners on response latency and generation throughput. */
+export class LatencyScorer extends BaseScorer {
+    get name() {
+        return 'latency';
+    }
+    async score({ result }) {
+        if (!result.success) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 0,
+                unit: 'ms',
+                details: { latency_ms: result.latency_ms, tokens_per_second: 0 },
+                reasoning: 'Execution failed',
+            });
+        }
+        let tokensPerSec = 0;
+        if (result.latency_ms > 0 && result.output_tokens > 0) {
+            tokensPerSec = round(result.output_tokens / (result.latency_ms / 1000), 2);
+        }
+        return makeScore({
+            scorer_name: this.name,
+            score: result.latency_ms,
+            unit: 'ms',
+            details: {
+                latency_ms: result.latency_ms,
+                output_tokens: result.output_tokens,
+                tokens_per_second: tokensPerSec,
+            },
+            reasoning: `Completed in ${result.latency_ms} ms (~${tokensPerSec} tokens/sec)`,
+        });
+    }
+}
+/** Scores runners on financial token cost in USD. */
+export class CostScorer extends BaseScorer {
+    get name() {
+        return 'cost';
+    }
+    async score({ result }) {
+        const totalTokens = result.input_tokens + result.output_tokens;
+        return makeScore({
+            scorer_name: this.name,
+            score: result.estimated_cost_usd,
+            unit: 'usd',
+            details: {
+                input_tokens: result.input_tokens,
+                output_tokens: result.output_tokens,
+                total_tokens: totalTokens,
+                estimated_cost_usd: result.estimated_cost_usd,
+            },
+            reasoning: `Cost: $${result.estimated_cost_usd.toFixed(6)} across ${totalTokens} total tokens`,
+        });
+    }
+}
+/**
+ * Scores output against a reference answer via exact match and token overlap.
+ *
+ * The word split uses `\p{L}\p{N}_` rather than JS's ASCII-only `\w` so the
+ * overlap ratio matches Python's Unicode-aware `re.findall(r"\w+", ...)`.
+ */
+export class ExactMatchScorer extends BaseScorer {
+    static WORD_PATTERN = /[\p{L}\p{N}_]+/gu;
+    get name() {
+        return 'reference_match';
+    }
+    async score({ result, referenceAnswer }) {
+        if (!referenceAnswer) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 0,
+                unit: 'percent',
+                details: { evaluated: false },
+                reasoning: 'No reference answer provided for matching',
+            });
+        }
+        if (!result.success || !result.output) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 0,
+                unit: 'percent',
+                details: { exact_match: false, token_overlap: 0 },
+                reasoning: 'Runner did not produce valid output',
+            });
+        }
+        const target = referenceAnswer.trim();
+        const actual = result.output.trim();
+        if (actual === target) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 100,
+                unit: 'percent',
+                details: { exact_match: true, token_overlap: 1 },
+                reasoning: 'Exact verbatim match with reference answer',
+            });
+        }
+        if (actual.toLowerCase() === target.toLowerCase()) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 95,
+                unit: 'percent',
+                details: { exact_match: false, case_insensitive_match: true },
+                reasoning: 'Case-insensitive match with reference answer',
+            });
+        }
+        if (actual.toLowerCase().includes(target.toLowerCase())) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 80,
+                unit: 'percent',
+                details: { substring_match: true },
+                reasoning: 'Reference answer is fully contained in output',
+            });
+        }
+        const targetWords = new Set(target.toLowerCase().match(ExactMatchScorer.WORD_PATTERN) ?? []);
+        const actualWords = new Set(actual.toLowerCase().match(ExactMatchScorer.WORD_PATTERN) ?? []);
+        let overlap = 0;
+        if (targetWords.size > 0) {
+            let intersection = 0;
+            for (const word of targetWords)
+                if (actualWords.has(word))
+                    intersection++;
+            overlap = intersection / targetWords.size;
+        }
+        return makeScore({
+            scorer_name: this.name,
+            score: round(overlap * 70, 1),
+            unit: 'percent',
+            details: { exact_match: false, token_overlap_ratio: round(overlap, 3) },
+            reasoning: `Token overlap ratio: ${(overlap * 100).toFixed(1)}% with reference answer`,
+        });
+    }
+}
+/** Uses a designated LLM runner as a judge to score outputs against a rubric. */
+export class LLMJudgeScorer extends BaseScorer {
+    judgeRunner;
+    static DEFAULT_RUBRIC = 'Evaluate the response on a scale from 0 to 100 based on:\n' +
+        '1. Correctness & Accuracy (40%)\n' +
+        '2. Completeness & Code Quality (30%)\n' +
+        '3. Conciseness & Clarity (20%)\n' +
+        '4. Strict adherence to instructions (10%)\n';
+    rubric;
+    constructor(judgeRunner, rubric) {
+        super();
+        this.judgeRunner = judgeRunner;
+        this.rubric = rubric || LLMJudgeScorer.DEFAULT_RUBRIC;
+    }
+    get name() {
+        return `llm_judge (${this.judgeRunner.id})`;
+    }
+    async score({ task, result, referenceAnswer }) {
+        if (!result.success) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 0,
+                unit: 'points',
+                details: { judge_model: this.judgeRunner.model },
+                reasoning: 'Output failed execution, 0 points assigned',
+            });
+        }
+        const refSection = referenceAnswer ? `\n\n[Optional Reference Answer]:\n${referenceAnswer}` : '';
+        const judgePrompt = `You are an impartial, expert AI benchmark judge.
+Your role is to evaluate an LLM's response to a given user task according to the scoring rubric.
+
+[Evaluation Rubric]:
+${this.rubric}
+
+[User Task/Prompt]:
+${task}
+${refSection}
+
+[Candidate Response to Evaluate]:
+${result.output}
+
+Provide your evaluation strictly in valid JSON format with the following keys:
+- "score": A numeric score from 0.0 to 100.0 (float).
+- "strengths": A short list of strengths (strings).
+- "weaknesses": A short list of weaknesses (strings).
+- "critique": A 1-2 sentence overall summary of the evaluation.
+
+Respond ONLY with valid JSON, without any markdown code fence wrappers if possible.
+`;
+        try {
+            const judgeResult = await this.judgeRunner.execute(judgePrompt, { temperature: 0.1, max_tokens: 512 });
+            if (!judgeResult.success) {
+                return makeScore({
+                    scorer_name: this.name,
+                    score: 50,
+                    unit: 'points',
+                    details: { error: judgeResult.error_message },
+                    reasoning: `Judge failed to evaluate: ${judgeResult.error_message}`,
+                });
+            }
+            let cleaned = judgeResult.output.trim();
+            if (cleaned.startsWith('```')) {
+                cleaned = cleaned.replace(/^```(?:json)?\s*/, '').replace(/\s*```$/, '');
+            }
+            const parsed = JSON.parse(cleaned);
+            const rawScore = 'score' in parsed ? Number(parsed['score']) : 50;
+            if (!Number.isFinite(rawScore))
+                throw new Error(`could not convert score to float: ${String(parsed['score'])}`);
+            const scoreValue = Math.max(0, Math.min(100, rawScore));
+            const critique = 'critique' in parsed ? String(parsed['critique']) : `Judge assigned ${scoreValue}/100`;
+            return makeScore({
+                scorer_name: this.name,
+                score: scoreValue,
+                unit: 'points',
+                details: {
+                    judge_runner: this.judgeRunner.id,
+                    judge_model: this.judgeRunner.model,
+                    strengths: parsed['strengths'] ?? [],
+                    weaknesses: parsed['weaknesses'] ?? [],
+                    critique: 'critique' in parsed ? parsed['critique'] : '',
+                },
+                reasoning: critique,
+            });
+        }
+        catch (error) {
+            return makeScore({
+                scorer_name: this.name,
+                score: 50,
+                unit: 'points',
+                details: { parse_error: errorMessage(error) },
+                reasoning: `Judge output parsing failed: ${errorMessage(error)}`,
+            });
+        }
+    }
+}
+//# sourceMappingURL=scorers.js.map

package/dist/benchmark/scorers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers.js","sourceRoot":"","sources":["../../src/benchmark/scorers.ts"],"names":[],"mappings":"AAAA,sDAAsD;AAGtD,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAiBjD,SAAS,SAAS,CAAC,IAMlB;IACC,OAAO;QACL,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,QAAQ;QAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;QAC3B,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;KAClC,CAAC;AACJ,CAAC;AAUD,iDAAiD;AACjD,MAAM,OAAgB,UAAU;CAK/B;AAED,oEAAoE;AACpE,MAAM,OAAO,aAAc,SAAQ,UAAU;IAC3C,IAAa,IAAI;QACf,OAAO,SAAS,CAAC;IACnB,CAAC;IAEQ,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAgB;QAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,IAAI;gBACV,OAAO,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,iBAAiB,EAAE,CAAC,EAAE;gBAChE,SAAS,EAAE,kBAAkB;aAC9B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,IAAI,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YACtD,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,SAAS,CAAC;YACf,WAAW,EAAE,IAAI,CAAC,IAAI;YACtB,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,IAAI,EAAE,IAAI;YACV,OAAO,EAAE;gBACP,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,iBAAiB,EAAE,YAAY;aAChC;YACD,SAAS,EAAE,gBAAgB,MAAM,CAAC,UAAU,SAAS,YAAY,cAAc;SAChF,CAAC,CAAC;IACL,CAAC;CACF;AAED,qDAAqD;AACrD,MAAM,OAAO,UAAW,SAAQ,UAAU;IACxC,IAAa,IAAI;QACf,OAAO,MAAM,CAAC;IAChB,CAAC;IAEQ,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAgB;QAC3C,MAAM,WAAW,GAAG,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC;QAE/D,OAAO,SAAS,CAAC;YACf,WAAW,EAAE,IAAI,CAAC,IAAI;YACtB,KAAK,EAAE,MAAM,CAAC,kBAAkB;YAChC,IAAI,EAAE,KAAK;YACX,OAAO,EAAE;gBACP,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,YAAY,EAAE,WAAW;gBACzB,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;aAC9C;YACD,SAAS,EAAE,UAAU,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,WAAW,eAAe;SAC/F,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACtC,MAAM,CAAU,YAAY,GAAG,kBAAkB,CAAC;IAE1D,IAAa,IAAI;QACf,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAEQ,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,eAAe,EAAgB;QAC5D,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;gBAC7B,SAAS,EAAE,2CAA2C;aACvD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACtC,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAAE;gBACjD,SAAS,EAAE,qCAAqC;aACjD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAEpC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,GAAG;gBACV,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE;gBAChD,SAAS,EAAE,4CAA4C;aACxD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YAClD,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,sBAAsB,EAAE,IAAI,EAAE;gBAC7D,SAAS,EAAE,8CAA8C;aAC1D,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACxD,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE;gBAClC,SAAS,EAAE,+CAA+C;aAC3D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7F,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QAE7F,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,IAAI,IAAI,WAAW;gBAAE,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,YAAY,EAAE,CAAC;YAC1E,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC;QAC5C,CAAC;QAED,OAAO,SAAS,CAAC;YACf,WAAW,EAAE,IAAI,CAAC,IAAI;YACtB,KAAK,EAAE,KAAK,CAAC,OAAO,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,mBAAmB,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE;YACvE,SAAS,EAAE,wBAAwB,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB;SACvF,CAAC,CAAC;IACL,CAAC;CACF;AAED,iFAAiF;AACjF,MAAM,OAAO,cAAe,SAAQ,UAAU;IAWjC,WAAW;IAVtB,MAAM,CAAU,cAAc,GAC5B,4DAA4D;QAC5D,mCAAmC;QACnC,wCAAwC;QACxC,kCAAkC;QAClC,6CAA6C,CAAC;IAEvC,MAAM,CAAS;IAExB,YACW,WAAuB,EAChC,MAAsB;QAEtB,KAAK,EAAE,CAAC;2BAHC,WAAW;QAIpB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,cAAc,CAAC,cAAc,CAAC;IACxD,CAAC;IAED,IAAa,IAAI;QACf,OAAO,cAAc,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC;IAC9C,CAAC;IAEQ,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAgB;QAClE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;gBAChD,SAAS,EAAE,4CAA4C;aACxD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,CAAC,CAAC,qCAAqC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEjG,MAAM,WAAW,GAAG;;;;EAItB,IAAI,CAAC,MAAM;;;EAGX,IAAI;EACJ,UAAU;;;EAGV,MAAM,CAAC,MAAM;;;;;;;;;CASd,CAAC;QAEE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YAEvG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,OAAO,SAAS,CAAC;oBACf,WAAW,EAAE,IAAI,CAAC,IAAI;oBACtB,KAAK,EAAE,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,aAAa,EAAE;oBAC7C,SAAS,EAAE,6BAA6B,WAAW,CAAC,aAAa,EAAE;iBACpE,CAAC,CAAC;YACL,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;YAE9D,MAAM,QAAQ,GAAG,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;YAChH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,UAAU,MAAM,CAAC;YAExG,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,UAAU;gBACjB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE;oBACP,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE;oBACjC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK;oBACnC,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE;oBACpC,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE;oBACtC,QAAQ,EAAE,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE;iBACzD;gBACD,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,SAAS,CAAC;gBACf,WAAW,EAAE,IAAI,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE,WAAW,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE;gBAC7C,SAAS,EAAE,gCAAgC,YAAY,CAAC,KAAK,CAAC,EAAE;aACjE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/http-api.d.ts

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+/** Fastify HTTP API for SmartRelay MCP delegation server.
+ *
+ * Provides an authenticated HTTP interface to all MCP tools, allowing external
+ * plugins (MCPHub, etc.) to call SmartRelay over HTTP instead of stdio.
+ *
+ * Transport: HTTP/JSON with Bearer token auth.
+ */
+import { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fastify';
+import { BenchmarkEngine } from './benchmark/engine.js';
+import { TaskRouter } from './router.js';
+import { RunnerRegistry } from './runners/registry.js';
+export declare function getRegistry(configPath?: string | null): RunnerRegistry;
+export declare function getEngine(): BenchmarkEngine;
+export declare function getRouter(): TaskRouter;
+export declare function verifyApiKey(request: FastifyRequest, reply: FastifyReply): void;
+export declare function dispatchTool(toolName: string, args?: Record<string, unknown>): Promise<unknown>;
+export declare function createHttpServer(): FastifyInstance;
+export declare function runHttpServer(options?: {
+    config?: string;
+    host?: string;
+    port?: number;
+}): Promise<void>;