@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/multishot/index.d.ts

@@ -1,579 +0,0 @@
-import { AgentProfile } from '@tangle-network/agent-interface';
-
-interface MultishotMessage {
-    role: 'user' | 'assistant' | 'tool';
-    content: string;
-    toolCallId?: string;
-    toolCalls?: Array<{
-        id: string;
-        name: string;
-        args: Record<string, unknown>;
-    }>;
-}
-interface MultishotArtifact {
-    type: string;
-    turn: number;
-    invocation: {
-        name: string;
-        args: Record<string, unknown>;
-    };
-    content: string;
-}
-interface MultishotResult {
-    transcript: MultishotMessage[];
-    artifacts: MultishotArtifact[];
-    toolCalls: number;
-    durationMs: number;
-    costUsd: number;
-}
-interface MultishotToolDefinition {
-    type: 'function';
-    function: {
-        name: string;
-        description: string;
-        parameters: Record<string, unknown>;
-    };
-}
-/** One chat-completion request the multishot loop issues for a single agent
- *  (or driver) inference step. Mirrors the OpenAI-compat body the loop would
- *  otherwise POST to the Tangle router. */
-interface MultishotTransportRequest {
-    model: string;
-    messages: Array<Record<string, unknown>>;
-    tools?: MultishotToolDefinition[];
-    temperature?: number;
-    maxTokens?: number;
-    signal?: AbortSignal;
-}
-interface MultishotTransportToolCall {
-    id: string;
-    type: 'function';
-    function: {
-        name: string;
-        arguments: string;
-    };
-}
-interface MultishotTransportResponse {
-    message: {
-        content?: string | null;
-        tool_calls?: MultishotTransportToolCall[];
-    };
-    usage?: {
-        prompt_tokens?: number;
-        completion_tokens?: number;
-    };
-    /** Actual spend for this call. When omitted, the loop meters cost from
-     *  `usage` via the per-model router estimator (estimateRouterCost). */
-    costUsd?: number;
-}
-/** Execution seam for one leg of the multishot loop. When provided, it
- *  replaces the internal router HTTP call for that leg — the loop still owns
- *  turn scheduling, tool dispatch, transcript capture, and cost metering.
- *  agent-eval has no dependency on agent-runtime; adapt agent-runtime's
- *  resolveAgentBackend (or any sandbox/cli-bridge/router client) into this
- *  signature product-side. */
-type MultishotTransport = (req: MultishotTransportRequest) => Promise<MultishotTransportResponse>;
-type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
-    apiKey: string;
-    baseUrl: string;
-    signal?: AbortSignal;
-}) => Promise<{
-    content: string;
-    costUsd: number;
-}>;
-interface MultishotPersona {
-    /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
-    id: string;
-    /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
-    [k: string]: unknown;
-}
-interface MultishotShape<TPersona extends MultishotPersona> {
-    /** Opening user message (turn 0) — the persona's first ask. */
-    buildOpener: (persona: TPersona) => string;
-    /** System prompt the driver LLM uses to roleplay the persona. Should set
-     *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
-    buildDriverSystemPrompt: (persona: TPersona) => string;
-}
-declare class MultishotDriverEmptyError extends Error {
-    readonly turn: number;
-    constructor(turn: number);
-}
-declare class MultishotFatalToolError extends Error {
-    constructor(message: string);
-}
-
-interface RouterCompletionRequest {
-    apiKey: string;
-    baseUrl: string;
-    model: string;
-    messages: Array<Record<string, unknown>>;
-    tools?: MultishotToolDefinition[];
-    temperature?: number;
-    maxTokens?: number;
-    signal?: AbortSignal;
-}
-interface RouterToolCall {
-    id: string;
-    type: 'function';
-    function: {
-        name: string;
-        arguments: string;
-    };
-}
-interface RouterCompletionResponse {
-    message: {
-        content?: string | null;
-        tool_calls?: RouterToolCall[];
-    };
-    usage?: {
-        prompt_tokens?: number;
-        completion_tokens?: number;
-    };
-}
-declare function routerCompletion(req: RouterCompletionRequest): Promise<RouterCompletionResponse>;
-declare function estimateRouterCost(model: string, usage?: {
-    prompt_tokens?: number;
-    completion_tokens?: number;
-}): number;
-declare function defaultRouterBaseUrl(): string;
-declare function requireRouterApiKey(): string;
-
-declare const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
-declare const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
-interface DefaultResearcherConfig {
-    /** Replace the system prompt to bias the researcher toward a domain's
-     *  citation style. Defaults to a generic "cite sources by name" prompt. */
-    systemPrompt?: string;
-    model?: string;
-}
-interface DefaultCoderConfig {
-    /** Replace the system prompt to bias the coder toward a language /
-     *  framework / artifact style. */
-    systemPrompt?: string;
-    model?: string;
-}
-declare const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition;
-declare const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition;
-declare function createResearchExecutor(config?: DefaultResearcherConfig): MultishotToolExecutor;
-declare function createCodeExecutor(config?: DefaultCoderConfig): MultishotToolExecutor;
-interface DefaultToolsConfig {
-    research?: DefaultResearcherConfig;
-    code?: DefaultCoderConfig;
-    /** When true (default), each tool result is recorded as a typed artifact:
-     *  research → type='research', code → type='code'. */
-    recordArtifacts?: boolean;
-}
-interface DefaultToolsBundle {
-    tools: MultishotToolDefinition[];
-    executors: Record<string, MultishotToolExecutor>;
-    artifactTypeFor: (toolName: string) => string | undefined;
-}
-declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToolsBundle;
-
-/**
- * LLM client with graceful degrade.
- *
- * OpenAI-compatible `/v1/chat/completions` client with:
- *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
- *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
- *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
- *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
- *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
- *     directly, cli-bridge subscriptions, and any router that speaks the spec.
- *
- * Usage:
- *   const { value, result } = await callLlmJson<MyType>(
- *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
- *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
- *   )
- *
- * This is THE llm-calling seam for agent-eval primitives that need structured
- * output (semantic concept judge, reviewer directives, critic scores). Primitives
- * that need free-form text use `callLlm` and parse output themselves.
- */
-
-interface LlmUsage {
-    promptTokens: number;
-    completionTokens: number;
-    totalTokens: number;
-    /** False when the provider omitted or malformed prompt/completion usage. */
-    captured?: boolean;
-    /** Proxies populate this when prompt caching is on. */
-    cachedPromptTokens?: number;
-}
-interface LlmCallResult {
-    /** The text content of the first choice. Empty string if none. */
-    content: string;
-    usage: LlmUsage;
-    /**
-     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
-     * `null` when neither the proxy nor the caller can derive it.
-     */
-    costUsd: number | null;
-    /** Model name actually used (echoed from response). */
-    model: string;
-    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
-    durationMs: number;
-    /**
-     * `finish_reason` echoed from the first choice (`stop`, `length`,
-     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
-     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
-     * (`length`) instead of treating a cut-off completion as complete. Note:
-     * `callLlm` does not itself reject on it — acting on this signal is the
-     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
-     */
-    finishReason?: string | null;
-    /**
-     * True when `content.trim()` is empty. An empty completion is a silent zero
-     * for free-form `callLlm` callers; this flag is the signal a caller can
-     * inspect to fail loud rather than proceed on an empty string. `callLlm`
-     * surfaces it but does not throw on it.
-     */
-    contentEmpty?: boolean;
-    /** Raw response body. */
-    raw: Record<string, unknown>;
-}
-type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
-
-/**
- * Pass A substrate types — `runCampaign` is the one primitive every
- * eval flow composes from. Three contracts in this file:
- *
- *   - `Scenario`            input set
- *   - `DispatchFn`          how to run one scenario → artifact
- *   - `CampaignResult`      defined output schema (the contract downstream tools depend on)
- *
- * Three more lifted from earlier substrate work (re-exported):
- *
- *   - `JudgeConfig`         pluggable dimensional scorer (0.38)
- *   - `Mutator`             optimization-loop surface mutator
- *   - `Gate`                promotion gate (`HeldOutGate` and friends adapt to this)
- *
- * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
- * can build dashboards / CI gates / regression diffs against a stable schema.
- */
-
-/** The canonical judge verdict shape — one declaration, shared by campaign
- *  judges and the multishot judge runner (which re-exports this type).
- *
- *  Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
- *  multishot runner emits 0-10. Cross-scale comparison must go through
- *  `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
- *  promotion-policy) — never renormalize a producer's values in place, as
- *  downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
- *  `>= 7` gates) key on the producer's native scale. */
-interface JudgeScore {
-    dimensions: Record<string, number>;
-    composite: number;
-    notes: string;
-    /** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
-    llmCall?: LlmCallMetadata;
-    /** Set when the judge itself failed (call error, unparseable output).
-     *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
-     *  failed scores from means instead of folding them into zeros. */
-    failed?: true;
-    /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
-     *  spread across surviving judges — the inter-rater signal. */
-    maxDisagreement?: number;
-    /** Ensemble extras: judge identities whose verdict failed. */
-    failedJudges?: string[];
-    /** Ensemble extras: each surviving judge's per-dimension scores. */
-    perJudge?: Record<string, Record<string, number>>;
-}
-
-declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
-interface JudgeDimension {
-    /** JSON field name + score key. */
-    key: string;
-    /** Description shown in the judge's user prompt. */
-    description: string;
-}
-interface JudgeConfig<TInput> {
-    /** Display name (for trace + log). */
-    name: string;
-    /** Model used for this judge. */
-    model?: string;
-    /** 0-10 scored dimensions. */
-    dimensions: JudgeDimension[];
-    /** Judge system prompt — sets persona + JSON-only constraint. */
-    systemPrompt: string;
-    /** Build the user prompt from the typed input. Must include "Respond with
-     *  ONLY this JSON: { ... }" listing each dimension key. */
-    buildPrompt: (input: TInput) => string;
-    /** Optional model + api overrides. */
-    apiKey?: string;
-    baseUrl?: string;
-    /** Maximum output tokens for the judge response. Defaults to 1500. */
-    maxTokens?: number;
-}
-declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeScore>;
-/** Convenience: stringified dimension list for inclusion in a judge prompt.
- *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
-declare function renderDimensions(dims: readonly JudgeDimension[]): string;
-/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
-declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
-
-/**
- * Validator-output verdict — substrate primitive for "did this output pass,
- * and how well?"
- *
- * Used by:
- *   - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
- *   - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
- *     Runtime keeps `Validator` because it's coupled to runtime-shaped
- *     `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
- *     itself is a substrate concept and lives here.
- *
- * Repo layering: agent-eval is the substrate (no upward deps). Both
- * agent-runtime and agent-knowledge consume this type FROM agent-eval —
- * never the other way around. See CLAUDE.md "Repo layering" for the rule.
- */
-/**
- * Minimal verdict shape — `valid` + `score` are required; `scores` +
- * `notes` are optional surface. Validators that need richer shapes
- * parameterise `Validator<Output, MyVerdict>` with their own type.
- *
- * Need structured extras? Extend DefaultVerdict with typed fields — never
- * serialize extras into `notes`.
- */
-interface DefaultVerdict {
-    /** Whether the output meets the validator's pass criteria. */
-    valid: boolean;
-    /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
-    score: number;
-    /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
-    scores?: Record<string, number>;
-    /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
-    notes?: string;
-}
-
-/**
- * N-axis cartesian matrix over substrate types — types module.
- *
- * The matrix is a runner + aggregator. It iterates the cartesian product of
- * caller-provided axes (any value type — `AgentProfile` from agent-interface,
- * `Driver` / `Validator` from agent-runtime, rubric records, thinking levels, anything)
- * and aggregates per-axis pass/score/cost summaries. Substrate types are
- * imported at the boundary by JSDoc only; the matrix never wraps them.
- */
-
-/** A cell carries one picked value from each axis, keyed by axis name. */
-interface MatrixCell {
-    axes: Record<string, {
-        id: string;
-        value: unknown;
-    }>;
-    /** 0-based replicate index within the same axis combination. */
-    rep: number;
-    /** Stable sort key — preserves cartesian order across concurrent execution. */
-    ordinal: number;
-}
-interface CellResult<Output> {
-    output: Output;
-    verdict: DefaultVerdict;
-    costUsd: number;
-    durationMs: number;
-    runId?: string;
-    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
-     *  meanScore regardless of `verdict`. */
-    error?: {
-        message: string;
-        kind: string;
-    };
-}
-interface AxisSummary {
-    axisName: string;
-    axisValue: string;
-    cells: number;
-    passRate: number;
-    meanScore: number;
-    p50Score: number;
-    p90Score: number;
-    totalCostUsd: number;
-    meanDurationMs: number;
-}
-interface MatrixResult<Output> {
-    cells: Array<{
-        cell: MatrixCell;
-        runs: CellResult<Output>[];
-    }>;
-    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
-     *  named in `aggregateBy` (default = every axis in `axes`). */
-    byAxis: Record<string, Record<string, AxisSummary>>;
-    summary: {
-        totalCells: number;
-        runsExecuted: number;
-        /** Cells removed by `filter` plus cells unscheduled after the cost
-         *  ceiling or abort signal tripped. */
-        cellsSkipped: number;
-        overallPassRate: number;
-        overallMeanScore: number;
-        totalCostUsd: number;
-        durationMs: number;
-    };
-    /** Stable id-like string generated at the end of the run. */
-    matrixId: string;
-}
-
-interface ConversationJudgeInput<TPersona extends MultishotPersona> {
-    transcript: MultishotMessage[];
-    persona: TPersona;
-}
-interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
-    artifact: MultishotArtifact;
-    persona: TPersona;
-}
-interface MultishotJudges<TPersona extends MultishotPersona> {
-    /** Scores the full transcript end-to-end (always runs). */
-    conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
-    /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
-    codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
-    /** Scores each non-code (research/content/template) artifact. Optional. */
-    contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
-    /** Which artifact types route to codeReview. Defaults to ['code']. */
-    codeArtifactTypes?: string[];
-    /** Which artifact types route to contentQuality. Defaults to ['research']. */
-    contentArtifactTypes?: string[];
-}
-interface CellCompositeScore {
-    composite: number;
-    conversation: JudgeScore;
-    codeReview?: {
-        perArtifact: Array<JudgeScore & {
-            turn: number;
-            type: string;
-        }>;
-        composite: number;
-    };
-    contentQuality?: {
-        perArtifact: Array<JudgeScore & {
-            turn: number;
-            type: string;
-        }>;
-        composite: number;
-    };
-}
-interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
-    /** AgentProfile axis (matrix primary). */
-    profiles: Array<{
-        id: string;
-        value: AgentProfile;
-    }>;
-    /** Persona axis. */
-    personas: TPersona[];
-    /** Persona-shaping callbacks. */
-    shape: MultishotShape<TPersona>;
-    /** Judge configurations. */
-    judges: MultishotJudges<TPersona>;
-    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
-    tools?: MultishotToolDefinition[];
-    /** Map from tool name → inline executor. Must align with `tools`. */
-    toolExecutors?: Record<string, MultishotToolExecutor>;
-    /** Tool name → artifact type label. Defaults to research/code mapping. */
-    artifactTypeFor?: (toolName: string) => string | undefined;
-    /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
-    runDir: string;
-    /** Replicates per (profile, persona) cell. */
-    reps?: number;
-    /** Max conversation turns per cell. */
-    maxTurns?: number;
-    /** Maximum tool calls the agent may dispatch inside one assistant turn. */
-    maxToolDispatches?: number;
-    /** Max concurrent cells. */
-    maxConcurrency?: number;
-    /** Total $ ceiling across the matrix; cells aborted past this. */
-    costCeiling?: number;
-    /** Agent model. */
-    agentModel?: string;
-    /** Driver model. */
-    driverModel?: string;
-    /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
-    driverFallbackModels?: string[];
-    /** Maximum output tokens for the first agent call in each assistant turn. */
-    agentMaxTokens?: number;
-    /** Maximum output tokens for agent follow-up calls after tool results. */
-    toolFollowupMaxTokens?: number;
-    /** Maximum output tokens for each simulated-user driver response. */
-    driverMaxTokens?: number;
-    /** Maximum output tokens for each judge response. */
-    judgeMaxTokens?: number;
-    /** Execution seam for the agent leg of every cell — replaces the router
-     *  HTTP call when provided (see RunMultishotOptions.agentTransport).
-     *  Judges are unaffected; configure those via MultishotJudges. */
-    agentTransport?: MultishotTransport;
-    /** Execution seam for the simulated-user driver leg of every cell. */
-    driverTransport?: MultishotTransport;
-    /** Pass-thru fields. */
-    apiKey?: string;
-    baseUrl?: string;
-}
-interface CellOutput {
-    turns: number;
-    toolCalls: number;
-    artifactCount: number;
-}
-interface CellCompositeInput {
-    conversation: JudgeScore;
-    /** Present iff the codeReview judge is configured. */
-    codeReviews?: ReadonlyArray<JudgeScore>;
-    /** Present iff the contentQuality judge is configured. */
-    contentReviews?: ReadonlyArray<JudgeScore>;
-}
-/** Cell composite = mean over configured judge slots, excluding failed
- *  scores: a failed conversation judge or an all-failed artifact slot carries
- *  no signal and is dropped from the mean. `composite` is 0 only when EVERY
- *  configured slot failed (`allJudgesFailed` distinguishes that from a real
- *  zero). Pure — exported for deterministic testing. */
-declare function computeCellComposite(input: CellCompositeInput): {
-    composite: number;
-    codeComposite: number;
-    contentComposite: number;
-    allJudgesFailed: boolean;
-};
-interface RunMultishotMatrixResult {
-    matrix: MatrixResult<CellOutput>;
-}
-declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
-
-interface RunMultishotOptions<TPersona extends MultishotPersona> {
-    profile: AgentProfile;
-    persona: TPersona;
-    shape: MultishotShape<TPersona>;
-    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
-    tools?: MultishotToolDefinition[];
-    /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
-    toolExecutors?: Record<string, MultishotToolExecutor>;
-    /** Map from tool name → artifact type label written into MultishotArtifact.type.
-     *  Tools without a mapping still execute, but their results aren't surfaced as
-     *  typed artifacts (only as tool messages in the transcript). */
-    artifactTypeFor?: (toolName: string) => string | undefined;
-    maxTurns?: number;
-    agentModel?: string;
-    driverModel?: string;
-    /** Fallback driver models tried when the primary simulated-user model returns empty twice. */
-    driverFallbackModels?: string[];
-    /** Maximum output tokens for the first agent call in each assistant turn. */
-    agentMaxTokens?: number;
-    /** Maximum output tokens for agent follow-up calls after tool results. */
-    toolFollowupMaxTokens?: number;
-    /** Maximum output tokens for each simulated-user driver response. */
-    driverMaxTokens?: number;
-    /** Maximum tool calls the agent may dispatch inside one assistant turn. */
-    maxToolDispatches?: number;
-    /** Execution seam for the agent leg. When provided, every agent inference
-     *  step goes through this function instead of the router HTTP call; the
-     *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
-     *  apiKey/baseUrl are still resolved for tool executors and any leg
-     *  without an injected transport. */
-    agentTransport?: MultishotTransport;
-    /** Execution seam for the simulated-user driver leg (symmetric to
-     *  agentTransport). Driver model fallback rotation still applies — the
-     *  transport receives each candidate model in turn. */
-    driverTransport?: MultishotTransport;
-    apiKey?: string;
-    baseUrl?: string;
-    signal?: AbortSignal;
-}
-declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
-
-export { type ArtifactJudgeInput, type CellCompositeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, type JudgeScore, type MultishotArtifact, MultishotDriverEmptyError, MultishotFatalToolError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, type MultishotTransport, type MultishotTransportRequest, type MultishotTransportResponse, type MultishotTransportToolCall, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };