@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/matrix/index.d.ts

@@ -1,16 +1,34 @@
+import { M as MatrixResult, a as MatrixAxis, A as AxisSummary, b as MatrixCell, C as CellResult, R as RunAgentMatrixOptions } from '../types-BUxNaJ8c.js';
+export { D as DefaultVerdict } from '../verdict-C9MlYujm.js';
+
 /**
- * @tangle-network/agent-eval/matrix
+ * Per-axis aggregation of cell runs into `AxisSummary` rows.
  *
- * N-axis cartesian runner over substrate types. The matrix is a runner +
- * aggregator: cells carry caller-supplied values (AgentProfile from
- * agent-interface, Driver / Validator from agent-runtime, rubric records, anything), and the
- * runner returns per-axis pass / score / cost / duration summaries.
+ * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
+ * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
+ * and duration always count — the budget was spent regardless.
+ */
+
+interface Row<Output> {
+    cell: MatrixCell;
+    result: CellResult<Output>;
+}
+declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
+declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
+
+/**
+ * N-axis cartesian runner.
+ *
+ * Expansion order: cartesian over `axes` in declared order, then `reps` as the
+ * inner-most dim → `ordinal = (cartIdx * reps) + rep`. The returned
+ * `cells[]` is sorted by `ordinal` so concurrent execution does not reorder
+ * the output.
  *
- * No "Spec" / "Wrapper" / "Config" types around substrate. The only
- * matrix-owned types are orchestration types: `MatrixAxis<V>`, `MatrixCell`,
- * `CellResult`, `AxisSummary`, `MatrixResult`.
+ * Scheduling is a sliding window of in-flight promises capped at
+ * `maxConcurrency`. The window stops admitting new cells when the cost
+ * ceiling trips or the abort signal fires; in-flight cells finish.
  */
-export { buildByAxis, summariseRows } from './aggregation';
-export { runAgentMatrix } from './runner';
-export type { AxisSummary, CellResult, DefaultVerdict, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, } from './types';
-//# sourceMappingURL=index.d.ts.map
+
+declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
+
+export { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };

package/dist/meta-eval/index.d.ts

@@ -1,6 +1,182 @@
-export * from './calibration';
-export * from './correlation-study';
-export * from './outcome-store';
-export * from './rubric-predictive-validity';
-export * from './sentinel';
-//# sourceMappingURL=index.d.ts.map
+export { C as CalibrationBin, a as CalibrationOptions, b as CalibrationPair, c as CalibrationReport, d as CorrelationResult, e as CorrelationStudyOptions, f as CorrelationStudyResult, E as EvalMetricSpec, O as OutcomePair, g as calibrationCurve, h as calibrationFromPairs, i as correlationStudy } from '../calibration-BPmzuVPk.js';
+export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeFilter, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
+export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-C2hDKM8Z.js';
+import { C as ContinuousAgreement, a as CalibrationResult, b as ContinuousCalibrationResult, c as CandidateScore, G as GoldenItem } from '../judge-calibration-0p2QcWNE.js';
+import { S as SeriesConvergenceOptions, a as SeriesConvergenceResult } from '../series-convergence-D5OWMBg6.js';
+import { C as CorpusAgreementReport } from '../statistics-CCJpTGOS.js';
+import '../store-BcFXE6LG.js';
+import '../schema-m0gsnbt3.js';
+import '../run-record-CP2ObebC.js';
+import '@tangle-network/agent-interface';
+import '../errors-CzMUYo7b.js';
+import '../types-C7DGg5ex.js';
+import '@tangle-network/tcloud';
+
+/**
+ * Judge sentinel — eval trustworthiness as a continuously measured,
+ * alarmed trend.
+ *
+ * Judges are models; models change underneath us; calibration decays.
+ * This module composes three existing instruments into one loop:
+ *
+ *   snapshot — adapters turn real instrument outputs (`calibrateJudge` /
+ *              `calibrateJudgeContinuous`, `corpusInterRaterAgreement` /
+ *              `continuousAgreement`, a rerun sentinel golden set) into
+ *              `SentinelSnapshot`s
+ *   series   — `analyzeSeries` (src/series-convergence) classifies each
+ *              judge×metric history as stabilized / drifting / noisy
+ *   alarm    — `judgeSentinelReport` turns trends + thresholds into named
+ *              alarms; `evalHealthStamp` is the tiny object a campaign
+ *              attaches to its verdicts
+ *
+ * Alarm conditions:
+ *   - convergence state `drifting-down` on any tracked metric
+ *   - irr below `minIrr`
+ *   - calibrationKappa / sentinelPassRate dropped vs the series baseline
+ *     beyond `maxKappaDrop` / `maxSentinelDrop`
+ *   - judge model changed with no post-change golden-grounded snapshot
+ *     (the silent-upgrade trap: judges agreeing with each other after a
+ *     model swap proves nothing — only re-measuring against gold does)
+ *   - newest snapshot older than `staleAfterDays` relative to `asOf`
+ *
+ * Wire-in: run the snapshot adapters from a post-campaign hook or a
+ * nightly job, append to a `SentinelStore`, then compute
+ * `judgeSentinelReport(await store.history(), { asOf })` with `asOf` =
+ * the campaign timestamp. Attach `evalHealthStamp(report)` to campaign
+ * verdicts: an alarmed sentinel means every downstream verdict carries
+ * an integrity warning until the judge is recalibrated.
+ *
+ * The module is clock-free — every timestamp (`at`, `asOf`) is
+ * caller-supplied ISO-8601, so reports are reproducible.
+ */
+
+declare const SENTINEL_METRIC_NAMES: readonly ["irr", "calibrationKappa", "sentinelPassRate"];
+type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
+interface SentinelMetrics {
+    /** Inter-rater reliability — ICC(2,1) from agreement instruments. */
+    irr?: number;
+    /** Weighted κ vs the human golden set (`calibrateJudge*`). */
+    calibrationKappa?: number;
+    /** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
+    sentinelPassRate?: number;
+}
+interface SentinelSnapshot {
+    /** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
+    at: string;
+    judgeId: string;
+    /** Exact model identity behind the judge (pin the full version string). */
+    judgeModel: string;
+    /**
+     * Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
+     * model-change marker — it records the new `judgeModel` immediately and
+     * the silent-upgrade alarm stays raised until a golden-grounded snapshot
+     * (calibrationKappa or sentinelPassRate) follows.
+     */
+    metrics: SentinelMetrics;
+}
+/** Identity + timestamp the caller supplies alongside an instrument output. */
+interface SnapshotMeta {
+    at: string;
+    judgeId: string;
+    judgeModel: string;
+}
+/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
+declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
+/**
+ * From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
+ * un-rounded continuous κ_w when the report carries one (integer κ
+ * discards information for [0,1] judges).
+ */
+declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
+/**
+ * From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
+ * (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
+ * coefficient both instruments compute. For ensemble-level agreement,
+ * `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
+ * composition (e.g. a joined list of member model strings).
+ */
+declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
+interface SentinelSetOptions {
+    /** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
+    tolerance?: number;
+}
+/**
+ * From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
+ * judge was originally calibrated on, re-scored periodically. Pass rate =
+ * fraction of joined items where the judge stays within `tolerance` of the
+ * human score. Items the judge didn't score are excluded from the
+ * denominator; zero joined items is an error, not a 0% pass rate.
+ */
+declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
+interface SentinelStore {
+    append(snapshot: SentinelSnapshot): Promise<void>;
+    /** All snapshots (optionally for one judge), in append order. */
+    history(judgeId?: string): Promise<SentinelSnapshot[]>;
+}
+declare function inMemorySentinelStore(initial?: SentinelSnapshot[]): SentinelStore;
+/**
+ * JSONL store — one snapshot per line, appended atomically per call.
+ * A corrupt or shape-invalid line is a loud error naming the file and
+ * line number: a sentinel history that silently drops records would
+ * defeat the drift detection it exists to provide.
+ */
+declare function fileSentinelStore(path: string): SentinelStore;
+interface SentinelThresholds {
+    /** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
+    minIrr?: number;
+    /** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
+    maxKappaDrop?: number;
+    /** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
+    maxSentinelDrop?: number;
+    /** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
+    staleAfterDays?: number;
+}
+interface JudgeSentinelOptions {
+    /** Caller-supplied ISO timestamp staleness is measured against. */
+    asOf: string;
+    thresholds?: SentinelThresholds;
+    /** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
+    convergence?: SeriesConvergenceOptions;
+}
+interface SentinelTrend {
+    judgeId: string;
+    metric: SentinelMetricName;
+    /** Verbatim `analyzeSeries` state for this judge×metric history. */
+    state: SeriesConvergenceResult['state'];
+    /** Newest value in the series. */
+    current: number;
+    /** Oldest value in the series — the reference the drop thresholds compare against. */
+    baseline: number;
+    /** current − baseline (signed; negative = decayed). */
+    drift: number;
+    alarmed: boolean;
+    reason?: string;
+}
+interface SentinelReport {
+    perJudge: SentinelTrend[];
+    alarms: string[];
+    healthy: boolean;
+    /**
+     * `judgeId:metric` pairs with too few snapshots for the convergence
+     * machine. Named, not counted — a blind spot you can't see is a blind
+     * spot you won't fix. Floor/drop checks still apply to thin series, so
+     * insufficient history alone never masks an alarm.
+     */
+    insufficientHistory: string[];
+}
+declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
+interface EvalHealthStamp {
+    healthy: boolean;
+    alarms: string[];
+}
+/**
+ * The object a campaign attaches to its verdicts. Compute it from the
+ * sentinel report in a post-campaign hook (or a nightly job feeding the
+ * next day's campaigns) and store it alongside the verdict payload.
+ * `healthy: false` means the judges that produced those verdicts have an
+ * unresolved drift / decay / silent-upgrade / staleness alarm — treat the
+ * verdicts as carrying an integrity warning until recalibration clears it.
+ */
+declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
+
+export { type EvalHealthStamp, type JudgeSentinelOptions, SENTINEL_METRIC_NAMES, type SentinelMetricName, type SentinelMetrics, type SentinelReport, type SentinelSetOptions, type SentinelSnapshot, type SentinelStore, type SentinelThresholds, type SentinelTrend, type SnapshotMeta, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };

package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts}

@@ -1,3 +1,5 @@
+import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
+
 /**
  * Multi-layer verifier — ordered pipeline of verification layers.
  *
@@ -21,10 +23,10 @@
  * Both primitives compose — JudgeRunner can be invoked as a single
  * layer inside a MultiLayerVerifier if that suits the caller.
  */
-import type { DefaultVerdict } from './verdict';
-export type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
-export type Severity = 'critical' | 'major' | 'minor' | 'info';
-export interface Finding {
+
+type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
+type Severity = 'critical' | 'major' | 'minor' | 'info';
+interface Finding {
     severity: Severity;
     message: string;
     evidence?: string;
@@ -37,7 +39,7 @@ export interface Finding {
      */
     detail?: Record<string, unknown>;
 }
-export interface LayerResult {
+interface LayerResult {
     layer: string;
     status: LayerStatus;
     /** 0..1 score, optional — layers that don't produce a numeric score omit. */
@@ -58,7 +60,7 @@ export interface LayerResult {
     /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
     detail?: Record<string, unknown>;
 }
-export interface VerifyContext<Env = unknown> {
+interface VerifyContext<Env = unknown> {
     /** Per-run opaque context the caller provides. Layers destructure what they need. */
     env: Env;
     /** Previously-computed results from layers that already ran. */
@@ -66,7 +68,7 @@ export interface VerifyContext<Env = unknown> {
     /** Signal — if aborted, layers MUST bail within reasonable wall. */
     signal: AbortSignal;
 }
-export interface Layer<Env = unknown> {
+interface Layer<Env = unknown> {
     name: string;
     /** Stages that must have `status: 'pass'` before this layer runs. */
     dependsOn?: string[];
@@ -85,7 +87,7 @@ export interface Layer<Env = unknown> {
     capMs?: number;
     run: (ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
 }
-export interface VerifyOptions<Env = unknown> {
+interface VerifyOptions<Env = unknown> {
     env: Env;
     /**
      * Overall wall cap. Default: sum of layer capMs, or Infinity if any layer
@@ -98,7 +100,7 @@ export interface VerifyOptions<Env = unknown> {
 /** Extends the substrate verdict spine: `valid` = `allPass` and `score` =
  *  `blendedScore` — derived where the report is aggregated, so spine
  *  consumers (drivers, gates) read this report without an adapter. */
-export interface VerificationReport extends DefaultVerdict {
+interface VerificationReport extends DefaultVerdict {
     layers: LayerResult[];
     passCount: number;
     failCount: number;
@@ -126,7 +128,7 @@ export interface VerificationReport extends DefaultVerdict {
  * overall score being >= 0.7. Now we trust the judge's own `severity` field:
  * `critical` findings veto; `major`/`minor` reduce the score but don't veto.
  */
-export declare function gradeSemanticStatus(input: {
+declare function gradeSemanticStatus(input: {
     score: number;
     findings: Array<{
         severity: Severity;
@@ -136,9 +138,10 @@ export declare function gradeSemanticStatus(input: {
     available: boolean;
     threshold?: number;
 }): LayerStatus;
-export declare class MultiLayerVerifier<Env = unknown> {
+declare class MultiLayerVerifier<Env = unknown> {
     private readonly layers;
     constructor(layers: Layer<Env>[]);
     run(opts: VerifyOptions<Env>): Promise<VerificationReport>;
 }
-//# sourceMappingURL=multi-layer-verifier.d.ts.map
+
+export { type Finding as F, type LayerResult as L, MultiLayerVerifier as M, type Severity as S, type VerifyOptions as V, type VerificationReport as a, type Layer as b, type VerifyContext as c, type LayerStatus as d, gradeSemanticStatus as g };

package/dist/multishot/index.d.ts

@@ -1,7 +1,290 @@
-export { createCodeExecutor, createResearchExecutor, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, defaultDelegationTools, } from './default-tools';
-export { DEFAULT_JUDGE_MODEL, type JudgeConfig, type JudgeDimension, type JudgeScore, renderDimensions, renderJsonFooter, runJudge, } from './judges';
-export { type ArtifactJudgeInput, type CellCompositeInput, type CellCompositeScore, type ConversationJudgeInput, computeCellComposite, type MultishotJudges, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, runMultishotMatrix, } from './matrix';
-export { type RunMultishotOptions, runMultishot } from './multishot';
-export { defaultRouterBaseUrl, estimateRouterCost, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, requireRouterApiKey, routerCompletion, } from './router';
-export { type MultishotArtifact, MultishotDriverEmptyError, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, } from './types';
-//# sourceMappingURL=index.d.ts.map
+import { J as JudgeScore } from '../types-DQRY8ZT-.js';
+import { AgentProfile } from '@tangle-network/agent-interface';
+import { M as MatrixResult } from '../types-BUxNaJ8c.js';
+import '../run-record-CP2ObebC.js';
+import '../errors-CzMUYo7b.js';
+import '../schema-m0gsnbt3.js';
+import '../verdict-C9MlYujm.js';
+
+interface MultishotMessage {
+    role: 'user' | 'assistant' | 'tool';
+    content: string;
+    toolCallId?: string;
+    toolCalls?: Array<{
+        id: string;
+        name: string;
+        args: Record<string, unknown>;
+    }>;
+}
+interface MultishotArtifact {
+    type: string;
+    turn: number;
+    invocation: {
+        name: string;
+        args: Record<string, unknown>;
+    };
+    content: string;
+}
+interface MultishotResult {
+    transcript: MultishotMessage[];
+    artifacts: MultishotArtifact[];
+    toolCalls: number;
+    durationMs: number;
+    costUsd: number;
+}
+interface MultishotToolDefinition {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: Record<string, unknown>;
+    };
+}
+type MultishotToolExecutor = (args: Record<string, unknown>, ctx: {
+    apiKey: string;
+    baseUrl: string;
+    signal?: AbortSignal;
+}) => Promise<{
+    content: string;
+    costUsd: number;
+}>;
+interface MultishotPersona {
+    /** Stable identifier — used for per-cell artifact paths + matrix axis keys. */
+    id: string;
+    /** Per-domain payload (income/profile/voice/etc.) shaped by the consumer. */
+    [k: string]: unknown;
+}
+interface MultishotShape<TPersona extends MultishotPersona> {
+    /** Opening user message (turn 0) — the persona's first ask. */
+    buildOpener: (persona: TPersona) => string;
+    /** System prompt the driver LLM uses to roleplay the persona. Should set
+     *  voice, goals, constraints, time-pressure, and the "never go silent" rule. */
+    buildDriverSystemPrompt: (persona: TPersona) => string;
+}
+declare class MultishotDriverEmptyError extends Error {
+    readonly turn: number;
+    constructor(turn: number);
+}
+
+interface RouterCompletionRequest {
+    apiKey: string;
+    baseUrl: string;
+    model: string;
+    messages: Array<Record<string, unknown>>;
+    tools?: MultishotToolDefinition[];
+    temperature?: number;
+    maxTokens?: number;
+    signal?: AbortSignal;
+}
+interface RouterToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+}
+interface RouterCompletionResponse {
+    message: {
+        content?: string | null;
+        tool_calls?: RouterToolCall[];
+    };
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+    };
+}
+declare function routerCompletion(req: RouterCompletionRequest): Promise<RouterCompletionResponse>;
+declare function estimateRouterCost(model: string, usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+}): number;
+declare function defaultRouterBaseUrl(): string;
+declare function requireRouterApiKey(): string;
+
+declare const DEFAULT_RESEARCHER_MODEL = "openai/gpt-4o-mini";
+declare const DEFAULT_CODER_MODEL = "openai/gpt-4o-mini";
+interface DefaultResearcherConfig {
+    /** Replace the system prompt to bias the researcher toward a domain's
+     *  citation style. Defaults to a generic "cite sources by name" prompt. */
+    systemPrompt?: string;
+    model?: string;
+}
+interface DefaultCoderConfig {
+    /** Replace the system prompt to bias the coder toward a language /
+     *  framework / artifact style. */
+    systemPrompt?: string;
+    model?: string;
+}
+declare const DEFAULT_DELEGATE_RESEARCH_TOOL: MultishotToolDefinition;
+declare const DEFAULT_DELEGATE_CODE_TOOL: MultishotToolDefinition;
+declare function createResearchExecutor(config?: DefaultResearcherConfig): MultishotToolExecutor;
+declare function createCodeExecutor(config?: DefaultCoderConfig): MultishotToolExecutor;
+interface DefaultToolsConfig {
+    research?: DefaultResearcherConfig;
+    code?: DefaultCoderConfig;
+    /** When true (default), each tool result is recorded as a typed artifact:
+     *  research → type='research', code → type='code'. */
+    recordArtifacts?: boolean;
+}
+interface DefaultToolsBundle {
+    tools: MultishotToolDefinition[];
+    executors: Record<string, MultishotToolExecutor>;
+    artifactTypeFor: (toolName: string) => string | undefined;
+}
+declare function defaultDelegationTools(config?: DefaultToolsConfig): DefaultToolsBundle;
+
+declare const DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
+interface JudgeDimension {
+    /** JSON field name + score key. */
+    key: string;
+    /** Description shown in the judge's user prompt. */
+    description: string;
+}
+interface JudgeConfig<TInput> {
+    /** Display name (for trace + log). */
+    name: string;
+    /** Model used for this judge. */
+    model?: string;
+    /** 0-10 scored dimensions. */
+    dimensions: JudgeDimension[];
+    /** Judge system prompt — sets persona + JSON-only constraint. */
+    systemPrompt: string;
+    /** Build the user prompt from the typed input. Must include "Respond with
+     *  ONLY this JSON: { ... }" listing each dimension key. */
+    buildPrompt: (input: TInput) => string;
+    /** Optional model + api overrides. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+declare function runJudge<TInput>(judge: JudgeConfig<TInput>, input: TInput): Promise<JudgeScore>;
+/** Convenience: stringified dimension list for inclusion in a judge prompt.
+ *  Returns lines like `- audience_fit: Does this match what the audience cares about? (0-10)`. */
+declare function renderDimensions(dims: readonly JudgeDimension[]): string;
+/** Convenience: build the "Respond with ONLY this JSON" footer for a judge prompt. */
+declare function renderJsonFooter(dims: readonly JudgeDimension[]): string;
+
+interface ConversationJudgeInput<TPersona extends MultishotPersona> {
+    transcript: MultishotMessage[];
+    persona: TPersona;
+}
+interface ArtifactJudgeInput<TPersona extends MultishotPersona> {
+    artifact: MultishotArtifact;
+    persona: TPersona;
+}
+interface MultishotJudges<TPersona extends MultishotPersona> {
+    /** Scores the full transcript end-to-end (always runs). */
+    conversation: JudgeConfig<ConversationJudgeInput<TPersona>>;
+    /** Scores each code-type artifact. Optional — omit when domain has no code artifacts. */
+    codeReview?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Scores each non-code (research/content/template) artifact. Optional. */
+    contentQuality?: JudgeConfig<ArtifactJudgeInput<TPersona>>;
+    /** Which artifact types route to codeReview. Defaults to ['code']. */
+    codeArtifactTypes?: string[];
+    /** Which artifact types route to contentQuality. Defaults to ['research']. */
+    contentArtifactTypes?: string[];
+}
+interface CellCompositeScore {
+    composite: number;
+    conversation: JudgeScore;
+    codeReview?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+    contentQuality?: {
+        perArtifact: Array<JudgeScore & {
+            turn: number;
+            type: string;
+        }>;
+        composite: number;
+    };
+}
+interface RunMultishotMatrixOptions<TPersona extends MultishotPersona> {
+    /** AgentProfile axis (matrix primary). */
+    profiles: Array<{
+        id: string;
+        value: AgentProfile;
+    }>;
+    /** Persona axis. */
+    personas: TPersona[];
+    /** Persona-shaping callbacks. */
+    shape: MultishotShape<TPersona>;
+    /** Judge configurations. */
+    judges: MultishotJudges<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → inline executor. Must align with `tools`. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Tool name → artifact type label. Defaults to research/code mapping. */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    /** Where per-cell artifacts land. Cells write to `<runDir>/<profileId>/<personaId>/rep-N/`. */
+    runDir: string;
+    /** Replicates per (profile, persona) cell. */
+    reps?: number;
+    /** Max conversation turns per cell. */
+    maxTurns?: number;
+    /** Max concurrent cells. */
+    maxConcurrency?: number;
+    /** Total $ ceiling across the matrix; cells aborted past this. */
+    costCeiling?: number;
+    /** Agent model. */
+    agentModel?: string;
+    /** Driver model. */
+    driverModel?: string;
+    /** Pass-thru fields. */
+    apiKey?: string;
+    baseUrl?: string;
+}
+interface CellOutput {
+    turns: number;
+    toolCalls: number;
+    artifactCount: number;
+}
+interface CellCompositeInput {
+    conversation: JudgeScore;
+    /** Present iff the codeReview judge is configured. */
+    codeReviews?: ReadonlyArray<JudgeScore>;
+    /** Present iff the contentQuality judge is configured. */
+    contentReviews?: ReadonlyArray<JudgeScore>;
+}
+/** Cell composite = mean over configured judge slots, excluding failed
+ *  scores: a failed conversation judge or an all-failed artifact slot carries
+ *  no signal and is dropped from the mean. `composite` is 0 only when EVERY
+ *  configured slot failed (`allJudgesFailed` distinguishes that from a real
+ *  zero). Pure — exported for deterministic testing. */
+declare function computeCellComposite(input: CellCompositeInput): {
+    composite: number;
+    codeComposite: number;
+    contentComposite: number;
+    allJudgesFailed: boolean;
+};
+interface RunMultishotMatrixResult {
+    matrix: MatrixResult<CellOutput>;
+}
+declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
+
+interface RunMultishotOptions<TPersona extends MultishotPersona> {
+    profile: AgentProfile;
+    persona: TPersona;
+    shape: MultishotShape<TPersona>;
+    /** Tool definitions advertised to the agent. Defaults to delegate_research + delegate_code. */
+    tools?: MultishotToolDefinition[];
+    /** Map from tool name → executor invoked inline when the agent emits a tool_call. */
+    toolExecutors?: Record<string, MultishotToolExecutor>;
+    /** Map from tool name → artifact type label written into MultishotArtifact.type.
+     *  Tools without a mapping still execute, but their results aren't surfaced as
+     *  typed artifacts (only as tool messages in the transcript). */
+    artifactTypeFor?: (toolName: string) => string | undefined;
+    maxTurns?: number;
+    agentModel?: string;
+    driverModel?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    signal?: AbortSignal;
+}
+declare function runMultishot<TPersona extends MultishotPersona>(opts: RunMultishotOptions<TPersona>): Promise<MultishotResult>;
+
+export { type ArtifactJudgeInput, type CellCompositeInput, type CellCompositeScore, type ConversationJudgeInput, DEFAULT_CODER_MODEL, DEFAULT_DELEGATE_CODE_TOOL, DEFAULT_DELEGATE_RESEARCH_TOOL, DEFAULT_JUDGE_MODEL, DEFAULT_RESEARCHER_MODEL, type DefaultCoderConfig, type DefaultResearcherConfig, type DefaultToolsBundle, type DefaultToolsConfig, type JudgeConfig, type JudgeDimension, JudgeScore, type MultishotArtifact, MultishotDriverEmptyError, type MultishotJudges, type MultishotMessage, type MultishotPersona, type MultishotResult, type MultishotShape, type MultishotToolDefinition, type MultishotToolExecutor, type RouterCompletionRequest, type RouterCompletionResponse, type RouterToolCall, type RunMultishotMatrixOptions, type RunMultishotMatrixResult, type RunMultishotOptions, computeCellComposite, createCodeExecutor, createResearchExecutor, defaultDelegationTools, defaultRouterBaseUrl, estimateRouterCost, renderDimensions, renderJsonFooter, requireRouterApiKey, routerCompletion, runJudge, runMultishot, runMultishotMatrix };