@tangle-network/agent-eval 0.117.0 → 0.118.0

package/dist/types-BUxNaJ8c.d.ts

@@ -1,108 +0,0 @@
-import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
-
-/**
- * N-axis cartesian matrix over substrate types — types module.
- *
- * The matrix is a runner + aggregator. It iterates the cartesian product of
- * caller-provided axes (any value type — `AgentProfile` from agent-interface,
- * `Driver` / `Validator` from agent-runtime, rubric records, thinking levels, anything)
- * and aggregates per-axis pass/score/cost summaries. Substrate types are
- * imported at the boundary by JSDoc only; the matrix never wraps them.
- */
-
-/** One axis = one dimension to iterate. `V` is the value type — pass any
- *  substrate type (AgentProfile, Driver, Validator, rubric record). */
-interface MatrixAxis<V> {
-    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
-    name: string;
-    /** Stable id per value. Used as the bucket key in aggregation. */
-    values: Array<{
-        id: string;
-        value: V;
-    }>;
-    /** Optional bucket label override. Receives the same `(value, id)` the
-     *  runner stored on the cell; default label is `id`. */
-    label?: (value: V, id: string) => string;
-}
-/** A cell carries one picked value from each axis, keyed by axis name. */
-interface MatrixCell {
-    axes: Record<string, {
-        id: string;
-        value: unknown;
-    }>;
-    /** 0-based replicate index within the same axis combination. */
-    rep: number;
-    /** Stable sort key — preserves cartesian order across concurrent execution. */
-    ordinal: number;
-}
-interface CellResult<Output> {
-    output: Output;
-    verdict: DefaultVerdict;
-    costUsd: number;
-    durationMs: number;
-    runId?: string;
-    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
-     *  meanScore regardless of `verdict`. */
-    error?: {
-        message: string;
-        kind: string;
-    };
-}
-interface AxisSummary {
-    axisName: string;
-    axisValue: string;
-    cells: number;
-    passRate: number;
-    meanScore: number;
-    p50Score: number;
-    p90Score: number;
-    totalCostUsd: number;
-    meanDurationMs: number;
-}
-interface MatrixResult<Output> {
-    cells: Array<{
-        cell: MatrixCell;
-        runs: CellResult<Output>[];
-    }>;
-    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
-     *  named in `aggregateBy` (default = every axis in `axes`). */
-    byAxis: Record<string, Record<string, AxisSummary>>;
-    summary: {
-        totalCells: number;
-        runsExecuted: number;
-        /** Cells removed by `filter` plus cells unscheduled after the cost
-         *  ceiling or abort signal tripped. */
-        cellsSkipped: number;
-        overallPassRate: number;
-        overallMeanScore: number;
-        totalCostUsd: number;
-        durationMs: number;
-    };
-    /** Stable id-like string generated at the end of the run. */
-    matrixId: string;
-}
-interface RunAgentMatrixOptions<Output> {
-    axes: MatrixAxis<unknown>[];
-    /** User-supplied cell executor. May throw; the matrix captures throws as
-     *  `CellResult.error` and continues. */
-    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
-    /** Replicates per cell. Default 1. */
-    reps?: number;
-    /** Prune cells from the cartesian BEFORE rep expansion. */
-    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
-    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
-    aggregateBy?: string[];
-    /** Max concurrent in-flight `runCell` invocations. Default 4. */
-    maxConcurrency?: number;
-    /** Cumulative-cost abort threshold (USD). When the running sum of
-     *  `result.costUsd` crosses this value, no new cells are scheduled.
-     *  In-flight cells finish. Default `Infinity`. */
-    costCeiling?: number;
-    /** Fires once per executed cell, after its promise settles. */
-    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
-    /** External cancellation. Aborts in-flight cells via a forwarded signal
-     *  and suppresses scheduling of new ones. */
-    signal?: AbortSignal;
-}
-
-export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };

package/dist/types-BkfcQnxV.d.ts

@@ -1,313 +0,0 @@
-import { C as CostLedger, b as CostLedgerSummary } from './cost-ledger-DWy3XdJc.js';
-import { TCloud } from '@tangle-network/tcloud';
-
-interface Scenario {
-    id: string;
-    persona: string;
-    label: string;
-    thesis: string;
-    dimensions: string[];
-    turns: Turn[];
-    artifactChecks: ArtifactCheck[];
-    systemPromptAppend?: string;
-}
-interface Turn {
-    user: string;
-    expectedBehaviors: string[];
-    adversarial?: boolean;
-    feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
-}
-interface ArtifactCheck {
-    type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
-    target: string;
-    contains?: string;
-    minCount?: number;
-    description: string;
-}
-interface JudgeConfig {
-    model: string;
-    temperature: number;
-    rubric: JudgeRubric;
-}
-interface JudgeRubric {
-    name: string;
-    description: string;
-    dimensions: RubricDimension[];
-}
-interface RubricDimension {
-    name: string;
-    description: string;
-    anchor_low: string;
-    anchor_high: string;
-    weight: number;
-}
-interface ScenarioResult {
-    scenarioId: string;
-    persona: string;
-    turns: TurnResult[];
-    artifactResults: ArtifactResult[];
-    judgeScores: JudgeScore[];
-    judgeErrors: number;
-    overallScore: number;
-    totalDurationMs: number;
-    artifacts: CollectedArtifacts;
-    /** Agent and judge spend attributed to this scenario. */
-    cost?: CostLedgerSummary;
-}
-interface TurnResult {
-    turnIndex: number;
-    userMessage: string;
-    agentResponse: string;
-    durationMs: number;
-    blocksExtracted: {
-        type: string;
-        title: string;
-    }[];
-    containsCode: boolean;
-    containsToolCall: boolean;
-}
-interface ArtifactResult {
-    check: ArtifactCheck;
-    passed: boolean;
-    detail?: string;
-}
-interface JudgeScore {
-    judgeName: string;
-    dimension: string;
-    score: number;
-    reasoning: string;
-    evidence?: string;
-}
-interface CollectedArtifacts {
-    vaultFiles: {
-        path: string;
-        content: string;
-    }[];
-    blocksExtracted: {
-        type: string;
-        fields: Record<string, string>;
-    }[];
-    codeBlocks: {
-        language: string;
-        code: string;
-    }[];
-    toolCalls: string[];
-}
-interface BenchmarkReport {
-    timestamp: string;
-    generation: number;
-    promptVersion: string;
-    scenarioCount: number;
-    results: ScenarioResult[];
-    cost?: CostLedgerSummary;
-    summary: {
-        overallAvg: number;
-        byPersona: Record<string, {
-            avg: number;
-            passed: number;
-            total: number;
-        }>;
-        byDimension: Record<string, {
-            avg: number;
-            scores: number[];
-        }>;
-        weakest: {
-            scenario: string;
-            score: number;
-            reason: string;
-        }[];
-        strongest: {
-            scenario: string;
-            score: number;
-            reason: string;
-        }[];
-    };
-}
-interface RouteMap {
-    signup?: string;
-    login?: string;
-    workspaces?: string;
-    threads?: string;
-    chat?: string;
-    tasks?: string;
-    events?: string;
-    approvals?: string;
-    vault?: string;
-    generations?: string;
-    [key: string]: string | undefined;
-}
-interface ProductClientConfig {
-    baseUrl: string;
-    routes: RouteMap;
-    /** Per-request timeout in ms before the request is aborted. Default 30s. */
-    timeoutMs?: number;
-}
-interface ScenarioFile {
-    id: string;
-    category: string;
-    persona: string;
-    label: string;
-    thesis: string;
-    isControl?: boolean;
-    rubric?: {
-        dimensions: {
-            name: string;
-            description: string;
-            weight: number;
-        }[];
-    };
-    turns: Turn[];
-    artifactChecks: ArtifactCheck[];
-}
-interface CompletionCriterion {
-    name: string;
-    check: (state: DriverState) => boolean;
-    progress?: (state: DriverState) => number;
-}
-interface FeedbackPattern {
-    trigger: string;
-    response: string;
-}
-/**
- * How hard the simulated user pushes back. The driver LLM scales its tone
- * and follow-up aggression to this:
- *   cooperative — forgiving early adopter; accepts reasonable answers.
- *   demanding   — experienced professional; rejects vague or hedged answers.
- *   relentless  — senior partner reviewing for a client who will litigate;
- *                 interrogates every claim, accepts nothing undefended.
- */
-type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
-interface PersonaConfig {
-    id: string;
-    role: string;
-    goal: string;
-    completionCriteria: CompletionCriterion[];
-    feedbackPatterns?: FeedbackPattern[];
-    maxTurns: number;
-    driverModel?: string;
-    /** How adversarial the simulated user is. Defaults to 'demanding'. */
-    rigor?: PersonaRigor;
-    /**
-     * Domain expertise the simulated user holds — quoted into the driver
-     * prompt so it challenges the agent with authority instead of vague
-     * dissatisfaction. e.g. "a 15-year M&A partner who knows GAAP
-     * working-capital mechanics cold".
-     */
-    expertise?: string;
-    /**
-     * Substantive issues a senior professional in this role would
-     * interrogate — traps the scenario hides, claims that must be defended.
-     * The driver probes these without revealing them verbatim; the agent
-     * must surface them on its own.
-     */
-    pressurePoints?: string[];
-    /**
-     * Curveballs the driver may inject once the agent is coasting — changed
-     * facts, a hostile counterparty position, a new constraint. Forces the
-     * agent to re-derive rather than recite.
-     */
-    curveballs?: string[];
-}
-interface DriverState {
-    tasks: number;
-    events: number;
-    proposals: {
-        pending: number;
-        approved: number;
-        rejected: number;
-    };
-    vaultFiles: string[];
-    codeBlocks: number;
-    generations: number;
-}
-interface TurnMetrics {
-    turn: number;
-    timestamp: string;
-    tasks: number;
-    events: number;
-    proposals: {
-        pending: number;
-        approved: number;
-        rejected: number;
-    };
-    vaultFiles: number;
-    responseLatencyMs: number;
-    responseChars: number;
-    codeBlocksProduced: number;
-    blocksExtracted: number;
-    qualityScore?: number;
-    inputTokens: number;
-    outputTokens: number;
-    estimatedCostUsd: number;
-    totalCostUsd: number;
-    completionPercent: number;
-}
-interface DriverResult {
-    personaId: string;
-    /** True when the simulated user professionally signed off (driver said DONE). */
-    completed: boolean;
-    /** Turn at which the simulated user signed off, or null if it never did. */
-    turnsToCompletion: number | null;
-    /**
-     * Turn at which nominal completionCriteria were first all met, or null.
-     * Distinct from turnsToCompletion: criteria can be met while the
-     * simulated professional is still unsatisfied with the work's rigor.
-     */
-    criteriaMetAtTurn: number | null;
-    totalTurns: number;
-    metrics: TurnMetrics[];
-    finalState: DriverState;
-    convergenceCurve: number[];
-    totalCostUsd: number;
-    finalQualityScore: number | null;
-}
-interface BenchmarkRunnerConfig {
-    scenarios: Scenario[];
-    judges: JudgeFn[];
-    systemPrompt: string;
-    model?: string;
-    judgeModel?: string;
-    passThreshold?: number;
-    generation?: number;
-    promptVersion?: string;
-    /** Shared ledger for agent and judge calls made by the benchmark. */
-    costLedger?: CostLedger;
-    /** Exact maximum provider attempts configured on the supplied TCloud client. */
-    tcloudMaximumAttempts?: number;
-}
-interface JudgeInput {
-    scenario: Scenario;
-    turns: TurnResult[];
-    artifacts: CollectedArtifacts;
-    /** Shared ledger for paid built-in judges. Direct calls default to an uncapped ledger. */
-    costLedger?: CostLedger;
-    costPhase?: string;
-    costTags?: Record<string, string>;
-    signal?: AbortSignal;
-    /** Exact maximum provider attempts configured on the supplied TCloud client. */
-    tcloudMaximumAttempts?: number;
-}
-type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
-
-interface TestResult {
-    name: string;
-    passed: boolean;
-    duration: number;
-    detail?: string;
-    checks: CheckResult[];
-}
-interface CheckResult {
-    name: string;
-    passed: boolean;
-    expected: string;
-    actual: string;
-}
-interface EvalResult {
-    scenario: string;
-    status: 'pass' | 'fail' | 'skip';
-    duration: number;
-    detail?: string;
-    artifact?: string;
-}
-
-export type { ArtifactCheck as A, BenchmarkRunnerConfig as B, CheckResult as C, DriverResult as D, EvalResult as E, FeedbackPattern as F, JudgeScore as J, ProductClientConfig as P, RouteMap as R, Scenario as S, TestResult as T, JudgeInput as a, JudgeFn as b, BenchmarkReport as c, PersonaConfig as d, DriverState as e, CollectedArtifacts as f, ScenarioResult as g, TurnMetrics as h, ScenarioFile as i, CompletionCriterion as j, ArtifactResult as k, JudgeConfig as l, JudgeRubric as m, PersonaRigor as n, RubricDimension as o, Turn as p, TurnResult as q };

package/dist/verdict-C9MlYujm.d.ts

@@ -1,35 +0,0 @@
-/**
- * Validator-output verdict — substrate primitive for "did this output pass,
- * and how well?"
- *
- * Used by:
- *   - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
- *   - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
- *     Runtime keeps `Validator` because it's coupled to runtime-shaped
- *     `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
- *     itself is a substrate concept and lives here.
- *
- * Repo layering: agent-eval is the substrate (no upward deps). Both
- * agent-runtime and agent-knowledge consume this type FROM agent-eval —
- * never the other way around. See CLAUDE.md "Repo layering" for the rule.
- */
-/**
- * Minimal verdict shape — `valid` + `score` are required; `scores` +
- * `notes` are optional surface. Validators that need richer shapes
- * parameterise `Validator<Output, MyVerdict>` with their own type.
- *
- * Need structured extras? Extend DefaultVerdict with typed fields — never
- * serialize extras into `notes`.
- */
-interface DefaultVerdict {
-    /** Whether the output meets the validator's pass criteria. */
-    valid: boolean;
-    /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
-    score: number;
-    /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
-    scores?: Record<string, number>;
-    /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
-    notes?: string;
-}
-
-export type { DefaultVerdict as D };