@datalayer/core 1.1.66 → 1.2.10

package/lib/api/evals/types.d.ts

@@ -0,0 +1,1335 @@
+/**
+ * The evals objects as the AI Agents service sends them.
+ *
+ * These are the wire shapes, snake_case and all: the service's pydantic
+ * models (`EvalRecord`, `EvalExperimentRecord`, `EvalRunRecord`, …) are the
+ * source of truth and the landings UI reads these fields by their wire
+ * names. Nothing here is camel-cased on the way in — `metrics`, `summary`
+ * and `report` are free-form documents whose keys mean what the runner
+ * wrote, and a converter that renamed them would rename data.
+ *
+ * Product words: a **Benchmark** is an evalset, an **Experiment** is an
+ * experiment, a **Run** is a run, a **Task result** is a case result.
+ *
+ * @module api/evals/types
+ */
+/** Where an evalset was created, which decides where its runs may execute. */
+export type EvalRunEnvironment = 'ui' | 'sdk';
+/** Fixed cases replayed (`batch`) or live events evaluated (`interactive`). */
+export type EvalKind = 'batch' | 'interactive';
+/** What a benchmark measures, the closed list section 12.5 renders by. */
+export type EvalsetCategory = 'data' | 'coding' | 'tool-use' | 'model' | 'visual' | 'performance';
+/** The dataset revision a benchmark reads; contents holds the revision. */
+export interface DatasetRef {
+    source_uid: string;
+    revision_uid: string;
+}
+/** A named evaluator with its configuration, as a spec declares it. */
+export interface EvalEvaluatorRef {
+    name: string;
+    config?: Record<string, unknown>;
+    [key: string]: unknown;
+}
+/** One case of an evalset: what is asked and what is expected. */
+export interface EvalCase {
+    id: string;
+    name: string;
+    inputs: Record<string, unknown>;
+    expected_output: unknown;
+    evaluators: EvalEvaluatorRef[];
+    metadata: Record<string, unknown>;
+}
+/** An evalset, the benchmark: versioned cases, evaluators, metadata. */
+export interface Evalset {
+    id: string;
+    owner_uid: string;
+    name: string;
+    description: string;
+    run_environment: EvalRunEnvironment;
+    kind: EvalKind;
+    category: EvalsetCategory | '';
+    dataset_ref: DatasetRef | null;
+    /** Moves on every change to cases, evaluators, schema or category. */
+    version: number;
+    /** How many launches this benchmark has had; the next is number + 1. */
+    launch_count: number;
+    /** What the runs say, kept by the launch refresh (B2-10). */
+    run_count?: number;
+    latest_launch_id?: string;
+    latest_pass_rate?: number | null;
+    best_pass_rate?: number | null;
+    last_run_status?: string;
+    last_run_at?: string | null;
+    estimated_cost?: number | null;
+    environment?: string;
+    subject_refs?: string[];
+    schema: Record<string, unknown>;
+    evalset_evaluators: EvalEvaluatorRef[];
+    report_evaluators: EvalEvaluatorRef[];
+    tags: string[];
+    metadata: Record<string, unknown>;
+    cases: EvalCase[];
+    is_public: boolean;
+    /** The benchmark this one was taken from, and its version then (B5-03). */
+    derived_from_uid?: string;
+    derived_from_version?: number | null;
+    created_at: string;
+    updated_at: string;
+    archived: boolean;
+}
+/** One version of a benchmark's definition, kept as it was. */
+export interface EvalsetVersion {
+    evalset_id: string;
+    version: number;
+    name: string;
+    category: string;
+    schema: Record<string, unknown>;
+    evalset_evaluators: EvalEvaluatorRef[];
+    report_evaluators: EvalEvaluatorRef[];
+    cases: EvalCase[];
+    created_at: string | null;
+}
+/** One experiment: an agentspec, model, prompt or configuration under test. */
+/** What an experiment runs (B2-11): `{kind, ref, model?, ...}`. */
+export interface EvalSubject {
+    kind: string;
+    ref: string;
+    model?: string;
+    endpoint?: string;
+    image?: string;
+    command?: string;
+    provider?: string;
+}
+export interface EvalExperiment {
+    id: string;
+    owner_uid: string;
+    evalset_id: string | null;
+    name: string;
+    description: string;
+    status: string;
+    /** What the experiment runs (B2-11); empty on an experiment without one. */
+    subject?: EvalSubject | Record<string, never>;
+    config: Record<string, unknown>;
+    summary: Record<string, unknown>;
+    tags: string[];
+    created_at: string;
+    updated_at: string;
+    archived: boolean;
+}
+/** A case's outcome inside a run's metrics. */
+export interface EvalCaseResult {
+    name: string;
+    passed: boolean;
+    status: string;
+    score: number;
+    category?: string;
+    difficulty?: string;
+    prompt?: string;
+    output?: string;
+    [key: string]: unknown;
+}
+/** An evaluator's aggregate inside a run's metrics. */
+export interface EvalEvaluatorResult {
+    name: string;
+    scope: 'evalset' | 'report' | string;
+    score?: number | null;
+    passed?: boolean | null;
+    passed_cases?: number;
+    total_cases?: number;
+    summary?: string;
+    threshold?: number;
+    observed?: number;
+    [key: string]: unknown;
+}
+/** What the runner measured for a run. Free-form beyond the known keys. */
+export interface EvalRunMetrics {
+    pass_rate?: number;
+    total_cases?: number;
+    passed?: number;
+    failed?: number;
+    avg_score?: number;
+    case_results?: EvalCaseResult[];
+    evaluator_results?: EvalEvaluatorResult[];
+    [key: string]: unknown;
+}
+/** One execution of an experiment, inside a launch. */
+export interface EvalRun {
+    id: string;
+    experiment_id: string;
+    /** The evalset the experiment belongs to; empty on rows not yet backfilled. */
+    evalset_id: string;
+    /** The launch this run is part of; every run has one once backfilled. */
+    launch_id: string;
+    /** Credits consumed so far (B2-06). */
+    cost_credits?: number | null;
+    /** The credits this run may spend; null is no cap (B2-06). */
+    budget_limit?: number | null;
+    /** The compute asked for: environment, slots, concurrency, time_reservation (B2-06). */
+    compute?: Record<string, unknown>;
+    elapsed_ms?: number | null;
+    /** Why the run is `blocked`, when it is (B2-06). */
+    blocked_reason?: string;
+    /** The durable workflow executing it, once one does. */
+    workflow_uid: string;
+    /** The evalset version the run was started against; null on rows not yet backfilled. */
+    evalset_version?: number | null;
+    owner_uid: string;
+    status: string;
+    started_at: string | null;
+    ended_at: string | null;
+    metrics: EvalRunMetrics;
+    summary: Record<string, unknown>;
+    report: Record<string, unknown>;
+    /** Cases done, running and failed so far, as the workflow projects them. */
+    progress: Record<string, unknown>;
+    created_at: string;
+    updated_at: string;
+}
+/** A task result: one case of one run, its own document. */
+export interface EvalTaskResult {
+    id: string;
+    run_id: string;
+    launch_id: string;
+    experiment_id: string;
+    evalset_id: string;
+    owner_uid: string;
+    case_id: string;
+    name: string;
+    status: string;
+    score: number | null;
+    passed: boolean | null;
+    explanation: string;
+    category: string;
+    difficulty: string;
+    failure_mode: string;
+    failure_stage: string;
+    prompt: string;
+    output: string;
+    trajectory: unknown[];
+    sandbox_snapshot_uid: string;
+    notebook_uid: string;
+    /** The investigation opened on this task, once one is (B3-02). */
+    investigation_uid?: string;
+    /**
+     * The root execution this task was delegated as — a team subject only; empty
+     * for an agentspec or a model, which are one call and no tree. Absent from a
+     * service older than the field.
+     */
+    execution_id?: string;
+    /** The trace that execution runs under (the W3C trace id), when there is one. */
+    trace_id?: string;
+    artifact_uids: string[];
+    cost_credits: number | null;
+    duration_ms: number | null;
+    reviewed_by: string;
+    started_at: string | null;
+    ended_at: string | null;
+    created_at: string | null;
+    updated_at: string | null;
+}
+/** A launch: one submission of a benchmark across its experiments. */
+export interface EvalLaunch {
+    id: string;
+    owner_uid: string;
+    account_uid: string;
+    evalset_id: string;
+    evalset_version: number | null;
+    /** The number people say: "Run 128". */
+    number: number | null;
+    experiment_ids: string[];
+    run_ids: string[];
+    status: string;
+    run_mode: EvalKind;
+    config: Record<string, unknown>;
+    progress: Record<string, unknown>;
+    created_by_uid: string;
+    /** The report document written for this launch, once one is (B3-03). */
+    report_document_uid?: string;
+    /** The credits the launch may spend; null is no cap (B2-06). */
+    budget_limit?: number | null;
+    /** The compute asked for: environment, slots, concurrency, time_reservation (B2-06). */
+    compute?: Record<string, unknown>;
+    /** Why the launch is `blocked`, when it is (B2-06). */
+    blocked_reason?: string;
+    /** The launch this one runs again, at the benchmark version it ran (B5-06). */
+    reproduces_launch_id?: string;
+    /**
+     * The window of live traffic an interactive launch is (B2-13):
+     * `{size, starts_at, ends_at}`. Empty on a batch launch.
+     */
+    window?: {
+        size?: string;
+        starts_at?: string;
+        ends_at?: string;
+    };
+    archived: boolean;
+    started_at: string | null;
+    ended_at: string | null;
+    created_at: string | null;
+    updated_at: string | null;
+}
+/** One problem the pre-launch validation found (B2-07). */
+export interface LaunchPlanProblem {
+    code: string;
+    severity: 'error' | 'warning';
+    message: string;
+    experiment_id?: string;
+}
+/** The plan of a launch before it is made (B2-07). */
+export interface LaunchPlanResponse {
+    success: boolean;
+    /** No error-level problem: the launch would run. */
+    ok: boolean;
+    /** The mode the plan was made for (B2-13). */
+    run_mode: EvalKind;
+    /**
+     * How run mode, run environment and execution target relate, stated by
+     * the service so the wizard explains it inline rather than failing after
+     * submission (BENCHMARK.md section 9).
+     */
+    run_mode_constraint: string;
+    problems: LaunchPlanProblem[];
+    estimate: {
+        cases: number;
+        experiments: number;
+        slots: number;
+        duration_seconds: number;
+        duration_basis: 'history' | 'assumed' | 'mixed';
+        /** Credits the launch would reserve; null when the burning rate is unknown. */
+        credits_reserved: number | null;
+        credits_available: number | null;
+        budget_limit: number | null;
+    };
+    compute: {
+        environment: string;
+        slots: number;
+        concurrency: number;
+        time_reservation: number;
+        burning_rate: number | null;
+    };
+    experiments: Array<{
+        id: string;
+        name: string;
+        subject: EvalSubject | Record<string, never>;
+        execution_target: string;
+        seconds_per_case: number;
+        duration_basis: 'history' | 'assumed';
+        duration_seconds: number;
+        credits_reserved: number | null;
+    }>;
+    approvals: {
+        pending_tool_approvals: number;
+        budget_limit: number | null;
+    };
+    unsupported_evaluators: string[];
+}
+/** What a claim moved, by document type; empty when it was a repeat (B2-14). */
+export interface ClaimTrialResponse {
+    success: boolean;
+    trial: Record<string, unknown> | null;
+    moved: Record<string, number>;
+}
+/** The subjects an experiment can have and the models offered (B2-11). */
+export interface SubjectsResponse {
+    success: boolean;
+    kinds: Array<{
+        kind: string;
+        executable: boolean;
+        label: string;
+    }>;
+    models: string[];
+    default_model: string;
+    default_provider: string;
+    /** False when AI Inference could not be asked. */
+    models_available: boolean;
+}
+/** One evaluated event of a live target. */
+export interface LiveEvalEvent {
+    id: string;
+    owner_uid: string;
+    target_id: string;
+    target_type: string;
+    evaluator_name: string;
+    metric_name: string;
+    value_num: number | null;
+    label: string;
+    passed: boolean | null;
+    attributes: Record<string, unknown>;
+    /** The experiment the event was produced for, graded with its evaluators (B2-13). */
+    experiment_id?: string;
+    evalset_id?: string;
+    /** The interactive launch whose window the event fell into, if one was open. */
+    launch_id?: string;
+    /** The case it answered, when it answered one. */
+    case_id?: string;
+    /** What the experiment's evaluators said of it, server-side. */
+    results?: Array<Record<string, unknown>>;
+    created_at: string;
+}
+/** What the rolling windows raised on a live target (B2-13). */
+export interface LiveEvalAlert {
+    id: string;
+    owner_uid: string;
+    target_id: string;
+    target_type: string;
+    experiment_id: string;
+    launch_id: string;
+    /** `failure_spike` or `drift`. */
+    kind: string;
+    message: string;
+    /** The failed share, or the pass rate that drifted. */
+    value: number | null;
+    /** The pass rate before the drift. */
+    baseline: number | null;
+    threshold: number | null;
+    window_events: number;
+    created_at: string | null;
+}
+/** A live target's rolling-window summary. */
+export interface LiveEvalTarget {
+    target_id: string;
+    target_type: string;
+    event_count: number;
+    passed_count: number;
+    failed_count?: number;
+    pass_rate: number | null;
+    avg_value: number | null;
+    last_event_at: string | null;
+    /** What the target's newest event is bound to (B2-13). */
+    experiment_id?: string;
+    evalset_id?: string;
+    launch_id?: string;
+    [key: string]: unknown;
+}
+export interface CreateEvalsetRequest {
+    name: string;
+    description?: string;
+    run_environment?: EvalRunEnvironment;
+    kind?: EvalKind;
+    category?: EvalsetCategory;
+    dataset_ref?: DatasetRef;
+    schema?: Record<string, unknown>;
+    evalset_evaluators?: EvalEvaluatorRef[];
+    report_evaluators?: EvalEvaluatorRef[];
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    cases?: Array<Partial<EvalCase>>;
+    is_public?: boolean;
+}
+/** Every field optional; `kind` is not updatable through the API. */
+export type UpdateEvalsetRequest = Partial<Omit<CreateEvalsetRequest, 'kind'>>;
+export interface CaseRequest {
+    name: string;
+    inputs?: Record<string, unknown>;
+    expected_output?: unknown;
+    evaluators?: EvalEvaluatorRef[];
+    metadata?: Record<string, unknown>;
+}
+export interface CreateExperimentRequest {
+    evalset_id?: string | null;
+    name: string;
+    description?: string;
+    status?: string;
+    config?: Record<string, unknown>;
+    summary?: Record<string, unknown>;
+    tags?: string[];
+}
+export type UpdateExperimentRequest = Partial<CreateExperimentRequest>;
+export interface CreateRunRequest {
+    status?: string;
+    started_at?: string | null;
+    ended_at?: string | null;
+    metrics?: Record<string, unknown>;
+    summary?: Record<string, unknown>;
+    report?: Record<string, unknown>;
+}
+export interface CreateLiveEventRequest {
+    target_id: string;
+    target_type?: string;
+    evaluator_name: string;
+    metric_name: string;
+    value_num?: number | null;
+    label?: string;
+    passed?: boolean | null;
+    attributes?: Record<string, unknown>;
+    created_at?: string | null;
+}
+export interface EvalsetResponse {
+    success: boolean;
+    evalset: Evalset;
+}
+/** One thing a published package holds, as the review enumerates it (B5-02). */
+export interface EvalsetPackageItem {
+    /** Which part of the package it belongs to: definition, results, report… */
+    section: string;
+    kind: string;
+    uid: string;
+    label: string;
+}
+/** What stands in the way of a publication (section 14.5). */
+export interface EvalsetPackageProblem {
+    code: string;
+    detail: string;
+    uid?: string;
+}
+/** What a package leaves out, and why: nothing is published by implication. */
+export interface EvalsetPackageExclusion {
+    kind: string;
+    uid: string;
+    reason: string;
+}
+/**
+ * The package a published benchmark is: a snapshot of everything the numbers
+ * depend on, what would be published, and what refuses it.
+ */
+export interface EvalsetPackage {
+    contents: Record<string, unknown>;
+    items: EvalsetPackageItem[];
+    problems: EvalsetPackageProblem[];
+    excluded: EvalsetPackageExclusion[];
+    checksum: string;
+}
+/** A package as it was written: immutable, and reversibly visible. */
+export interface EvalsetPublication {
+    id: string;
+    evalset_id: string;
+    evalset_version: number | null;
+    owner_uid: string;
+    actor_uid?: string;
+    report_id: string;
+    launch_ids: string[];
+    contents: Record<string, unknown>;
+    items: EvalsetPackageItem[];
+    checksum: string;
+    status: string;
+    note: string;
+    created_at?: string | null;
+    unpublished_at?: string | null;
+}
+/** One launch in a comparison of launches of the same benchmark (B5-08). */
+export interface EvalsetComparisonLaunch {
+    launch_id: string;
+    number: number | null;
+    status: string;
+    run_mode: string;
+    evalset_version: number | null;
+    started_at?: string | null;
+    ended_at?: string | null;
+    created_at?: string | null;
+    run_ids: string[];
+    pass_rate: number | null;
+    cost_credits: number | null;
+    elapsed_ms: number | null;
+    reproduces_launch_id: string;
+}
+/** What one subject scored in each of the launches compared. */
+export interface EvalsetComparisonSubject {
+    ref: string;
+    label: string;
+    kind: string;
+    pass_rates: Record<string, number | null>;
+    /** The last launch against the first; `null` where either has no rate. */
+    delta: number | null;
+}
+/** What one task did in each launch: what broke, and what was fixed. */
+export interface EvalsetComparisonTask {
+    name: string;
+    statuses: Record<string, string>;
+    scores: Record<string, number>;
+    regressed: boolean;
+    fixed: boolean;
+}
+export interface EvalsetComparisonDelta {
+    earlier_launch_id: string;
+    earlier: number | null;
+    later_launch_id: string;
+    later: number | null;
+    delta: number;
+}
+export interface EvalsetComparison {
+    launches: EvalsetComparisonLaunch[];
+    subjects: EvalsetComparisonSubject[];
+    tasks: EvalsetComparisonTask[];
+    deltas: EvalsetComparisonDelta[];
+    /** How a delta is read, stated rather than assumed. */
+    convention: string;
+    regressions: number;
+    fixes: number;
+}
+/** One place on a published benchmark's leaderboard (B5-08). */
+export interface EvalsetLeaderboardRow {
+    place: number;
+    ref: string;
+    label: string;
+    kind: string;
+    runs: number;
+    best_pass_rate: number | null;
+    latest_pass_rate: number | null;
+    best_run_id: string;
+    /** The package the best run was published in, so a reader can open it. */
+    publication_id: string;
+    evalset_version: number | null;
+}
+export interface EvalsetLeaderboard {
+    rows: EvalsetLeaderboardRow[];
+    /** Runs a package published; only these are counted. */
+    published_runs: number;
+    counted_runs: number;
+}
+/** A benchmark that has run one subject, with what its runs scored (B5-12). */
+export interface SubjectBenchmark {
+    evalset_id: string;
+    name: string;
+    category: string;
+    is_public: boolean;
+    runs: number;
+    latest_pass_rate: number | null;
+    best_pass_rate: number | null;
+    last_run_at: string | null;
+    cost_credits: number;
+}
+/** One run of a subject, as its page lists it. */
+export interface SubjectRun {
+    run_id: string;
+    evalset_id: string;
+    evalset_name: string;
+    launch_id: string;
+    experiment_id: string;
+    status: string;
+    pass_rate: number | null;
+    cost_credits: number | null;
+    elapsed_ms: number | null;
+    evalset_version: number | null;
+    ended_at?: string | null;
+    created_at?: string | null;
+}
+export interface SubjectScore {
+    at: string | null;
+    pass_rate: number;
+    evalset_id: string;
+    run_id: string;
+}
+export interface SubjectLaunch {
+    launch_id: string;
+    evalset_id: string;
+    number: number | null;
+    status: string;
+}
+export interface SubjectTotals {
+    benchmarks: number;
+    runs: number;
+    cost_credits: number | null;
+    elapsed_ms: number | null;
+    latest_pass_rate: number | null;
+    best_pass_rate: number | null;
+    active_launches: number;
+}
+/**
+ * What an agent or a model has actually run. Empty where it has never been
+ * run: the page says so rather than drawing a history nobody ran.
+ */
+export interface SubjectUsage {
+    subject_ref: string;
+    benchmarks: SubjectBenchmark[];
+    runs: SubjectRun[];
+    scores: SubjectScore[];
+    active_launches: SubjectLaunch[];
+    totals: SubjectTotals;
+}
+/** What a search made of the words somebody typed (B5-11, section 19). */
+export interface EvalsSearchQuery {
+    raw: string;
+    /** What is left to look for by name, once the words that mean something else are taken out. */
+    text: string;
+    /** A status word, as the records spell it: `failed`, `in_review`… */
+    status: string;
+    /** `run 128` names a launch. */
+    launch_number: number | null;
+    /** A dataset revision somebody named, such as `v2026.09`. */
+    revision: string;
+}
+export type EvalsSearchKind = 'benchmark' | 'launch' | 'task' | 'report' | 'investigation';
+/** One thing found, keeping its kind and what it belongs to. */
+export interface EvalsSearchRow {
+    kind: EvalsSearchKind | string;
+    uid: string;
+    title: string;
+    subtitle: string;
+    status: string;
+    score?: number | null;
+    is_public?: boolean;
+    evalset_id?: string;
+    launch_id?: string;
+    run_id?: string;
+    case_id?: string;
+    at?: string | null;
+}
+export interface EvalsSearchResponse {
+    success: boolean;
+    query: EvalsSearchQuery;
+    results: EvalsSearchRow[];
+    counts: Record<string, number>;
+    total: number;
+}
+export interface SubjectUsageResponse {
+    success: boolean;
+    usage: SubjectUsage;
+}
+/**
+ * What benchmark work a sandbox is doing (B5-12, section 15.4).
+ *
+ * A pool slot's sandbox is named after the run it serves, and a sandbox
+ * restored from a task's snapshot is found by that snapshot; either way the
+ * page can say which benchmark, launch, run and task the machine belongs to.
+ * `found` is false for a sandbox somebody opened themselves, which is the
+ * ordinary case and not an error.
+ */
+export interface SandboxProvenance {
+    found: boolean;
+    benchmark: {
+        id: string;
+        name: string;
+    } | null;
+    launch: {
+        id: string;
+        number: number | null;
+        status: string;
+    } | null;
+    run: {
+        id: string;
+        status: string;
+        experiment_id: string;
+        experiment_name: string;
+        /** Which of the pool's sandboxes this one is, where it is one. */
+        slot: number | null;
+    } | null;
+    task: {
+        case_id: string;
+        name: string;
+        status: string;
+        snapshot_uid: string;
+    } | null;
+    /** The investigation a restored sandbox was brought back for (B3-05). */
+    investigation: {
+        id: string;
+        title: string;
+        status: string;
+    } | null;
+    /** Paths of the app. */
+    links: Partial<Record<'benchmark' | 'launch' | 'run' | 'task' | 'investigation', string>>;
+}
+export interface SandboxProvenanceResponse {
+    success: boolean;
+    provenance: SandboxProvenance;
+}
+/**
+ * Whether anything will run the account's benchmarks (B5-12, section 15.2).
+ *
+ * A launch at `queued` looks the same whether the queue is busy or nothing
+ * polls it. `serving` is the difference. The depth is the account's own
+ * launches: the executor's other work belongs to nobody on this page.
+ */
+export interface BenchmarkCompute {
+    serving: boolean;
+    engine: string;
+    queue: string;
+    queue_served: boolean;
+    application_version: string;
+    /** Why nothing is serving, when nothing is. */
+    detail: string;
+    queued: number;
+    running: number;
+    /** The slots the running launches hold. */
+    sandboxes: number;
+}
+export interface BenchmarkComputeResponse {
+    success: boolean;
+    compute: BenchmarkCompute;
+}
+export interface EvalsetComparisonResponse {
+    success: boolean;
+    comparison: EvalsetComparison;
+}
+export interface EvalsetLeaderboardResponse {
+    success: boolean;
+    leaderboard: EvalsetLeaderboard;
+}
+export interface EvalsetPackagePreviewResponse {
+    success: boolean;
+    package: EvalsetPackage;
+}
+export interface EvalsetPublicationResponse {
+    success: boolean;
+    publication: EvalsetPublication;
+}
+export interface EvalsetPublicationListResponse {
+    success: boolean;
+    total: number;
+    publications: EvalsetPublication[];
+}
+export interface EvalsetListResponse {
+    success: boolean;
+    total: number;
+    evalsets: Evalset[];
+}
+export interface EvalsetDeleteResponse {
+    success: boolean;
+    cascade: {
+        experiments_deleted: number;
+        runs_deleted: number;
+        cases_deleted: number;
+    };
+}
+export interface CaseListResponse {
+    success: boolean;
+    total: number;
+    cases: EvalCase[];
+}
+export interface CaseResponse {
+    success: boolean;
+    case: EvalCase;
+}
+export interface PublicEvalsetDetailsResponse {
+    success: boolean;
+    evalset: Evalset;
+    cases: EvalCase[];
+    experiments: EvalExperiment[];
+    experiments_total: number;
+    runs_limit: number;
+    runs_by_experiment: Record<string, EvalRun[]>;
+    runs_total_by_experiment: Record<string, number>;
+}
+export interface ExperimentResponse {
+    success: boolean;
+    experiment: EvalExperiment;
+}
+export interface ExperimentListResponse {
+    success: boolean;
+    total: number;
+    experiments: EvalExperiment[];
+}
+export interface ExperimentDeleteResponse {
+    success: boolean;
+    cascade: {
+        runs_deleted: number;
+    };
+}
+export interface RunResponse {
+    success: boolean;
+    run: EvalRun;
+}
+export interface RunListResponse {
+    success: boolean;
+    total: number;
+    runs: EvalRun[];
+}
+export interface LiveEventResponse {
+    success: boolean;
+    event: LiveEvalEvent;
+}
+export interface LiveTargetListResponse {
+    success: boolean;
+    window: string;
+    targets: LiveEvalTarget[];
+}
+export interface LiveEventListResponse {
+    success: boolean;
+    window: string;
+    total: number;
+    events: LiveEvalEvent[];
+}
+export interface LiveEventCreateResponse {
+    success: boolean;
+    event: LiveEvalEvent;
+    /** The alerts this event's arrival raised, if any (B2-13). */
+    alerts: LiveEvalAlert[];
+}
+export interface LiveAlertListResponse {
+    success: boolean;
+    window: string;
+    alerts: LiveEvalAlert[];
+}
+export interface LiveTargetDeleteResponse {
+    success: boolean;
+    target_id: string;
+    target_type: string;
+    deleted_events: number;
+}
+export interface SuccessResponse {
+    success: boolean;
+}
+export interface CreateLaunchRequest {
+    experiment_ids: string[];
+    run_mode?: EvalKind;
+    config?: Record<string, unknown>;
+    /** The launch this one runs again, at the benchmark version it ran (B5-06). */
+    reproduces_launch_id?: string;
+}
+export interface ReviewCaseRequest {
+    status: 'passed' | 'failed';
+    explanation?: string;
+    failure_mode?: string;
+}
+export interface LaunchResponse {
+    success: boolean;
+    launch: EvalLaunch;
+    runs: EvalRun[];
+    /** Whether anything executes the queued runs yet. */
+    executes?: boolean;
+}
+export interface LaunchListResponse {
+    success: boolean;
+    total: number;
+    launches: EvalLaunch[];
+}
+export interface LaunchCancelResponse {
+    success: boolean;
+    launch: EvalLaunch;
+    cancelled_runs: number;
+}
+export interface RunCancelResponse {
+    success: boolean;
+    run: EvalRun;
+    execution_stopped: boolean;
+}
+export interface CaseResultListResponse {
+    success: boolean;
+    total: number;
+    cases: EvalTaskResult[];
+}
+/** What a message of a run's network says: never its payload. */
+export interface EvalNetworkSummary {
+    kind: 'text' | 'object' | 'list' | 'empty';
+    /** One scrubbed line of text. */
+    preview: string;
+    /** An object's field names, never their values. */
+    fields: string[];
+    items?: number;
+    bytes: number;
+    /** What was taken out, in words. */
+    redactions: string[];
+}
+export interface EvalNetworkMessage {
+    id: string;
+    from: string;
+    to: string;
+    type: 'delegation' | 'request' | 'result' | 'review' | 'evaluation';
+    taskId: string;
+    /** Milliseconds from the run's start. */
+    atMs: number;
+    durationMs: number;
+    status: 'delivered' | 'in-transit' | 'queued' | 'failed';
+    summary: EvalNetworkSummary;
+    toolCalls: Array<{
+        name: string;
+    }>;
+    links: Record<string, string>;
+    correlation: {
+        traceId?: string;
+        executionId?: string;
+        parentMessageId?: string;
+        protocolTaskId?: string;
+    };
+}
+export interface EvalNetworkTask {
+    id: string;
+    title: string;
+    category: string;
+    status: 'passed' | 'review' | 'running' | 'failed' | 'queued';
+    agentId: string;
+    startedMs: number;
+    endedMs: number;
+    tokens: number;
+    costUsd: number;
+    /** What the platform charged, which is credits and not dollars. */
+    costCredits: number;
+    failureMode?: string;
+    finding?: string;
+    links: Record<string, string>;
+}
+export interface EvalNetworkAgent {
+    id: string;
+    role: 'runner' | 'analyst' | 'reviewer' | 'evaluator' | 'agent';
+    name: string;
+    detail?: string;
+    protocol?: 'a2a' | 'acp' | 'datalayer';
+}
+/**
+ * A run as the network of agents that worked on it: who asked whom, about
+ * which task, when — with summaries and never payloads. Aggregated to a
+ * `budget` by the service, which says what it left out.
+ */
+export interface EvalRunNetwork {
+    run: {
+        id: string;
+        label: string;
+        benchmark: string;
+        demo: boolean;
+        status: 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
+        counts: Record<EvalNetworkTask['status'], number>;
+        startedAt: string;
+        durationMs: number;
+        tokens: number;
+        costUsd: number;
+        costCredits: number;
+    };
+    agents: EvalNetworkAgent[];
+    tasks: EvalNetworkTask[];
+    messages: EvalNetworkMessage[];
+    subject: {
+        kind: string;
+        ref: string;
+    };
+    /** A team's tasks that name no execution: stored before the field existed. */
+    untraced: number;
+    budget: number;
+    /** Messages of passed tasks left out to stay within the budget. */
+    messagesOmitted: number;
+    /** Tasks past the most a network reads. */
+    tasksOmitted: number;
+}
+/** One message of a run's network, in full: more parts, each a summary. */
+export interface RunNetworkMessageResponse {
+    success: boolean;
+    message: {
+        id: string;
+        parts: Array<{
+            label: string;
+            summary: EvalNetworkSummary;
+        }>;
+    };
+}
+/** A launch's runs, each as one line of its own network. */
+export interface EvalLaunchNetwork {
+    launchId: string;
+    status: string;
+    counts: Record<EvalNetworkTask['status'], number>;
+    runs: Array<{
+        runId: string;
+        experimentId: string;
+        status: EvalRunNetwork['run']['status'];
+        subject: {
+            kind: string;
+            ref: string;
+        };
+        counts: Record<EvalNetworkTask['status'], number>;
+        tasks: number;
+        startedAt: string;
+        endedAt: string;
+    }>;
+}
+export interface LaunchNetworkResponse {
+    success: boolean;
+    network: EvalLaunchNetwork;
+}
+export interface RunNetworkResponse {
+    success: boolean;
+    network: EvalRunNetwork;
+}
+export interface CaseResultResponse {
+    success: boolean;
+    case: EvalTaskResult;
+}
+export interface ImportEvalsetRequest {
+    /** An `*.evalset.json` spec, as the CLI and the action read it. */
+    spec: Record<string, unknown>;
+    run_environment?: EvalRunEnvironment;
+}
+export interface ImportEvalsetResponse extends EvalsetResponse {
+    /** Evaluators the platform cannot run, left out of the evalset. */
+    unsupported_evaluators: string[];
+}
+export interface EvalsetVersionListResponse {
+    success: boolean;
+    total: number;
+    versions: EvalsetVersion[];
+}
+export interface EvalsetVersionResponse {
+    success: boolean;
+    version: EvalsetVersion;
+}
+export type InvestigationScope = 'case' | 'run' | 'launch' | 'window';
+export type InvestigationStatus = 'open' | 'closed';
+/** The restored sandbox, per section 12.6. */
+export type SandboxState = 'available' | 'restoring' | 'running' | 'expired' | 'none';
+/** The work a person does on a benchmark result: one per task, run, launch or window. */
+export interface EvalInvestigation {
+    id: string;
+    owner_uid: string;
+    account_uid: string;
+    scope: InvestigationScope;
+    evalset_id: string;
+    launch_id: string;
+    run_id: string;
+    case_id: string;
+    window_id: string;
+    title: string;
+    status: InvestigationStatus;
+    decision: string;
+    /** The report document (spacer lexical) the investigation is written in. */
+    document_uid: string;
+    /** The notebook the investigation works in; a copy, never the evidence. */
+    notebook_uid: string;
+    /** The task's evidence notebook, written by the run. */
+    evidence_notebook_uid: string;
+    sandbox_snapshot_uid: string;
+    /** The restored sandbox while it lives. */
+    runtime_name: string;
+    sandbox_workflow_uid: string;
+    sandbox_state: SandboxState;
+    created_by_uid: string;
+    assignees: string[];
+    /** Published to the library by its owner, with what it is written in. */
+    is_public: boolean;
+    metadata: Record<string, unknown>;
+    created_at: string | null;
+    updated_at: string | null;
+}
+export interface InvestigationResponse extends SuccessResponse {
+    investigation: EvalInvestigation;
+}
+export interface TaskInvestigationResponse extends InvestigationResponse {
+    case: EvalTaskResult;
+}
+export interface InvestigationListResponse extends SuccessResponse {
+    total: number;
+    investigations: EvalInvestigation[];
+}
+export interface UpdateInvestigationRequest {
+    status?: InvestigationStatus;
+    decision?: string;
+    title?: string;
+    assignees?: string[];
+}
+/**
+ * A view of an investigation's page kept under a name for everybody on it
+ * (B4-11), in the investigation's `metadata.views`.
+ */
+export interface SavedInvestigationView {
+    name: string;
+    /** What the page shows: `surface`, and `block`, `case` or `cell`. */
+    query: string;
+    saved_by_uid?: string;
+    saved_at?: string;
+}
+export interface SaveInvestigationViewRequest {
+    name: string;
+    query: string;
+}
+export interface ResumeSandboxRequest {
+    /** Minutes the restored sandbox is reserved for. */
+    time_reservation?: number;
+    environment?: string;
+}
+export interface ResumeSandboxResponse extends InvestigationResponse {
+    sandbox: {
+        state: SandboxState;
+        runtime_name: string;
+        workflow_uid?: string;
+        reused?: boolean;
+    };
+}
+/** A serialized Lexical editor state, as the editor writes it. */
+export interface LexicalReportResponse extends SuccessResponse {
+    document: {
+        root: Record<string, unknown>;
+    };
+}
+export interface ReportDocumentResponse extends SuccessResponse {
+    document_uid: string;
+    launch: EvalLaunch;
+}
+/** The records that carry grants, as their routes name them. */
+export type SharedEvalsRecord = 'evalsets' | 'launches' | 'investigations' | 'reports';
+/**
+ * The levels a grant gives, lowest first, each allowing what the ones before
+ * it allow: Viewer, Reviewer, Runner, Editor.
+ */
+export type EvalsAccessLevel = 'view' | 'review' | 'execute' | 'update';
+/** A level, or the owner — which is never granted: only an owner shares or publishes. */
+export type EvalsRole = EvalsAccessLevel | 'owner';
+export interface EvalsPrincipals {
+    userUids: string[];
+    teamUids: string[];
+    organizationUids: string[];
+}
+export type EvalsSharingUpdate = Partial<Record<EvalsAccessLevel, Partial<EvalsPrincipals>>>;
+export interface EvalsSharingResponse extends SuccessResponse {
+    sharing: {
+        kind: 'evalset' | 'launch' | 'investigation';
+        uid: string;
+        /** The account the record belongs to. */
+        owner_uid: string;
+        access: Record<EvalsAccessLevel, EvalsPrincipals>;
+        shared: boolean;
+    };
+}
+/**
+ * How far a record reaches (BENCHMARKS.md, B5-09, section 14.5), narrowest
+ * first. Derived from the grants and the public flag rather than stored, so it
+ * cannot disagree with who may actually read the thing.
+ */
+export type EvalsVisibility = 'private' | 'users' | 'team' | 'organization' | 'public';
+export interface EvalsPermissionsResponse extends SuccessResponse {
+    kind: string;
+    uid: string;
+    role: EvalsRole;
+    permissions: Record<EvalsRole, boolean>;
+    /** The widest thing that is true of this record. */
+    visibility: EvalsVisibility;
+}
+/** The states of a report (section 13.3), in the order a report goes through them. */
+export type ReportState = 'draft' | 'in_review' | 'approved' | 'published_private' | 'published_public' | 'superseded';
+/** A report over runs of one benchmark: its document and its state. */
+export interface EvalReport {
+    id: string;
+    owner_uid: string;
+    account_uid: string;
+    evalset_id: string;
+    evalset_version: number | null;
+    launch_ids: string[];
+    run_ids: string[];
+    /** The Lexical document in the benchmarks space people read and edit. */
+    document_uid: string;
+    title: string;
+    state: ReportState;
+    version: number;
+    /** Who was asked to approve it. */
+    reviewer_uids: string[];
+    approved_by_uid: string;
+    approved_at: string | null;
+    /** The version of the document kept when it was approved. */
+    approved_version_uid: string;
+    published_at: string | null;
+    supersedes_uid: string;
+    superseded_by_uid: string;
+    created_by_uid: string;
+    created_at: string | null;
+    updated_at: string | null;
+}
+export interface ReportResponse extends SuccessResponse {
+    report: EvalReport;
+}
+export interface ReportListResponse extends SuccessResponse {
+    total: number;
+    reports: EvalReport[];
+}
+export interface CreateReportRequest {
+    evalset_id: string;
+    launch_ids: string[];
+    title?: string;
+}
+/** A move of a report; `superseded` is not one, a regeneration is. */
+export interface MoveReportRequest {
+    state: Exclude<ReportState, 'superseded'>;
+    /** Who is asked to approve it, when it is sent for review. */
+    reviewer_uids?: string[];
+    /** What the approval says. */
+    message?: string;
+}
+export interface RegenerateReportResponse extends ReportResponse {
+    superseded: EvalReport;
+}
+export type ReportsQuery = {
+    evalset_id?: string;
+    launch_id?: string;
+    state?: ReportState;
+    limit?: number;
+    offset?: number;
+};
+export type DecisionKind = 'accepted_regression' | 'expected_change' | 'evaluator_issue' | 'data_issue' | 'action_required';
+export type DecisionOutcome = 'approved' | 'blocked' | 'accepted_with_limitations';
+/** What a decision is about: a block of a report, a task, a run, a launch. */
+export type DecisionScope = 'block' | 'case' | 'run' | 'launch';
+/** The records a decision is made on, as their routes name them. */
+export type DecisionSubject = 'reports' | 'investigations';
+/** A decision as it was made; decisions are appended, never edited. */
+export interface EvalDecision {
+    id: string;
+    owner_uid: string;
+    account_uid: string;
+    subject: 'report' | 'investigation';
+    subject_uid: string;
+    kind: DecisionKind;
+    outcome: DecisionOutcome;
+    scope: DecisionScope;
+    /** What in the scope: the block, the task, the run or the launch. */
+    scope_ref: string;
+    note: string;
+    decided_by_uid: string;
+    decided_at: string | null;
+    /** The comment thread the decision resolves. */
+    thread_uid: string;
+    evalset_id: string;
+    launch_ids: string[];
+}
+export interface DecisionRequest {
+    kind: DecisionKind;
+    outcome: DecisionOutcome;
+    scope: DecisionScope;
+    scope_ref?: string;
+    note?: string;
+    thread_uid?: string;
+}
+export interface DecisionResponse extends SuccessResponse {
+    decision: EvalDecision;
+}
+export interface DecisionListResponse extends SuccessResponse {
+    total: number;
+    decisions: EvalDecision[];
+}
+/** A report or an investigation somebody shared with the caller. */
+export interface SharedEvalsItem {
+    kind: 'report' | 'investigation';
+    uid: string;
+    title: string;
+    owner_uid: string;
+    /** The level the grants give the caller. */
+    role: EvalsAccessLevel;
+    /** The report's state or the investigation's status. */
+    state: string;
+    /** The page of the app it opens. */
+    link: string;
+    updated_at: string | null;
+}
+export interface SharedWithMeResponse extends SuccessResponse {
+    total: number;
+    shared: SharedEvalsItem[];
+}
+/** A report file CI produced, kept on its benchmark as it was. */
+export interface EvalReportImport {
+    id: string;
+    owner_uid: string;
+    account_uid: string;
+    evalset_id: string;
+    format: 'csv' | 'markdown';
+    name: string;
+    run_ids: string[];
+    launch_ids: string[];
+    /** Run ids the file names that are no run of this benchmark here. */
+    unmatched_run_ids: string[];
+    imported_by_uid: string;
+    /** The live report continued from it, once one is. */
+    live_report_uid: string;
+    investigation_uid: string;
+    /** The file's text; left out of a listing. */
+    content: string | null;
+    created_at: string | null;
+    updated_at: string | null;
+}
+export interface ReportImportRequest {
+    format: 'csv' | 'markdown';
+    content: string;
+    name?: string;
+}
+export interface ReportImportResponse extends SuccessResponse {
+    import: EvalReportImport;
+}
+export interface ReportImportListResponse extends SuccessResponse {
+    total: number;
+    imports: EvalReportImport[];
+}
+export interface ContinueReportImportResponse extends ReportImportResponse {
+    report_link: string;
+    investigation_link: string;
+}