openmerit 0.4.1 → 0.4.2

package/dist/scout/workflow.d.ts

@@ -0,0 +1,221 @@
+import { type Catalogue, type CatalogueModel } from "../catalogue/catalogue.js";
+import { type TaskEvidence } from "../connectors/langfuse/index.js";
+import type { Harness, HarnessUsage } from "../harness/harness.js";
+import { type Endpoint, type EvalSpec } from "../state/config.js";
+import type { RunStatus, Spend, State } from "../state/db.js";
+import type { Repo } from "../state/paths.js";
+import { type PoolModel } from "./pool.js";
+import { type ScanResult } from "./scan.js";
+import { type Stage } from "./search.js";
+export type Answer = string | string[] | boolean | number;
+export type Answers = Record<string, Answer>;
+export interface Question {
+    id: string;
+    kind: "choice" | "multi" | "confirm" | "text";
+    prompt: string;
+    detail?: string;
+    options?: {
+        value: string;
+        label: string;
+        detail?: string;
+    }[];
+    default?: Answer;
+    /** What to type, for answers beyond the options, such as a mapping correction. */
+    hint?: string;
+}
+export interface Task {
+    name: string;
+    description: string;
+    usageIds: string[];
+    usesTools: boolean;
+    needsVision: boolean;
+    structuredOutput: boolean;
+}
+/** A candidate in a task's pool: ECI-ranked, allowed, cheaper than the current model. */
+export interface PoolEntry {
+    key: string;
+    model: string;
+    endpoint: string;
+    endpointModel: string;
+    eci: number;
+    eciSource: string | null;
+    price: number | null;
+}
+interface PlannedEval extends EvalSpec {
+    source: "saved" | "harness" | "user";
+    helperFiles?: {
+        path: string;
+        content: string;
+        purpose: string;
+    }[];
+    explanation?: string;
+}
+/** One eval run's result as recorded in SQLite: pass/fail per item, nothing else from the item. */
+export interface UnitResult {
+    items: Record<string, boolean>;
+    passes: number;
+    total: number;
+    source: "langfuse" | "eval";
+    experiment: string;
+    requests: number;
+    durationMs: number;
+}
+export interface SearchSummary {
+    status: "running" | "selected" | "no_passing_model" | "stopped";
+    /** ECI-ranked candidates searched, and eligible ones left out for lack of an ECI. */
+    pool: number;
+    unranked: number;
+    sampleIds: string[];
+    pick: string | null;
+    trials: {
+        stage: Stage;
+        model: string;
+        eci: number;
+        passes: number;
+        total: number;
+        accepted: boolean;
+    }[];
+    skipped: {
+        model: string;
+        stage: string;
+        reason: string;
+    }[];
+    finalists: string[];
+    challengers: string[];
+    shortfall?: string;
+}
+export interface ModelResult {
+    model: string;
+    endpoint: string;
+    endpointModel: string;
+    role: "baseline" | "candidate";
+    eci: number | null;
+    status: "measured" | "failed" | "unavailable" | "incomplete" | "budget_paused" | "not_run";
+    /** Passes over every item run (items × repeats). */
+    passes: number;
+    total: number;
+    runs: number;
+    /** Pass rate, passes / total. */
+    score: number | null;
+    /** Where per-item results came from: the Langfuse experiment the eval recorded, or the eval's own report. */
+    source: "langfuse" | "eval" | null;
+    /** Measured eval cost per item run, from the metering proxy. */
+    costPerItemUsd: number | null;
+    costIsEstimate: boolean;
+    meanLatencyMs: number | null;
+    requests: number;
+    qualified: boolean;
+    note: string;
+    /** Eval units in SQLite this result rests on, and the Langfuse experiments they read. */
+    evidence: string[];
+    experiments: string[];
+}
+export interface TaskComparison {
+    task: string;
+    eval: {
+        command: string;
+        source: string;
+        threshold: number | null;
+        langfuse: string | null;
+    };
+    baseline: ModelResult | null;
+    candidates: ModelResult[];
+    /** Up to three qualified candidates, cheapest measured cost first. */
+    top: ModelResult[];
+    status: "ranked" | "no_qualified" | "no_passing_model" | "stopped" | "baseline_failed" | "no_eval" | "non_discriminating" | "incomplete";
+    note: string;
+    search?: SearchSummary;
+    langfuse?: TaskEvidence | {
+        error: string;
+    };
+}
+interface ConfigureState {
+    decision?: "yes" | "no";
+    edits?: {
+        file: string;
+        line: number;
+        before: string;
+        after: string;
+        hash: string;
+        from: string;
+        to: string;
+    }[];
+    manual?: string[];
+    applied?: boolean;
+}
+export interface RunState {
+    scan?: ScanResult;
+    tasks?: Task[];
+    model?: string;
+    corrected?: boolean;
+    selected?: string[];
+    pool?: Record<string, PoolEntry[]>;
+    unranked?: Record<string, number>;
+    evals?: Record<string, PlannedEval | {
+        skipped: string;
+    }>;
+    evalApproved?: boolean;
+    repeats?: number;
+    datasetItems?: Record<string, string[]>;
+    samples?: Record<string, string[]>;
+    search?: Record<string, SearchSummary>;
+    stopped?: Record<string, string>;
+    retryDeclined?: string[];
+    compare?: Record<string, TaskComparison>;
+    accepted?: Record<string, "accept" | "stop">;
+    configure?: Record<string, ConfigureState>;
+    harnessUsage?: HarnessUsage;
+    [key: string]: unknown;
+}
+export interface Context {
+    state: State;
+    repo: Repo;
+    catalogue: Catalogue;
+    env: Record<string, string | undefined>;
+    harness: () => Promise<Harness>;
+    log: (line: string) => void;
+    fetch?: typeof fetch;
+    signal?: AbortSignal;
+    /** How long to wait for Langfuse to ingest an eval run's experiment. */
+    langfusePoll?: {
+        intervalMs: number;
+        timeoutMs: number;
+    };
+}
+export interface Step {
+    runId: string;
+    status: RunStatus;
+    phase: string;
+    questions: Question[];
+    notices: string[];
+    spend: Spend;
+    budgetUsd: number;
+    harnessUsage: HarnessUsage | null;
+    result?: ScoutResult;
+    error?: string;
+}
+export interface ScoutResult {
+    model: string | null;
+    tasks: TaskComparison[];
+    configuration: Record<string, ConfigureState>;
+}
+export declare const TOP = 3;
+export declare const TASK_QUESTION = "Select which Agents/Tasks do you want OpenMerit to review?";
+export declare const CONFIGURE_QUESTION = "Should OpenMerit configure this? Yes / No.";
+export declare function newRunId(): string;
+/** Advances a run as far as it can go. Answers are keyed by question ID. */
+export declare function advance(ctx: Context, runId: string, answers?: Answers): Promise<Step>;
+export declare function step(ctx: Context, runId: string, questions: Question[], notices: string[]): Step;
+/** Upper estimate of eval runs: baseline and finals on every item, screens on the sample. */
+export declare function estimate(poolSize: number, repeats: number): {
+    probes: number;
+    screens: number;
+    fullRuns: number;
+};
+/**
+ * Allowed-pool models a task could switch to: priced, cheaper than the current model, and able to do
+ * what the task needs (tools, images, structured output). The binary search ranks them by ECI.
+ */
+export declare function eligibleCandidates(catalogue: Catalogue, configured: readonly Endpoint[], baseline: CatalogueModel, task: Pick<Task, "usesTools" | "needsVision" | "structuredOutput">): PoolModel[];
+export declare const DEFAULT_BUDGET = 10;
+export {};