@plumloom/cli 0.1.0

package/dist/gate/decision.d.ts

@@ -0,0 +1,97 @@
+import type { JsonObject } from '../domain/common.js';
+import type { EvaluationContextType, EvaluationResults } from '../domain/types.js';
+import { type SuiteGatePolicy } from './policy.js';
+/**
+ * Gate plane.
+ *
+ * Execution and result fetching happen elsewhere; this module is pure. It
+ * converts an already-fetched result payload plus the terminal run status into
+ * one release decision per eval, then rolls the eval decisions into a single
+ * suite decision. It never performs I/O and reads only fields the CLI already
+ * receives from existing result and run-status payloads.
+ *
+ * Deliberately unused for gating: conversation / agent-trace `outcome.*`,
+ * generated summaries and insights, and agent-trace model-performance CI.
+ */
+export type GateDecision = 'PASS' | 'FAIL' | 'INCONCLUSIVE' | 'ERROR';
+export declare function worstDecision(decisions: readonly GateDecision[]): GateDecision;
+export type GateCheckEvidence = {
+    mean?: number | null;
+    ci95Lower?: number | null;
+    ci95Upper?: number | null;
+    score?: number | null;
+    hasData?: boolean;
+    insufficientSamples?: boolean;
+};
+export type GateCheckResult = {
+    /** Stable machine identifier, e.g. `overall`, `scenario:Checkout`, `metric:factuality`. */
+    metric: string;
+    label: string;
+    threshold: string;
+    actual: string;
+    decision: GateDecision;
+    reason: string;
+    basis: 'mean' | 'confidence-interval';
+    evidence: GateCheckEvidence;
+};
+export type ConvergenceStatus = 'CONVERGED' | 'NOT_CONVERGED' | 'NOT_APPLICABLE' | 'ERROR' | 'UNKNOWN';
+export type ConvergenceEvidence = {
+    status: ConvergenceStatus;
+    autoStopReason?: string;
+    achievedConsistency?: number;
+    targetConsistency?: number;
+    /** Comparison direction reported by the backend; `<` means lower is better. */
+    targetConsistencyOperator?: string;
+    failureReason?: string;
+    /** True when the run status carried none of the convergence fields. */
+    fieldsUnavailable: boolean;
+};
+export type EvalGateEvidence = {
+    runMode: 'single_run' | 'multi_run' | 'unknown';
+    runState?: string;
+    totalRuns?: number;
+    convergence?: ConvergenceEvidence;
+};
+export type EvalGateDecision = {
+    index: number;
+    inputFile: string;
+    evaluationName?: string;
+    evaluationId?: string;
+    runId?: string;
+    contextType?: EvaluationContextType;
+    decision: GateDecision;
+    checks: GateCheckResult[];
+    reason: string;
+    evidence: EvalGateEvidence;
+};
+export type SuiteGateReport = {
+    suiteDecision: GateDecision;
+    counts: Record<GateDecision, number>;
+    perEvalDecisions: EvalGateDecision[];
+};
+export type GateEvalInput = {
+    index: number;
+    inputFile: string;
+    evaluationName?: string;
+    evaluationId?: string;
+    runId?: string;
+    contextType?: EvaluationContextType;
+    /** Terminal state of the execution and results planes for this eval. */
+    status: 'completed' | 'execution_failed' | 'result_failed';
+    error?: string;
+    results?: EvaluationResults;
+    runState?: string;
+    runStatusRaw?: JsonObject;
+    policy: SuiteGatePolicy;
+};
+/**
+ * Classify convergence from the terminal run status. `MAX_RUNS_REACHED` alone
+ * does not prove a convergence failure, so the consistency target decides
+ * whether convergence was enabled at all.
+ */
+export declare function classifyConvergence(runStatusRaw: JsonObject | undefined, runState: string | undefined): ConvergenceEvidence;
+/** Convert one completed (or failed) suite eval into a release decision. */
+export declare function decideEvalGate(input: GateEvalInput): EvalGateDecision;
+/** Roll per-eval decisions up with ERROR > FAIL > INCONCLUSIVE > PASS. */
+export declare function rollUpSuiteDecision(perEvalDecisions: readonly EvalGateDecision[]): SuiteGateReport;
+//# sourceMappingURL=decision.d.ts.map

package/dist/gate/decision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision.d.ts","sourceRoot":"","sources":["../../src/gate/decision.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACnF,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AAElE;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,cAAc,GAAG,OAAO,CAAC;AAStE,wBAAgB,aAAa,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,CAK9E;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,2FAA2F;IAC3F,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAAC;IACtC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAC3B,WAAW,GAAG,eAAe,GAAG,gBAAgB,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzE,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,+EAA+E;IAC/E,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,YAAY,GAAG,WAAW,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,YAAY,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACrC,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,wEAAwE;IACxE,MAAM,EAAE,WAAW,GAAG,kBAAkB,GAAG,eAAe,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB,CAAC;AAoOF;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,YAAY,EAAE,UAAU,GAAG,SAAS,EACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAC3B,mBAAmB,CAoDrB;AA4GD,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,gBAAgB,CA6FrE;AAED,0EAA0E;AAC1E,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,SAAS,gBAAgB,EAAE,GAC5C,eAAe,CAgBjB"}

package/dist/gate/decision.js

@@ -0,0 +1,436 @@
+import { z } from 'zod';
+import { isEmptyPolicy } from './policy.js';
+const PRECEDENCE = {
+    ERROR: 3,
+    FAIL: 2,
+    INCONCLUSIVE: 1,
+    PASS: 0,
+};
+export function worstDecision(decisions) {
+    return decisions.reduce((worst, decision) => (PRECEDENCE[decision] > PRECEDENCE[worst] ? decision : worst), 'PASS');
+}
+const NOT_AVAILABLE = 'Not available';
+function formatNumber(value) {
+    return Number.isInteger(value) ? value.toFixed(2) : value.toFixed(2);
+}
+function formatActual(evidence, basis) {
+    if (basis === 'confidence-interval') {
+        const { ci95Lower: lower, ci95Upper: upper } = evidence;
+        if (typeof lower !== 'number' || typeof upper !== 'number')
+            return NOT_AVAILABLE;
+        const mean = typeof evidence.mean === 'number' ? `${formatNumber(evidence.mean)} ` : '';
+        return `${mean}95% CI ${formatNumber(lower)}-${formatNumber(upper)}`;
+    }
+    const value = evidence.score ?? evidence.mean;
+    return typeof value === 'number' ? formatNumber(value) : NOT_AVAILABLE;
+}
+/** Point-estimate comparison. A missing value is never treated as zero. */
+function checkMean(metric, label, threshold, evidence) {
+    const value = evidence.score ?? evidence.mean;
+    const base = {
+        metric,
+        label,
+        threshold: `>= ${formatNumber(threshold)}`,
+        basis: 'mean',
+        evidence,
+        actual: formatActual(evidence, 'mean'),
+    };
+    if (evidence.hasData === false) {
+        return { ...base, decision: 'INCONCLUSIVE', reason: 'metric reported has_data: false' };
+    }
+    if (typeof value !== 'number') {
+        return { ...base, decision: 'INCONCLUSIVE', reason: 'required score is unavailable' };
+    }
+    return value >= threshold
+        ? { ...base, decision: 'PASS', reason: 'score meets the required threshold' }
+        : { ...base, decision: 'FAIL', reason: 'score is below the required threshold' };
+}
+/** Interval comparison for multi-run evals. */
+function checkConfidenceInterval(metric, label, threshold, evidence) {
+    const base = {
+        metric,
+        label,
+        threshold: `>= ${formatNumber(threshold)}`,
+        basis: 'confidence-interval',
+        evidence,
+        actual: formatActual(evidence, 'confidence-interval'),
+    };
+    const lower = evidence.ci95Lower;
+    const upper = evidence.ci95Upper;
+    if (typeof lower !== 'number' || typeof upper !== 'number') {
+        return {
+            ...base,
+            decision: 'INCONCLUSIVE',
+            reason: 'required 95% confidence interval is unavailable',
+        };
+    }
+    if (lower >= threshold) {
+        return {
+            ...base,
+            decision: 'PASS',
+            reason: 'confidence interval is entirely at or above the threshold',
+        };
+    }
+    if (upper < threshold) {
+        return {
+            ...base,
+            decision: 'FAIL',
+            reason: 'confidence interval is entirely below the threshold',
+        };
+    }
+    return {
+        ...base,
+        decision: 'INCONCLUSIVE',
+        reason: 'confidence interval crosses release threshold',
+    };
+}
+const nullableNumber = z.number().nullable().optional();
+const scoreCellSchema = z
+    .object({
+    mean: nullableNumber,
+    ci95_lower: nullableNumber,
+    ci95_upper: nullableNumber,
+    insufficient_samples: z.boolean().nullable().optional(),
+    has_data: z.boolean().nullable().optional(),
+    error: z.unknown().optional(),
+})
+    .loose();
+const modelPerformanceSchema = z
+    .object({
+    models: z
+        .array(z
+        .object({
+        display_name: z.string().optional(),
+        model_key: z.string().optional(),
+        is_primary: z.boolean().optional(),
+        scores: z.object({ overall: scoreCellSchema.optional() }).loose().optional(),
+    })
+        .loose())
+        .optional(),
+})
+    .loose();
+const scenarioComparisonSchema = z
+    .object({
+    scenarios: z
+        .array(z
+        .object({
+        scenario_name: z.string().optional(),
+        model_scores: z
+            .array(z
+            .object({
+            model_key: z.string().optional(),
+            is_primary: z.boolean().optional(),
+            score: scoreCellSchema.optional(),
+            error: z.unknown().optional(),
+        })
+            .loose())
+            .optional(),
+    })
+        .loose())
+        .optional(),
+})
+    .loose();
+const perMetricSchema = z
+    .object({
+    per_metric: z
+        .record(z.string(), z
+        .object({
+        score: nullableNumber,
+        has_data: z.boolean().nullable().optional(),
+    })
+        .loose())
+        .optional(),
+})
+    .loose();
+const runStatusGateSchema = z
+    .object({
+    evaluationState: z.string().optional(),
+    progress: z.object({ totalRuns: z.number().nullable().optional() }).loose().optional(),
+    evaluationMetrics: z
+        .object({
+        autoStopTriggered: z
+            .object({ reason: z.string().nullable().optional() })
+            .loose()
+            .nullable()
+            .optional(),
+        achievedConsistency: nullableNumber,
+        targetConsistency: nullableNumber,
+        targetConsistencyOperator: z.string().nullable().optional(),
+    })
+        .loose()
+        .nullable()
+        .optional(),
+    achievedConsistency: nullableNumber,
+    targetConsistency: nullableNumber,
+    targetConsistencyOperator: z.string().nullable().optional(),
+    failureDetails: z
+        .object({ failureReason: z.string().nullable().optional() })
+        .loose()
+        .nullable()
+        .optional(),
+    failure_reason: z.string().nullable().optional(),
+    failureReason: z.string().nullable().optional(),
+})
+    .loose();
+function toEvidence(cell) {
+    return {
+        mean: cell?.mean ?? null,
+        ci95Lower: cell?.ci95_lower ?? null,
+        ci95Upper: cell?.ci95_upper ?? null,
+        ...(cell?.has_data === null || cell?.has_data === undefined ? {} : { hasData: cell.has_data }),
+        ...(cell?.insufficient_samples === null || cell?.insufficient_samples === undefined
+            ? {}
+            : { insufficientSamples: cell.insufficient_samples }),
+    };
+}
+/**
+ * Consistency can be reported either as a score to exceed or as a spread to
+ * stay under. The backend states the direction in `targetConsistencyOperator`;
+ * without it, a higher-is-better target is assumed.
+ */
+function meetsConsistencyTarget(achieved, target, operator) {
+    switch ((operator ?? '>=').trim()) {
+        case '<':
+            return achieved < target;
+        case '<=':
+            return achieved <= target;
+        case '>':
+            return achieved > target;
+        default:
+            return achieved >= target;
+    }
+}
+/**
+ * Classify convergence from the terminal run status. `MAX_RUNS_REACHED` alone
+ * does not prove a convergence failure, so the consistency target decides
+ * whether convergence was enabled at all.
+ */
+export function classifyConvergence(runStatusRaw, runState) {
+    const parsed = runStatusGateSchema.safeParse(runStatusRaw ?? {});
+    const data = parsed.success ? parsed.data : {};
+    const metrics = data.evaluationMetrics ?? undefined;
+    const autoStopReason = metrics?.autoStopTriggered?.reason ?? undefined;
+    const achieved = metrics?.achievedConsistency ?? data.achievedConsistency ?? undefined;
+    const target = metrics?.targetConsistency ?? data.targetConsistency ?? undefined;
+    const operator = metrics?.targetConsistencyOperator ?? data.targetConsistencyOperator ?? undefined;
+    const failureReason = data.failureDetails?.failureReason ?? data.failureReason ?? data.failure_reason ?? undefined;
+    const fieldsUnavailable = autoStopReason === undefined &&
+        achieved === null &&
+        target === null &&
+        failureReason === undefined
+        ? true
+        : autoStopReason === undefined &&
+            achieved === undefined &&
+            target === undefined &&
+            failureReason === undefined;
+    const base = {
+        ...(autoStopReason ? { autoStopReason } : {}),
+        ...(typeof achieved === 'number' ? { achievedConsistency: achieved } : {}),
+        ...(typeof target === 'number' ? { targetConsistency: target } : {}),
+        ...(operator ? { targetConsistencyOperator: operator } : {}),
+        ...(failureReason ? { failureReason } : {}),
+        fieldsUnavailable,
+    };
+    if (failureReason !== undefined && failureReason !== null) {
+        return { ...base, status: 'ERROR' };
+    }
+    const state = (runState ?? data.evaluationState ?? '').toUpperCase();
+    if (state !== '' && state !== 'COMPLETED' && state !== 'COMPLETE' && state !== 'SUCCEEDED') {
+        return { ...base, status: 'ERROR' };
+    }
+    if (fieldsUnavailable)
+        return { ...base, status: 'UNKNOWN' };
+    if (autoStopReason === 'OPTIMAL_CONFIDENCE_REACHED' ||
+        autoStopReason === 'CONSISTENCY_TARGET_MET') {
+        return { ...base, status: 'CONVERGED' };
+    }
+    if (typeof target !== 'number' || target <= 0)
+        return { ...base, status: 'NOT_APPLICABLE' };
+    if (typeof achieved !== 'number')
+        return { ...base, status: 'NOT_CONVERGED' };
+    return {
+        ...base,
+        status: meetsConsistencyTarget(achieved, target, operator) ? 'CONVERGED' : 'NOT_CONVERGED',
+    };
+}
+function detectRunMode(runStatusRaw, cells) {
+    const parsed = runStatusGateSchema.safeParse(runStatusRaw ?? {});
+    const totalRuns = parsed.success ? (parsed.data.progress?.totalRuns ?? undefined) : undefined;
+    if (typeof totalRuns === 'number' && totalRuns > 0) {
+        return { runMode: totalRuns > 1 ? 'multi_run' : 'single_run', totalRuns };
+    }
+    const hasInterval = cells.some((cell) => typeof cell.ci95Lower === 'number' && typeof cell.ci95Upper === 'number');
+    return { runMode: hasInterval ? 'multi_run' : 'single_run' };
+}
+function scenarioChecks(results, policy) {
+    const cells = [];
+    const performance = modelPerformanceSchema.safeParse(results.modelPerformance);
+    const models = performance.success ? (performance.data.models ?? []) : [];
+    const primary = models.find((model) => model.is_primary === true) ?? models[0];
+    const primaryKey = primary?.model_key;
+    if (policy.minOverall !== undefined) {
+        cells.push({
+            metric: 'overall',
+            label: 'Primary model overall score',
+            threshold: policy.minOverall,
+            evidence: toEvidence(primary?.scores?.overall),
+        });
+    }
+    if (policy.minScenario !== undefined) {
+        const comparison = scenarioComparisonSchema.safeParse(results.scenarioComparison);
+        const scenarios = comparison.success ? (comparison.data.scenarios ?? []) : [];
+        for (const scenario of scenarios) {
+            const modelScores = scenario.model_scores ?? [];
+            const cell = modelScores.find((entry) => entry.is_primary === true) ??
+                (primaryKey === undefined
+                    ? modelScores[0]
+                    : (modelScores.find((entry) => entry.model_key === primaryKey) ?? undefined));
+            if (cell === undefined)
+                continue;
+            const name = scenario.scenario_name ?? 'scenario';
+            cells.push({
+                metric: `scenario:${name}`,
+                label: `Scenario ${name}`,
+                threshold: policy.minScenario,
+                evidence: cell.error === undefined || cell.error === null
+                    ? toEvidence(cell.score)
+                    : { ...toEvidence(cell.score), hasData: false },
+            });
+        }
+    }
+    return { cells };
+}
+function sessionChecks(payload, policy) {
+    const parsed = perMetricSchema.safeParse(payload);
+    const perMetric = parsed.success ? (parsed.data.per_metric ?? {}) : {};
+    return Object.entries(policy.metrics ?? {}).map(([name, threshold]) => {
+        const cell = perMetric[name];
+        const evidence = {
+            score: cell?.score ?? null,
+            ...(cell?.has_data === null || cell?.has_data === undefined
+                ? {}
+                : { hasData: cell.has_data }),
+        };
+        if (cell === undefined) {
+            return {
+                metric: `metric:${name}`,
+                label: name,
+                threshold: `>= ${formatNumber(threshold)}`,
+                actual: NOT_AVAILABLE,
+                decision: 'INCONCLUSIVE',
+                reason: 'required metric is missing from the result payload',
+                basis: 'mean',
+                evidence,
+            };
+        }
+        return checkMean(`metric:${name}`, name, threshold, evidence);
+    });
+}
+function summarize(decision, checks) {
+    if (decision === 'PASS') {
+        return checks.length === 0
+            ? 'no blocking criteria configured'
+            : 'every configured criterion was met';
+    }
+    const offending = checks.find((check) => check.decision === decision);
+    return offending === undefined
+        ? 'gate evidence is unusable'
+        : `${offending.label}: ${offending.reason}`;
+}
+/** Convert one completed (or failed) suite eval into a release decision. */
+export function decideEvalGate(input) {
+    const identity = {
+        index: input.index,
+        inputFile: input.inputFile,
+        ...(input.evaluationName ? { evaluationName: input.evaluationName } : {}),
+        ...(input.evaluationId ? { evaluationId: input.evaluationId } : {}),
+        ...(input.runId ? { runId: input.runId } : {}),
+        ...(input.contextType ? { contextType: input.contextType } : {}),
+    };
+    if (input.status !== 'completed' || input.results === undefined) {
+        return {
+            ...identity,
+            decision: 'ERROR',
+            checks: [],
+            reason: input.error ??
+                (input.status === 'execution_failed'
+                    ? 'eval execution failed'
+                    : 'eval results could not be fetched'),
+            evidence: {
+                runMode: 'unknown',
+                ...(input.runState ? { runState: input.runState } : {}),
+            },
+        };
+    }
+    const results = input.results;
+    if (results.contextType !== 'scenario') {
+        const payload = results.contextType === 'conversation' ? results.conversation : results.agentTrace;
+        const checks = sessionChecks(payload, input.policy);
+        const decision = isEmptyPolicy(input.policy)
+            ? 'INCONCLUSIVE'
+            : worstDecision(checks.map((check) => check.decision));
+        return {
+            ...identity,
+            decision,
+            checks,
+            reason: isEmptyPolicy(input.policy)
+                ? 'no metric thresholds are configured for this eval'
+                : summarize(decision, checks),
+            evidence: {
+                runMode: 'single_run',
+                ...(input.runState ? { runState: input.runState } : {}),
+            },
+        };
+    }
+    const { cells } = scenarioChecks(results, input.policy);
+    const { runMode, totalRuns } = detectRunMode(input.runStatusRaw, cells.map((cell) => cell.evidence));
+    const convergence = runMode === 'multi_run' ? classifyConvergence(input.runStatusRaw, input.runState) : undefined;
+    const checks = cells.map((cell) => runMode === 'multi_run'
+        ? checkConfidenceInterval(cell.metric, cell.label, cell.threshold, cell.evidence)
+        : checkMean(cell.metric, cell.label, cell.threshold, cell.evidence));
+    const evidence = {
+        runMode,
+        ...(input.runState ? { runState: input.runState } : {}),
+        ...(totalRuns === undefined ? {} : { totalRuns }),
+        ...(convergence ? { convergence } : {}),
+    };
+    if (isEmptyPolicy(input.policy)) {
+        return {
+            ...identity,
+            decision: 'INCONCLUSIVE',
+            checks,
+            reason: 'no score thresholds are configured for this eval',
+            evidence,
+        };
+    }
+    let decision = worstDecision(checks.map((check) => check.decision));
+    let reason = summarize(decision, checks);
+    if (convergence?.status === 'ERROR') {
+        decision = 'ERROR';
+        reason = convergence.failureReason ?? 'the multi-run evaluation did not complete successfully';
+    }
+    else if (convergence?.status === 'NOT_CONVERGED' && decision !== 'FAIL') {
+        decision = 'INCONCLUSIVE';
+        reason = 'convergence was enabled but the consistency target was not reached';
+    }
+    return { ...identity, decision, checks, reason, evidence };
+}
+/** Roll per-eval decisions up with ERROR > FAIL > INCONCLUSIVE > PASS. */
+export function rollUpSuiteDecision(perEvalDecisions) {
+    const counts = {
+        PASS: 0,
+        FAIL: 0,
+        INCONCLUSIVE: 0,
+        ERROR: 0,
+    };
+    for (const entry of perEvalDecisions)
+        counts[entry.decision] += 1;
+    return {
+        suiteDecision: perEvalDecisions.length === 0
+            ? 'INCONCLUSIVE'
+            : worstDecision(perEvalDecisions.map((entry) => entry.decision)),
+        counts,
+        perEvalDecisions: [...perEvalDecisions].sort((left, right) => left.index - right.index),
+    };
+}
+//# sourceMappingURL=decision.js.map

package/dist/gate/decision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decision.js","sourceRoot":"","sources":["../../src/gate/decision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,aAAa,EAAwB,MAAM,aAAa,CAAC;AAiBlE,MAAM,UAAU,GAA2C;IACzD,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,YAAY,EAAE,CAAC;IACf,IAAI,EAAE,CAAC;CACR,CAAC;AAEF,MAAM,UAAU,aAAa,CAAC,SAAkC;IAC9D,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAClF,MAAM,CACP,CAAC;AACJ,CAAC;AAgFD,MAAM,aAAa,GAAG,eAAe,CAAC;AAEtC,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CAAC,QAA2B,EAAE,KAA+B;IAChF,IAAI,KAAK,KAAK,qBAAqB,EAAE,CAAC;QACpC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QACxD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,aAAa,CAAC;QACjF,MAAM,IAAI,GAAG,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,OAAO,GAAG,IAAI,UAAU,YAAY,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACvE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC;IAC9C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;AACzE,CAAC;AAED,2EAA2E;AAC3E,SAAS,SAAS,CAChB,MAAc,EACd,KAAa,EACb,SAAiB,EACjB,QAA2B;IAE3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC;IAC9C,MAAM,IAAI,GAAG;QACX,MAAM;QACN,KAAK;QACL,SAAS,EAAE,MAAM,YAAY,CAAC,SAAS,CAAC,EAAE;QAC1C,KAAK,EAAE,MAAe;QACtB,QAAQ;QACR,MAAM,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC;KACvC,CAAC;IACF,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC/B,OAAO,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,iCAAiC,EAAE,CAAC;IAC1F,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,+BAA+B,EAAE,CAAC;IACxF,CAAC;IACD,OAAO,KAAK,IAAI,SAAS;QACvB,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,oCAAoC,EAAE;QAC7E,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,uCAAuC,EAAE,CAAC;AACrF,CAAC;AAED,+CAA+C;AAC/C,SAAS,uBAAuB,CAC9B,MAAc,EACd,KAAa,EACb,SAAiB,EACjB,QAA2B;IAE3B,MAAM,IAAI,GAAG;QACX,MAAM;QACN,KAAK;QACL,SAAS,EAAE,MAAM,YAAY,CAAC,SAAS,CAAC,EAAE;QAC1C,KAAK,EAAE,qBAA8B;QACrC,QAAQ;QACR,MAAM,EAAE,YAAY,CAAC,QAAQ,EAAE,qBAAqB,CAAC;KACtD,CAAC;IACF,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;IACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO;YACL,GAAG,IAAI;YACP,QAAQ,EAAE,cAAc;YACxB,MAAM,EAAE,iDAAiD;SAC1D,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;QACvB,OAAO;YACL,GAAG,IAAI;YACP,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,2DAA2D;SACpE,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;QACtB,OAAO;YACL,GAAG,IAAI;YACP,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,qDAAqD;SAC9D,CAAC;IACJ,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,QAAQ,EAAE,cAAc;QACxB,MAAM,EAAE,+CAA+C;KACxD,CAAC;AACJ,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC;AAExD,MAAM,eAAe,GAAG,CAAC;KACtB,MAAM,CAAC;IACN,IAAI,EAAE,cAAc;IACpB,UAAU,EAAE,cAAc;IAC1B,UAAU,EAAE,cAAc;IAC1B,oBAAoB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACvD,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC3C,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9B,CAAC;KACD,KAAK,EAAE,CAAC;AAEX,MAAM,sBAAsB,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,MAAM,EAAE,CAAC;SACN,KAAK,CACJ,CAAC;SACE,MAAM,CAAC;QACN,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACnC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAChC,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAClC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,eAAe,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;KAC7E,CAAC;SACD,KAAK,EAAE,CACX;SACA,QAAQ,EAAE;CACd,CAAC;KACD,KAAK,EAAE,CAAC;AAEX,MAAM,wBAAwB,GAAG,CAAC;KAC/B,MAAM,CAAC;IACN,SAAS,EAAE,CAAC;SACT,KAAK,CACJ,CAAC;SACE,MAAM,CAAC;QACN,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACpC,YAAY,EAAE,CAAC;aACZ,KAAK,CACJ,CAAC;aACE,MAAM,CAAC;YACN,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAChC,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAClC,KAAK,EAAE,eAAe,CAAC,QAAQ,EAAE;YACjC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAC9B,CAAC;aACD,KAAK,EAAE,CACX;aACA,QAAQ,EAAE;KACd,CAAC;SACD,KAAK,EAAE,CACX;SACA,QAAQ,EAAE;CACd,CAAC;KACD,KAAK,EAAE,CAAC;AAEX,MAAM,eAAe,GAAG,CAAC;KACtB,MAAM,CAAC;IACN,UAAU,EAAE,CAAC;SACV,MAAM,CACL,CAAC,CAAC,MAAM,EAAE,EACV,CAAC;SACE,MAAM,CAAC;QACN,KAAK,EAAE,cAAc;QACrB,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;KAC5C,CAAC;SACD,KAAK,EAAE,CACX;SACA,QAAQ,EAAE;CACd,CAAC;KACD,KAAK,EAAE,CAAC;AAEX,MAAM,mBAAmB,GAAG,CAAC;KAC1B,MAAM,CAAC;IACN,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;IACtF,iBAAiB,EAAE,CAAC;SACjB,MAAM,CAAC;QACN,iBAAiB,EAAE,CAAC;aACjB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;aACpD,KAAK,EAAE;aACP,QAAQ,EAAE;aACV,QAAQ,EAAE;QACb,mBAAmB,EAAE,cAAc;QACnC,iBAAiB,EAAE,cAAc;QACjC,yBAAyB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;KAC5D,CAAC;SACD,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,QAAQ,EAAE;IACb,mBAAmB,EAAE,cAAc;IACnC,iBAAiB,EAAE,cAAc;IACjC,yBAAyB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC3D,cAAc,EAAE,CAAC;SACd,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;SAC3D,KAAK,EAAE;SACP,QAAQ,EAAE;SACV,QAAQ,EAAE;IACb,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,KAAK,EAAE,CAAC;AAEX,SAAS,UAAU,CAAC,IAAiD;IACnE,OAAO;QACL,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI;QACxB,SAAS,EAAE,IAAI,EAAE,UAAU,IAAI,IAAI;QACnC,SAAS,EAAE,IAAI,EAAE,UAAU,IAAI,IAAI;QACnC,GAAG,CAAC,IAAI,EAAE,QAAQ,KAAK,IAAI,IAAI,IAAI,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9F,GAAG,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,IAAI,IAAI,EAAE,oBAAoB,KAAK,SAAS;YACjF,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,oBAAoB,EAAE,CAAC;KACxD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAC7B,QAAgB,EAChB,MAAc,EACd,QAA4B;IAE5B,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,KAAK,GAAG;YACN,OAAO,QAAQ,GAAG,MAAM,CAAC;QAC3B,KAAK,IAAI;YACP,OAAO,QAAQ,IAAI,MAAM,CAAC;QAC5B,KAAK,GAAG;YACN,OAAO,QAAQ,GAAG,MAAM,CAAC;QAC3B;YACE,OAAO,QAAQ,IAAI,MAAM,CAAC;IAC9B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,YAAoC,EACpC,QAA4B;IAE5B,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,IAAI,SAAS,CAAC;IACpD,MAAM,cAAc,GAAG,OAAO,EAAE,iBAAiB,EAAE,MAAM,IAAI,SAAS,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,EAAE,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,IAAI,SAAS,CAAC;IACvF,MAAM,MAAM,GAAG,OAAO,EAAE,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,IAAI,SAAS,CAAC;IACjF,MAAM,QAAQ,GACZ,OAAO,EAAE,yBAAyB,IAAI,IAAI,CAAC,yBAAyB,IAAI,SAAS,CAAC;IACpF,MAAM,aAAa,GACjB,IAAI,CAAC,cAAc,EAAE,aAAa,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,SAAS,CAAC;IAE/F,MAAM,iBAAiB,GACrB,cAAc,KAAK,SAAS;QAC5B,QAAQ,KAAK,IAAI;QACjB,MAAM,KAAK,IAAI;QACf,aAAa,KAAK,SAAS;QACzB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,cAAc,KAAK,SAAS;YAC5B,QAAQ,KAAK,SAAS;YACtB,MAAM,KAAK,SAAS;YACpB,aAAa,KAAK,SAAS,CAAC;IAElC,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,iBAAiB;KAClB,CAAC;IAEF,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;QAC1D,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACrE,IAAI,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;QAC3F,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,IAAI,iBAAiB;QAAE,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC7D,IACE,cAAc,KAAK,4BAA4B;QAC/C,cAAc,KAAK,wBAAwB,EAC3C,CAAC;QACD,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC5F,IAAI,OAAO,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IAC9E,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe;KAC3F,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,YAAoC,EACpC,KAAmC;IAEnC,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC;IAC5E,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAC5B,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CACnF,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,cAAc,CACrB,OAAgE,EAChE,MAAuB;IAEvB,MAAM,KAAK,GAKL,EAAE,CAAC;IAET,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAG,OAAO,EAAE,SAAS,CAAC;IAEtC,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,6BAA6B;YACpC,SAAS,EAAE,MAAM,CAAC,UAAU;YAC5B,QAAQ,EAAE,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAClF,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,GACR,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;gBACtD,CAAC,UAAU,KAAK,SAAS;oBACvB,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;oBAChB,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,UAAU,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC;YAClF,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,IAAI,UAAU,CAAC;YAClD,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,YAAY,IAAI,EAAE;gBAC1B,KAAK,EAAE,YAAY,IAAI,EAAE;gBACzB,SAAS,EAAE,MAAM,CAAC,WAAW;gBAC7B,QAAQ,EACN,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;oBAC7C,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;oBACxB,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;aACpD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,MAAuB;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE;QACpE,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAsB;YAClC,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI;YAC1B,GAAG,CAAC,IAAI,EAAE,QAAQ,KAAK,IAAI,IAAI,IAAI,EAAE,QAAQ,KAAK,SAAS;gBACzD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;SAChC,CAAC;QACF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO;gBACL,MAAM,EAAE,UAAU,IAAI,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,MAAM,YAAY,CAAC,SAAS,CAAC,EAAE;gBAC1C,MAAM,EAAE,aAAa;gBACrB,QAAQ,EAAE,cAAc;gBACxB,MAAM,EAAE,oDAAoD;gBAC5D,KAAK,EAAE,MAAM;gBACb,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,SAAS,CAAC,UAAU,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,QAAsB,EAAE,MAAkC;IAC3E,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC;YACxB,CAAC,CAAC,iCAAiC;YACnC,CAAC,CAAC,oCAAoC,CAAC;IAC3C,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;IACtE,OAAO,SAAS,KAAK,SAAS;QAC5B,CAAC,CAAC,2BAA2B;QAC7B,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;AAChD,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,KAAoB;IACjD,MAAM,QAAQ,GAAG;QACf,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACjE,CAAC;IAEF,IAAI,KAAK,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChE,OAAO;YACL,GAAG,QAAQ;YACX,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,EAAE;YACV,MAAM,EACJ,KAAK,CAAC,KAAK;gBACX,CAAC,KAAK,CAAC,MAAM,KAAK,kBAAkB;oBAClC,CAAC,CAAC,uBAAuB;oBACzB,CAAC,CAAC,mCAAmC,CAAC;YAC1C,QAAQ,EAAE;gBACR,OAAO,EAAE,SAAS;gBAClB,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxD;SACF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAE9B,IAAI,OAAO,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,OAAO,GACX,OAAO,CAAC,WAAW,KAAK,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QACrF,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1C,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzD,OAAO;YACL,GAAG,QAAQ;YACX,QAAQ;YACR,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;gBACjC,CAAC,CAAC,mDAAmD;gBACrD,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC/B,QAAQ,EAAE;gBACR,OAAO,EAAE,YAAY;gBACrB,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxD;SACF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,aAAa,CAC1C,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CACnC,CAAC;IACF,MAAM,WAAW,GACf,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEhG,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChC,OAAO,KAAK,WAAW;QACrB,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC;QACjF,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CACtE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC,OAAO;QACP,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QACjD,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;IAEF,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,OAAO;YACL,GAAG,QAAQ;YACX,QAAQ,EAAE,cAAc;YACxB,MAAM;YACN,MAAM,EAAE,kDAAkD;YAC1D,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpE,IAAI,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEzC,IAAI,WAAW,EAAE,MAAM,KAAK,OAAO,EAAE,CAAC;QACpC,QAAQ,GAAG,OAAO,CAAC;QACnB,MAAM,GAAG,WAAW,CAAC,aAAa,IAAI,wDAAwD,CAAC;IACjG,CAAC;SAAM,IAAI,WAAW,EAAE,MAAM,KAAK,eAAe,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC1E,QAAQ,GAAG,cAAc,CAAC;QAC1B,MAAM,GAAG,oEAAoE,CAAC;IAChF,CAAC;IAED,OAAO,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AAC7D,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,mBAAmB,CACjC,gBAA6C;IAE7C,MAAM,MAAM,GAAiC;QAC3C,IAAI,EAAE,CAAC;QACP,IAAI,EAAE,CAAC;QACP,YAAY,EAAE,CAAC;QACf,KAAK,EAAE,CAAC;KACT,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,gBAAgB;QAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClE,OAAO;QACL,aAAa,EACX,gBAAgB,CAAC,MAAM,KAAK,CAAC;YAC3B,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpE,MAAM;QACN,gBAAgB,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;KACxF,CAAC;AACJ,CAAC"}

package/dist/gate/policy.d.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+/**
+ * Release-gate policy for one eval in a suite.
+ *
+ * The policy is authored by the developer next to the suite manifest; it adds
+ * no backend fields. Scenario evals gate on score thresholds, conversation and
+ * agent-trace evals gate on configured `per_metric.<metric>.score` thresholds.
+ */
+export type SuiteGatePolicy = {
+    /** Minimum primary-model overall score. */
+    minOverall?: number;
+    /** Minimum score for every configured scenario cell of the primary model. */
+    minScenario?: number;
+    /** Per-metric minimums for conversation and agent-trace evals. */
+    metrics?: Readonly<Record<string, number>>;
+};
+export declare const suiteGatePolicySchema: z.ZodObject<{
+    minOverall: z.ZodOptional<z.ZodNumber>;
+    minScenario: z.ZodOptional<z.ZodNumber>;
+    metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+}, z.core.$strict>;
+export declare function isEmptyPolicy(policy: SuiteGatePolicy | undefined): boolean;
+/** Per-eval policy wins over the suite-level default, field by field. */
+type LoosePolicy = {
+    minOverall?: number | undefined;
+    minScenario?: number | undefined;
+    metrics?: Record<string, number> | undefined;
+};
+export declare function mergePolicies(base: LoosePolicy | undefined, override: LoosePolicy | undefined): SuiteGatePolicy;
+export {};
+//# sourceMappingURL=policy.d.ts.map

package/dist/gate/policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy.d.ts","sourceRoot":"","sources":["../../src/gate/policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5C,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;kBAMvB,CAAC;AAEZ,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,CAI1E;AAED,yEAAyE;AACzE,KAAK,WAAW,GAAG;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,wBAAgB,aAAa,CAC3B,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,QAAQ,EAAE,WAAW,GAAG,SAAS,GAChC,eAAe,CASjB"}

package/dist/gate/policy.js

@@ -0,0 +1,26 @@
+import { z } from 'zod';
+export const suiteGatePolicySchema = z
+    .object({
+    minOverall: z.number().optional(),
+    minScenario: z.number().optional(),
+    metrics: z.record(z.string().min(1), z.number()).optional(),
+})
+    .strict();
+export function isEmptyPolicy(policy) {
+    if (policy === undefined)
+        return true;
+    if (policy.minOverall !== undefined || policy.minScenario !== undefined)
+        return false;
+    return Object.keys(policy.metrics ?? {}).length === 0;
+}
+export function mergePolicies(base, override) {
+    const metrics = { ...(base?.metrics ?? {}), ...(override?.metrics ?? {}) };
+    const minOverall = override?.minOverall ?? base?.minOverall;
+    const minScenario = override?.minScenario ?? base?.minScenario;
+    return {
+        ...(minOverall === undefined ? {} : { minOverall }),
+        ...(minScenario === undefined ? {} : { minScenario }),
+        ...(Object.keys(metrics).length === 0 ? {} : { metrics }),
+    };
+}
+//# sourceMappingURL=policy.js.map

package/dist/gate/policy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy.js","sourceRoot":"","sources":["../../src/gate/policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAkBxB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC5D,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,UAAU,aAAa,CAAC,MAAmC;IAC/D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACtF,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;AACxD,CAAC;AASD,MAAM,UAAU,aAAa,CAC3B,IAA6B,EAC7B,QAAiC;IAEjC,MAAM,OAAO,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,QAAQ,EAAE,UAAU,IAAI,IAAI,EAAE,UAAU,CAAC;IAC5D,MAAM,WAAW,GAAG,QAAQ,EAAE,WAAW,IAAI,IAAI,EAAE,WAAW,CAAC;IAC/D,OAAO;QACL,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;QACrD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC"}

package/dist/guide/eval-creation-backend.d.ts

@@ -0,0 +1,20 @@
+import type { ActionContext } from '../actions/context.js';
+import type { SupportedModel, Workspace } from '../domain/types.js';
+/**
+ * Everything the file guide needs from the backend, expressed as a
+ * narrow port so the flow itself stays deterministic and testable. Every method
+ * is backed by an existing shared action; the flow adds no network boundary.
+ */
+export interface EvalCreationBackend {
+    listWorkspaces(signal?: AbortSignal): Promise<readonly Workspace[]>;
+    listModels(signal?: AbortSignal): Promise<readonly SupportedModel[]>;
+}
+export declare class ActionEvalCreationBackend implements EvalCreationBackend {
+    #private;
+    constructor(input: {
+        actions: ActionContext;
+    });
+    listWorkspaces(signal?: AbortSignal): Promise<readonly Workspace[]>;
+    listModels(signal?: AbortSignal): Promise<readonly SupportedModel[]>;
+}
+//# sourceMappingURL=eval-creation-backend.d.ts.map

package/dist/guide/eval-creation-backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-creation-backend.d.ts","sourceRoot":"","sources":["../../src/guide/eval-creation-backend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG3D,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpE;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;IACpE,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC,CAAC;CACtE;AAED,qBAAa,yBAA0B,YAAW,mBAAmB;;gBAGvD,KAAK,EAAE;QAAE,OAAO,EAAE,aAAa,CAAA;KAAE;IAIvC,cAAc,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC;IAKzE,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;CAGrE"}