@tangle-network/agent-eval 0.100.1 → 0.101.0

package/dist/index.d.ts

@@ -18,8 +18,8 @@ import { A as AgentEvalError, J as JudgeError, a as ConfigError } from './errors
 export { b as AgentEvalErrorCode, C as CaptureIntegrityError, N as NotFoundError, R as ReplayError, V as ValidationError, c as VerificationError } from './errors-CzMUYo7b.js';
 import { b as FeedbackLabel, F as FeedbackTrajectoryStore, a as FeedbackTrajectory } from './feedback-trajectory-BxY0cKfs.js';
 export { c as FeedbackArtifactType, d as FeedbackAttempt, e as FeedbackLabelKind, f as FeedbackLabelSource, g as FeedbackOptimizerRow, h as FeedbackOutcome, i as FeedbackReplayAdapter, j as FeedbackReplayResult, k as FeedbackSeverity, l as FeedbackSplitPolicy, m as FeedbackTask, n as FeedbackTrajectoryFilter, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-BxY0cKfs.js';
-import { b as CorrectnessChecker } from './pre-registration-DB8oDqZJ.js';
-export { A as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, h as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, c as LlmJudgeOptions, i as ProducedProposal, P as ProducedState, j as ProposalEventLike, k as RequirementCheck, R as RuntimeEventLike, m as SatisfiedBy, S as SignedManifest, n as SignedManifestAlgo, T as TaskGold, o as ToolCallEventLike, V as ValidationContext, p as ValidationIssue, q as ValidationResult, r as assertRealBackend, s as byteLengthRange, t as canonicalize, u as completionVerdict, v as composeValidators, w as containsAll, x as createLlmCorrectnessChecker, y as createTokenRecallChecker, z as evaluateHypothesis, D as extractProducedState, E as hashJson, F as jsonHasKeys, l as llmJudge, G as parseCorrectnessResponse, I as regexMatch, J as signManifest, K as summarizeBackendIntegrity, M as verifyCompletion, N as verifyManifest } from './pre-registration-DB8oDqZJ.js';
+import { b as CorrectnessChecker } from './pre-registration-BjGZf9YA.js';
+export { A as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, h as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, c as LlmJudgeOptions, i as ProducedProposal, P as ProducedState, j as ProposalEventLike, k as RequirementCheck, R as RuntimeEventLike, m as SatisfiedBy, S as SignedManifest, n as SignedManifestAlgo, T as TaskGold, o as ToolCallEventLike, V as ValidationContext, p as ValidationIssue, q as ValidationResult, r as assertRealBackend, s as byteLengthRange, t as canonicalize, u as completionVerdict, v as composeValidators, w as containsAll, x as createLlmCorrectnessChecker, y as createTokenRecallChecker, z as evaluateHypothesis, D as extractProducedState, E as hashJson, F as jsonHasKeys, l as llmJudge, G as parseCorrectnessResponse, I as regexMatch, J as signManifest, K as summarizeBackendIntegrity, M as verifyCompletion, N as verifyManifest } from './pre-registration-BjGZf9YA.js';
 export { DataAcquisitionPlan, KnowledgeAcquisitionMode, KnowledgeBundle, KnowledgeFallbackPolicy, KnowledgeFreshness, KnowledgeImportance, KnowledgeReadinessReport, KnowledgeRecommendedAction, KnowledgeRequirement, KnowledgeRequirementCategory, KnowledgeResponsibleSurface, KnowledgeSensitivity, ScoreKnowledgeReadinessOptions, UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps } from './knowledge/index.js';
 import { h as ReleaseConfidenceThresholds, f as ReleaseConfidenceScorecard } from './release-report-B1tA6pKu.js';
 export { A as ActionableSideInfo, o as AsiSeverity, B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, g as ReleaseConfidenceStatus, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-B1tA6pKu.js';
@@ -30,7 +30,7 @@ import { a as AnalyzeTracesInput, A as AnalyzeTracesOptions, b as AnalyzeTracesR
 export { c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
 import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, b as DatasetOverview, E as ErrorCluster, Q as QueryTracesPage, S as SearchSpanResult, c as SearchTraceResult, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, T as TraceAnalysisStore, f as TraceAnalystByteBudgets, g as TraceAnalystFilters, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, V as ViewSpansResult, k as ViewTraceOversized, l as ViewTraceResult } from './store-C1YxJDEK.js';
-import { a as JudgeConfig, J as JudgeScore, S as Scenario$1, g as Gate } from './types-Cv1bo4_a.js';
+import { a as JudgeConfig, J as JudgeScore, S as Scenario$1, g as Gate } from './types-fWqEJm7h.js';
 import { A as AnalyzeRunsOptions } from './analyze-runs-BlJRBniC.js';
 import { S as SteeringBundle } from './harness-optimizer-mOl9XX_O.js';
 export { D as DEFAULT_HARNESS_OBJECTIVES, H as HarnessAdapter, a as HarnessExperimentConfig, b as HarnessExperimentResult, c as HarnessIntervention, d as HarnessRunRequest, e as HarnessRunResult, f as HarnessScenario, g as HarnessSelection, h as HarnessVariant, i as HarnessVariantReport, M as MeasurementPolicy, j as SteeringDelta, k as SteeringRolePrompt, W as WorkflowTopology, m as mergeSteeringBundle, r as renderSteeringText, l as runHarnessExperiment, s as selectHarnessVariant, n as summarizeHarnessResults } from './harness-optimizer-mOl9XX_O.js';
@@ -52,8 +52,8 @@ import { a as BaselineReport } from './baseline-Bbid3WoO.js';
 export { B as BaselineOptions, M as MetricSamples, b as MetricVerdict, T as ToolStats, d as ToolUseMetrics, e as ToolUseOptions, f as compareToBaseline, c as computeToolUseMetrics, i as iqr, w as welchsTTest } from './baseline-Bbid3WoO.js';
 import { a as TrajectoryStep, T as Trajectory } from './trajectory-2TkpSEVh.js';
 export { b as buildTrajectory } from './trajectory-2TkpSEVh.js';
-import { AgentProfile } from '@tangle-network/agent-interface';
-export { AgentProfile } from '@tangle-network/agent-interface';
+import { HarnessType, AgentProfile } from '@tangle-network/agent-interface';
+export { AgentProfile, HarnessType } from '@tangle-network/agent-interface';
 import { b as ChannelRollup, C as CostLedger } from './cost-ledger-DuSqlw5B.js';
 export { a as CostChannel, c as CostLedgerEntry, d as CostLedgerSummary, e as CostResult, f as CostUsage, g as costForUsage, m as modelPriceKey } from './cost-ledger-DuSqlw5B.js';
 export { D as Direction, O as Objective, P as ParetoResult, c as crowdingDistance, d as dominates, p as paretoFrontier, a as paretoFrontierWithCrowding, s as scalarScore } from './pareto-E-pembql.js';
@@ -75,8 +75,9 @@ export { C as CallbackResearcher, d as CallbackResearcherOptions, e as CampaignF
 export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, m as GateDecision, n as GateEvidence, H as HeldOutGate, o as HeldOutGateConfig, q as HeldOutGateRejectionCode, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-C4uzRWh8.js';
 export { L as LockedJsonlAppender } from './testing-C21CHsq2.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
-import { e as GepaProposerConstraints, b as RunImprovementLoopResult } from './gepa-BRgNnmGZ.js';
+import { j as GepaProposerConstraints, b as RunImprovementLoopResult } from './gepa-CEy1AIWp.js';
 export { IntegrityResult, IntegrityViolation, JourneySpec, PerfBaseline, PerfGateResult, PerfRegression, PerfScenario, PerfStat, ScenarioAxes, assertRecordIntegrity, checkRecordIntegrity, expandMatrix, gatePerf, scenarioKey, summarizeRecords } from './perf/index.js';
+export { AgentProfileRuntimeReceipt, ProductBenchmarkArm, ProductBenchmarkArtifactPaths, ProductBenchmarkBudgets, ProductBenchmarkManifest, ProductBenchmarkProfileRef, ProductBenchmarkRecord, ProductBenchmarkRepoRef, ProductBenchmarkRunInput, ProductBenchmarkScenario, ProductBenchmarkSplit, ProductBenchmarkSubstrateVersions, ProductBenchmarkValidationReport, RuntimeResolution, findProductBenchmarkArtifacts, productBenchmarkIntegrityFailures, productBenchmarkSplits, readProductBenchmarkManifest, readProductBenchmarkRecords, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun } from './product-benchmark/index.js';
 import '@ax-llm/ax';
 import 'zod';
 import './insight-report-C02J3q4T.js';
@@ -1969,6 +1970,55 @@ declare function blendHeldout(heldoutPassRate: number, judgeScore: number, weigh
  */
 declare function withHeldoutBlend<TArtifact>(score: (input: JudgeScoreInput<TArtifact>) => JudgeScore | Promise<JudgeScore>, heldoutPassRate: (artifact: TArtifact) => number, weights?: BlendWeights): (input: JudgeScoreInput<TArtifact>) => Promise<JudgeScore>;
 
+/**
+ * The agentic coding harnesses an eval sweeps by default — the ones we care about
+ * ranking. This is the SINGLE source of that list; consumers import it instead of
+ * re-declaring their own (a re-declared list is how the fleet drifts). Pass an
+ * explicit `harnesses` (e.g. `harnessTypeSchema.options` for literally every known
+ * harness) to widen beyond these.
+ */
+declare const CODING_HARNESSES: readonly HarnessType[];
+interface ProfileAxisSpec {
+    /** The domain profile to sweep. Its prompt/tools/skills are held fixed; only the
+     *  harness and model vary. `model.default` is the fallback model. */
+    base: AgentProfile;
+    /** Harnesses to cross. Default: {@link CODING_HARNESSES}. */
+    harnesses?: readonly HarnessType[];
+    /** Models to cross. Default: `[base.model.default]` — one model, i.e. today's
+     *  single-model behaviour, so omitting this never changes an existing run. */
+    models?: readonly string[];
+    /** Keep (harness, model) pairs the harness can't run instead of dropping them.
+     *  Default: drop (via `harnessSupportsModel`), so a vendor-locked harness paired
+     *  with a foreign model doesn't become a guaranteed-failing cell. */
+    keepIncompatible?: boolean;
+}
+/**
+ * Expand a base profile across the harness × model matrix into the `AgentProfile[]`
+ * that `runProfileMatrix` / `selfImprove` score — the ONE place "which harnesses ×
+ * which models do we evaluate" lives, so no product hand-rolls its own harness list
+ * or column→profile mapping (the pattern that let those copies drift and silently
+ * break the harness pivot).
+ *
+ * Each cell clones `base`, sets `model.default`, and stamps `metadata.harness` +
+ * `metadata.harnessModel` (both hash-bearing, so every cell gets a distinct
+ * `agentProfileId` row and results join back by harness/model via {@link harnessAxisOf}
+ * with no hand-recomputed key). Incompatible pairs are dropped unless `keepIncompatible`.
+ *
+ * Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
+ * everything we care about" switch, identical in shape whether one harness or all.
+ */
+declare function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[];
+/**
+ * Read the (harness, model) a matrix cell ran under, off a profile or a result row's
+ * profile — the join-back for a `byHarness` pivot. Returns undefined when the profile
+ * wasn't produced by {@link expandProfileAxes}. Callers group `result.byProfile` by
+ * this instead of recomputing an id (recomputing the wrong key is what broke the pivot
+ * in the hand-rolled copies).
+ */
+declare function harnessAxisOf(profile: Pick<AgentProfile, 'metadata'>): {
+    harness: HarnessType;
+    model: string;
+} | undefined;
 /**
  * Collision-resistant, path-safe, human-readable profile id for eval artifacts.
  * Scorecard joins still use `agentProfileHash`; this id is for run ids, matrix
@@ -2355,6 +2405,51 @@ declare class ExperimentTracker {
     verdictFor(experimentId: string): Promise<ImprovementVerdictResult>;
 }
 
+/**
+ * Cross-profile leaderboard — rank agent profiles (or any grouping) over a corpus
+ * of RunRecords on pass-rate with a Wilson CI, joined with the cost / token /
+ * latency rollup every benchmark wants next to the score.
+ *
+ * This is substrate, not benchmark-specific: any consumer of agent-eval gets a
+ * cross-harness × model leaderboard without re-implementing grouping, binomial
+ * CIs, or the observability join. The benchmark supplies only what "passed"
+ * means; everything else composes existing primitives (`wilson`, the RunRecord
+ * cost/token/wall fields, the AgentProfileCell grouping key).
+ */
+
+interface LeaderboardRow {
+    /** Group key — the agent-profile cellId by default, else a caller dimension. */
+    key: string;
+    /** Display label: `harness · model` when the profile carries them, else the key. */
+    label: string;
+    harness?: string;
+    model?: string;
+    rank: number;
+    /** Measured runs in this group. */
+    n: number;
+    passRate: number;
+    passRateCi95: [number, number];
+    /** Observability rollup — means over the group; null when no run reported it. */
+    meanCostUsd: number | null;
+    meanTokensIn: number | null;
+    meanTokensOut: number | null;
+    meanWallMs: number | null;
+}
+interface LeaderboardOptions {
+    /** Per-run pass predicate — the benchmark defines what "passed" means (real-build
+     *  hit-rate, test-pass, etc.). The leaderboard owns only the grouping + stats. */
+    passed: (record: RunRecord) => boolean;
+    /** Group key. Default: the agent-profile cellId (harness × model × dimensions),
+     *  so persona/harness/model sweeps separate without parsing labels. */
+    groupBy?: (record: RunRecord) => string;
+}
+/**
+ * Group `records`, compute pass-rate + Wilson CI + cost/token/latency means per
+ * group, and rank by pass-rate (cost as the tiebreaker — cheaper wins a tie).
+ * Pure projection: no I/O, deterministic, safe to call on any RunRecord[].
+ */
+declare function leaderboard(records: RunRecord[], opts: LeaderboardOptions): LeaderboardRow[];
+
 /**
  * muffled-gate-scanner — test helper that greps consumer source for
  * gate + measurement anti-patterns and fails with file:line locations.
@@ -5974,4 +6069,4 @@ type CachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1> = JudgeCo
  */
 declare function cachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1>(judge: JudgeConfig<TArtifact, TScenario>, store: VerdictCacheStore, options: CachedJudgeOptions): CachedJudge<TArtifact, TScenario>;
 
-export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, type CostReport, type CostSummary, CostTracker, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SpanPredicate, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, bisect, blendHeldout, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, sentenceReorderMutator, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
+export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, type CostReport, type CostSummary, CostTracker, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HarnessConfig, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatcherResult, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, ProductClient, ProductClientConfig, type ProfileAxisSpec, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, type SpanPredicate, type SplitGoldOptions, SteeringBundle, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, bisect, blendHeldout, buildAgreementJudge, buildDriverSystemPrompt, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, harnessAxisOf, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, leaderboard, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, index as profile, promptBisect, proposeSynthesisTargets, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runDistillation, runE2EWorkflow, runExpectations, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, sentenceReorderMutator, splitGold, statusAdvanced, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-QZYXA7ZO.js";
+} from "./chunk-X5OUZB4T.js";
 import {
   classifyEuAiRisk,
   euAiActReport,
@@ -34,6 +34,17 @@ import {
   summarizeRecords
 } from "./chunk-STGVSCDH.js";
 import {
+  findProductBenchmarkArtifacts,
+  productBenchmarkIntegrityFailures,
+  productBenchmarkSplits,
+  readProductBenchmarkManifest,
+  readProductBenchmarkRecords,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  validateProductBenchmarkRun
+} from "./chunk-63MBSQTX.js";
+import {
+  CODING_HARNESSES,
   JudgeParseError,
   adversarialJudge,
   agentProfileHash,
@@ -47,11 +58,13 @@ import {
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
   defaultJudges,
+  expandProfileAxes,
   extractProducedState,
+  harnessAxisOf,
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-VWQ6PO5O.js";
+} from "./chunk-GUII3E73.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -75,12 +88,17 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-2KTBHICD.js";
+} from "./chunk-NK77GPUH.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
+  cachedJudge,
+  canonicalJson,
+  contentHash,
+  fileVerdictCache,
+  inMemoryVerdictCache,
   summarizeBackendIntegrity
-} from "./chunk-HRGTA6U5.js";
+} from "./chunk-XIOQHCHU.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -4491,9 +4509,9 @@ function median(sorted) {
   const mid = Math.floor(n / 2);
   return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function stddev(values, mean5) {
+function stddev(values, mean6) {
   if (values.length < 2) return 0;
-  const variance = values.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / values.length;
+  const variance = values.reduce((acc, v) => acc + (v - mean6) ** 2, 0) / values.length;
   return Math.sqrt(variance);
 }
 function computeExperimentStats(reps, thresholds) {
@@ -4519,15 +4537,15 @@ function computeExperimentStats(reps, thresholds) {
     return r.score;
   });
   const sorted = [...scores2].sort((a, b) => a - b);
-  const mean5 = scores2.reduce((s, v) => s + v, 0) / n;
-  const sd = stddev(scores2, mean5);
+  const mean6 = scores2.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores2, mean6);
   const spread = iqr(scores2);
   const rated = reps.filter((r) => typeof r.passed === "boolean");
   const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
   const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
   return {
     median: median(sorted),
-    mean: mean5,
+    mean: mean6,
     min: sorted[0],
     max: sorted[n - 1],
     iqr: spread,
@@ -4697,6 +4715,49 @@ var ExperimentTracker = class {
   }
 };
 
+// src/leaderboard.ts
+function mean3(xs) {
+  return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null;
+}
+function leaderboard(records, opts) {
+  const keyOf = opts.groupBy ?? ((r) => r.agentProfile?.cellId ?? "ungrouped");
+  const groups = /* @__PURE__ */ new Map();
+  for (const r of records) {
+    const k = keyOf(r);
+    const g = groups.get(k);
+    if (g) g.push(r);
+    else groups.set(k, [r]);
+  }
+  const rows = [...groups.entries()].map(([key, rs]) => {
+    const passes = rs.filter((r) => opts.passed(r)).length;
+    const ci = wilson(passes, rs.length);
+    const profile = rs[0].agentProfile;
+    const harness = profile?.harness?.id;
+    const model = profile?.model;
+    return {
+      key,
+      label: harness && model ? `${harness} \xB7 ${model}` : key,
+      ...harness ? { harness } : {},
+      ...model ? { model } : {},
+      rank: 0,
+      n: rs.length,
+      passRate: ci.estimate,
+      passRateCi95: [ci.lower, ci.upper],
+      meanCostUsd: mean3(rs.map((r) => r.costUsd)),
+      meanTokensIn: mean3(rs.map((r) => r.tokenUsage.input)),
+      meanTokensOut: mean3(rs.map((r) => r.tokenUsage.output)),
+      meanWallMs: mean3(rs.map((r) => r.wallMs))
+    };
+  });
+  rows.sort(
+    (a, b) => b.passRate - a.passRate || (a.meanCostUsd ?? Infinity) - (b.meanCostUsd ?? Infinity)
+  );
+  rows.forEach((row, i) => {
+    row.rank = i + 1;
+  });
+  return rows;
+}
+
 // src/muffled-gate-scanner.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
 import { join } from "path";
@@ -6242,12 +6303,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean5 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance = all.reduce((a, v) => a + (v - mean5) ** 2, 0) / all.length;
+  const mean6 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance = all.reduce((a, v) => a + (v - mean6) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance);
-  const ref = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
+  const ref = Math.abs(mean6) > 1e-9 ? Math.abs(mean6) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean5, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean6, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6313,8 +6374,8 @@ async function paraphraseRobustnessScenarios(args) {
         });
         scores2.push(out.score);
       }
-      const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-      deltas[m.name] = mean5 - originalScore;
+      const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+      deltas[m.name] = mean6 - originalScore;
       paraphrasedAll.push(...scores2);
     }
     const paraphrasedMean = paraphrasedAll.length === 0 ? originalScore : paraphrasedAll.reduce((a, b) => a + b, 0) / paraphrasedAll.length;
@@ -6648,8 +6709,8 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
     const scores2 = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
     if (scores2.length < 3) continue;
-    const mean5 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
-    const variance = scores2.reduce((a, b) => a + (b - mean5) ** 2, 0) / scores2.length;
+    const mean6 = scores2.reduce((a, b) => a + b, 0) / scores2.length;
+    const variance = scores2.reduce((a, b) => a + (b - mean6) ** 2, 0) / scores2.length;
     if (variance > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -8104,7 +8165,7 @@ function decideReferenceReplayPromotion(baseline, candidate, policy = {}) {
       regressions
     };
   }
-  const requiredMeanDelta = mean3(compared.map((item) => item.f1Delta));
+  const requiredMeanDelta = mean4(compared.map((item) => item.f1Delta));
   if (requiredMeanDelta < minF1Delta) {
     return {
       promote: false,
@@ -8340,7 +8401,7 @@ function clamp012(value) {
 function sum2(values) {
   return values.reduce((acc, value) => acc + value, 0);
 }
-function mean3(values) {
+function mean4(values) {
   return values.length ? sum2(values) / values.length : 0;
 }
 function formatPct(value) {
@@ -8794,10 +8855,10 @@ var HeldOutGate = class {
       afterHoldout.push(run.outcome.holdoutScore);
     }
     const productiveRuns = beforeHoldout.length;
-    const candidateSearchMean = mean4(scores(candidate, "searchScore", "search"));
-    const candidateHoldoutMean = mean4(scores(candidate, "holdoutScore", "holdout"));
-    const baselineSearchMean = mean4(scores(baseline, "searchScore", "search"));
-    const baselineHoldoutMean = mean4(scores(baseline, "holdoutScore", "holdout"));
+    const candidateSearchMean = mean5(scores(candidate, "searchScore", "search"));
+    const candidateHoldoutMean = mean5(scores(candidate, "holdoutScore", "holdout"));
+    const baselineSearchMean = mean5(scores(baseline, "searchScore", "search"));
+    const baselineHoldoutMean = mean5(scores(baseline, "holdoutScore", "holdout"));
     const overfitGap = safeDiff(candidateSearchMean, candidateHoldoutMean);
     const baselineOverfitGap = safeDiff(baselineSearchMean, baselineHoldoutMean);
     const medianCandidateCost = medianFinite(candidate.map((r) => r.costUsd));
@@ -8909,7 +8970,7 @@ function scores(runs, field, splitFilter) {
   }
   return out;
 }
-function mean4(xs) {
+function mean5(xs) {
   if (xs.length === 0) return Number.NaN;
   return xs.reduce((s, x) => s + x, 0) / xs.length;
 }
@@ -8949,9 +9010,9 @@ function passOrthogonality(input) {
       sims.push(cosineSimilarity(vectors[i], vectors[j]));
     }
   }
-  const mean5 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
+  const mean6 = sims.length === 0 ? 0 : sims.reduce((a, b) => a + b, 0) / sims.length;
   return {
-    orthogonality: Math.max(0, Math.min(1, 1 - mean5)),
+    orthogonality: Math.max(0, Math.min(1, 1 - mean6)),
     passCount: passes.length,
     similarities: sims
   };
@@ -9893,134 +9954,6 @@ function resolveSeat(seats, seat, fallback) {
   throw new SeatUnsetError(seat);
 }
 
-// src/verdict-cache.ts
-import { createHash } from "crypto";
-import { appendFileSync as appendFileSync3, existsSync as existsSync5, readFileSync as readFileSync6 } from "fs";
-function canonicalizeAt(value, path) {
-  if (value === null) return "null";
-  switch (typeof value) {
-    case "boolean":
-      return value ? "true" : "false";
-    case "number":
-      if (!Number.isFinite(value)) {
-        throw new Error(
-          `canonicalJson: non-finite number (${value}) at ${path} \u2014 ambiguity is an error, not a coercion`
-        );
-      }
-      return JSON.stringify(value);
-    case "string":
-      return JSON.stringify(value);
-    case "undefined":
-    case "function":
-    case "symbol":
-      throw new Error(
-        `canonicalJson: ${typeof value} at ${path} \u2014 ambiguity is an error, not a coercion`
-      );
-    case "bigint":
-      throw new Error(`canonicalJson: bigint at ${path} \u2014 not representable in JSON`);
-    case "object":
-      break;
-  }
-  const obj = value;
-  if (typeof obj["toJSON"] === "function") {
-    return canonicalizeAt(obj.toJSON(), path);
-  }
-  if (Array.isArray(obj)) {
-    return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
-  }
-  if (obj instanceof Map || obj instanceof Set) {
-    throw new Error(
-      `canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} \u2014 would serialize as '{}'; convert to a plain object/array first`
-    );
-  }
-  const keys = Object.keys(obj).sort();
-  const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`);
-  return `{${parts.join(",")}}`;
-}
-function canonicalJson(value) {
-  return canonicalizeAt(value, "$");
-}
-function contentHash(value) {
-  return createHash("sha256").update(canonicalJson(value)).digest("hex");
-}
-function inMemoryVerdictCache() {
-  const entries = /* @__PURE__ */ new Map();
-  return {
-    get: (key) => entries.get(key),
-    set: (key, score) => {
-      entries.set(key, score);
-    }
-  };
-}
-function parseCacheLine(line, path, lineNo) {
-  let parsed;
-  try {
-    parsed = JSON.parse(line);
-  } catch (err) {
-    throw new Error(
-      `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} \u2014 ${err instanceof Error ? err.message : String(err)}`
-    );
-  }
-  const rec = parsed;
-  if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") {
-    throw new Error(
-      `fileVerdictCache: invalid record shape at ${path}:${lineNo} \u2014 expected {key, score:{dimensions, composite, notes}}`
-    );
-  }
-  return rec;
-}
-function fileVerdictCache(path) {
-  const entries = /* @__PURE__ */ new Map();
-  if (existsSync5(path)) {
-    const lines = readFileSync6(path, "utf8").split("\n");
-    for (let i = 0; i < lines.length; i++) {
-      const line = lines[i];
-      if (line === void 0 || line.trim() === "") continue;
-      const rec = parseCacheLine(line, path, i + 1);
-      entries.set(rec.key, rec.score);
-    }
-  }
-  return {
-    get: (key) => entries.get(key),
-    set: (key, score) => {
-      appendFileSync3(path, `${JSON.stringify({ key, score })}
-`, "utf8");
-      entries.set(key, score);
-    }
-  };
-}
-function cachedJudge(judge, store, options) {
-  if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") {
-    throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
-  }
-  const stats = { hits: 0, misses: 0 };
-  const wrapped = {
-    name: judge.name,
-    dimensions: judge.dimensions,
-    async score(input) {
-      const key = contentHash({
-        artifact: canonicalJson(input.artifact),
-        scenarioId: input.scenario.id,
-        judgeName: judge.name,
-        dimensions: judge.dimensions,
-        judgeVersion: options.judgeVersion
-      });
-      const cached = await store.get(key);
-      if (cached !== void 0) {
-        stats.hits += 1;
-        return cached;
-      }
-      const score = await judge.score(input);
-      await store.set(key, score);
-      stats.misses += 1;
-      return score;
-    },
-    stats: () => ({ ...stats })
-  };
-  if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
-  return wrapped;
-}
-
 // src/attestation.ts
 var ATTESTATION_ALGORITHM = "sha256/canonical-json";
 function attest(report, provenance) {
@@ -10067,6 +10000,7 @@ export {
   BenchmarkRunner,
   BudgetBreachError,
   BudgetGuard,
+  CODING_HARNESSES,
   CallbackResearcher,
   CaptureIntegrityError,
   ConfigError,
@@ -10344,6 +10278,7 @@ export {
   evaluateTraceContract,
   executeScenario,
   expandMatrix,
+  expandProfileAxes,
   expectAgent,
   exportRewardModel,
   exportRunAsOtlp,
@@ -10367,6 +10302,7 @@ export {
   findConstructorCwdDropped,
   findFallbackToPass,
   findLiteralTruePass,
+  findProductBenchmarkArtifacts,
   findSkipCountsAsPass,
   firstNumberAttr,
   firstStringAttr,
@@ -10390,6 +10326,7 @@ export {
   gradeSemanticStatus,
   groupBy,
   groupRunsByAgentProfileCell,
+  harnessAxisOf,
   hashContent,
   hashJson,
   hashScenarios,
@@ -10431,6 +10368,7 @@ export {
   judgeSpans,
   keyPreserved,
   knowledgeReadinessTracePayload,
+  leaderboard,
   linterJudge,
   llmJudge,
   llmSpanFromProvider,
@@ -10494,6 +10432,8 @@ export {
   preflightModels,
   printDriverSummary,
   probeLlm,
+  productBenchmarkIntegrityFailures,
+  productBenchmarkSplits,
   profile_exports as profile,
   projectOtlpFlatLine,
   projectRuntimeTrajectoryEvidence,
@@ -10503,6 +10443,8 @@ export {
   pytestTestParser,
   ranks,
   readOtlpStatus,
+  readProductBenchmarkManifest,
+  readProductBenchmarkRecords,
   recordRuns,
   recordRunsToScorecard,
   redTeamDataset,
@@ -10615,6 +10557,9 @@ export {
   userQuestionsForKnowledgeGaps,
   validateAgentProfileCell,
   validatePolicyEdit,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  validateProductBenchmarkRun,
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,