@tangle-network/agent-eval 0.121.0 → 0.122.0

package/dist/contract/index.d.ts

@@ -1,4 +1,4 @@
-import { AgentImprovementMeasuredComparison, Sha256Digest } from '@tangle-network/agent-interface';
+import { AgentCandidateExperiment, AgentCandidateBundle, AgentCandidateBenchmarkTask, AgentCandidateBenchmarkCellRef, AgentCandidateExperimentMeasurement, AgentImprovementMeasuredComparison, CandidateExecutionEvidence, AgentCandidateBenchmarkSuiteInputs, AgentCandidateBenchmarkTaskMaterial, AgentCandidateExperimentMaterial } from '@tangle-network/agent-interface';
 import { AxFunction, AxAIService } from '@ax-llm/ax';
 import { z } from 'zod';
 
@@ -1582,10 +1582,15 @@ interface ScoredSurfaceOutcome {
     surfaceHash: string;
     composite: number;
     dimensions: Record<string, number>;
+    /** Same per-scenario carrier shape as `GenerationCandidate.scenarios` and
+     *  `CampaignBreakdown.scenarios`: judge notes (the "why") + an optional
+     *  bounded `emitted` excerpt of the candidate's raw output (the "what it
+     *  actually did"). Copied verbatim from the measuring campaign's breakdown. */
     scenarios: Array<{
         scenarioId: string;
         composite: number;
         notes?: string;
+        emitted?: string;
     }>;
     coverage: {
         expectedCells: number;
@@ -1937,11 +1942,15 @@ interface GenerationCandidate {
      *  reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
      *  (which checks/lines/dimensions failed and how), NOT case-specific ground
      *  truth: leaking expected answers into the prompt is memorization, and the
-     *  held-out gate would reject it anyway. */
+     *  held-out gate would reject it anyway. `emitted` is a bounded excerpt of
+     *  the candidate's raw output for the scenario (worst rep when reps > 1) —
+     *  the "what it actually did" evidence; optional so trajectory capture is
+     *  never required of a dispatch. */
     scenarios: Array<{
         scenarioId: string;
         composite: number;
         notes?: string;
+        emitted?: string;
     }>;
     /** Proposer-supplied short label for the change. Present when the proposer
      *  returned a `ProposedCandidate`; absent for bare-surface mutators. */
@@ -4026,18 +4035,59 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
  */
 declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
 
-interface MeasuredComparisonFromSelfImproveResultOptions<TScenario extends Scenario, TArtifact> {
-    result: SelfImproveResult<TScenario, TArtifact>;
-    benchmark: AgentImprovementMeasuredComparison['benchmark'];
-    /** Canonical digest verified by the runtime that owns the baseline profile. */
-    baselineProfileDigest: Sha256Digest;
-    /** Canonical digest verified by the runtime that owns candidate sealing. */
-    candidateBundleDigest: Sha256Digest;
-    /** Exact baseline surface; contradictory recorded provenance is rejected. */
-    baselineSurface: MutableSurface;
+interface SealCandidateBenchmarkSuiteOptions {
+    tasks: [AgentCandidateBenchmarkTask, ...AgentCandidateBenchmarkTask[]];
+    reps: number;
+    seeds: [number, ...number[]];
+}
+interface CandidateExperimentExecutionInput {
+    experiment: AgentCandidateExperiment;
+    arm: 'baseline' | 'candidate';
+    bundle: AgentCandidateBundle;
+    task: AgentCandidateBenchmarkTask;
+    benchmarkCell: AgentCandidateBenchmarkCellRef;
+    seed: number;
+    signal?: AbortSignal;
+}
+interface RunCandidateExperimentOptions {
+    experiment: AgentCandidateExperiment;
+    execute(input: CandidateExperimentExecutionInput): Promise<CandidateExecutionEvidence>;
+    maxConcurrency?: number;
+    signal?: AbortSignal;
 }
-/** Convert one paired self-improvement result into the portable Interface evidence record. */
-declare function measuredComparisonFromSelfImproveResult<TScenario extends Scenario, TArtifact>(options: MeasuredComparisonFromSelfImproveResultOptions<TScenario, TArtifact>): AgentImprovementMeasuredComparison;
+interface CompareCandidateExperimentOptions {
+    experiment: AgentCandidateExperiment;
+    measurements: AgentCandidateExperimentMeasurement[];
+    runId: string;
+    candidate?: AgentImprovementMeasuredComparison['candidate'];
+    generationsExplored?: number;
+    searchDurationMs?: number;
+    searchCostUsd?: number;
+    metadata?: AgentImprovementMeasuredComparison['metadata'];
+}
+/** Content-address one task before any measured execution can see it. */
+declare function sealCandidateBenchmarkTask(material: AgentCandidateBenchmarkTaskMaterial): AgentCandidateBenchmarkTask;
+/** Freeze task order, repetitions, and every seed before either arm runs. */
+declare function sealCandidateBenchmarkSuite(options: SealCandidateBenchmarkSuiteOptions): AgentCandidateBenchmarkSuiteInputs;
+/** Freeze both complete agent states and their exact held-out work. */
+declare function sealCandidateExperiment(material: AgentCandidateExperimentMaterial): AgentCandidateExperiment;
+declare function verifyCandidateExperiment(input: unknown): AgentCandidateExperiment;
+/** Execute each signed cell for both arms. The callback is Runtime's one executor. */
+declare function runCandidateExperiment(options: RunCandidateExperimentOptions): Promise<AgentCandidateExperimentMeasurement[]>;
+/** Build the only publishable comparison: paired statistics over Runtime receipts. */
+declare function measuredComparisonFromCandidateExperiment(options: CompareCandidateExperimentOptions): AgentImprovementMeasuredComparison;
+/** Recompute every statistic and decision from the signed experiment receipts. */
+declare function verifyCandidateExperimentComparison(input: unknown): AgentImprovementMeasuredComparison;
+declare function verifyCandidateBenchmarkTask(input: unknown): AgentCandidateBenchmarkTask;
+declare function verifyCandidateBenchmarkSuiteInputs(input: unknown): AgentCandidateBenchmarkSuiteInputs;
+declare function verifyCandidateBenchmarkSuite(input: unknown): {
+    kind: "agent-candidate-benchmark-suite";
+    digestAlgorithm: "rfc8785-sha256";
+    taskDigests: [`sha256:${string}`, ...`sha256:${string}`[]];
+    reps: number;
+    seeds: [number, ...number[]];
+    digest: `sha256:${string}`;
+};
 
 /**
  * Typed Ax output for analyst findings.
@@ -4764,6 +4814,49 @@ interface PartitionByAuthoringModelResult {
 declare function partitionRunsByAuthoringModel(runs: RunRecord[], index: AgentTraceIndex): PartitionByAuthoringModelResult;
 
 type CodeAgentSessionSource = 'codex' | 'claude-code' | 'opencode' | 'kimi-code' | 'pi';
+type CodeAgentSessionTerminalStatus = 'completed' | 'failed' | 'unknown';
+type CodeAgentSessionActionKind = 'tool' | 'patch' | 'terminal' | 'graph-completion';
+type CodeAgentSessionActionSurface = 'tool' | 'mcp' | 'subagent' | 'skill' | 'hook' | 'web' | 'code';
+type CodeAgentSessionActionStatus = 'started' | 'completed' | 'failed' | 'unknown';
+interface CodeAgentSessionExecutionReceipt {
+    exitCode: number;
+    startedAtMs?: number;
+    completedAtMs?: number;
+}
+interface CodeAgentSessionAction {
+    id: string;
+    stepIndex: number;
+    kind: CodeAgentSessionActionKind;
+    surface: CodeAgentSessionActionSurface;
+    name: string;
+    status: CodeAgentSessionActionStatus;
+    timestampMs?: number;
+    costUsd?: number;
+    metadata: Record<string, unknown>;
+}
+interface CodeAgentSessionObservation {
+    source: CodeAgentSessionSource;
+    sessionId: string;
+    finalText: string | null;
+    terminal: {
+        status: CodeAgentSessionTerminalStatus;
+        explicit: boolean;
+    };
+    actions: CodeAgentSessionAction[];
+}
+interface ObserveCodeAgentSessionOptions {
+    source: CodeAgentSessionSource;
+    entries: unknown[];
+    sourcePath?: string;
+    execution?: CodeAgentSessionExecutionReceipt;
+}
+/**
+ * Project one provider session into the exact user-visible answer and a
+ * provider-neutral action stream. Raw prompts, tool inputs, and tool outputs
+ * stay out of this projection; callers retain the source JSONL as evidence.
+ */
+declare function observeCodeAgentSession(options: ObserveCodeAgentSessionOptions): CodeAgentSessionObservation;
+
 interface ParsedCodeAgentJsonl {
     entries: unknown[];
     malformedLines: number;
@@ -4783,6 +4876,12 @@ interface CodeAgentSessionMetrics {
     turnsCompleted: number;
     turnsAborted: number;
     contextCompactions: number;
+    mcpCalls: number;
+    subagentCalls: number;
+    skillCalls: number;
+    hookCalls: number;
+    webCalls: number;
+    codeActions: number;
     prLinks: number;
     fileSnapshots: number;
     graphNodes: number;
@@ -4810,6 +4909,7 @@ interface CodeAgentSessionDiagnostic {
     malformedLines: number;
     inferredScore: boolean;
     hasExplicitTerminalSignal: boolean;
+    hasFinalOutput: boolean;
     hasQualityLabel: boolean;
     hasTokenUsage: boolean;
     hasCost: boolean;
@@ -4820,6 +4920,7 @@ interface CodeAgentSessionIntakeResult {
     runs: RunRecord[];
     diagnostics: CodeAgentSessionDiagnostic[];
     metrics: CodeAgentSessionMetrics[];
+    observations: CodeAgentSessionObservation[];
 }
 interface CodeAgentSessionIntakeOptions {
     entries: unknown[];
@@ -4840,6 +4941,9 @@ interface CodeAgentSessionIntakeOptions {
      *  sentinel as observed. When omitted, source-reported cost wins, then a
      *  token-priced estimate, then uncaptured. */
     costProvenance?: RunCostProvenance;
+    /** Exact executor-owned process result. This is required when a provider's
+     * JSON stream has no terminal event, as with `opencode run --format json`. */
+    execution?: CodeAgentSessionExecutionReceipt;
 }
 declare function parseCodeAgentJsonl(jsonl: string): ParsedCodeAgentJsonl;
 declare function fromCodexSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
@@ -4969,4 +5073,4 @@ interface FromOtelSpansOptions {
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
+export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type CandidateExperimentExecutionInput, type ChatClient, type CodeAgentSessionAction, type CodeAgentSessionActionKind, type CodeAgentSessionActionStatus, type CodeAgentSessionActionSurface, type CodeAgentSessionDiagnostic, type CodeAgentSessionExecutionReceipt, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionObservation, type CodeAgentSessionSource, type CodeAgentSessionTerminalStatus, type CodeSurface, type CompareCandidateExperimentOptions, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunCandidateExperimentOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SealCandidateBenchmarkSuiteOptions, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, summarizeExecution, verifyCandidateBenchmarkSuite, verifyCandidateBenchmarkSuiteInputs, verifyCandidateBenchmarkTask, verifyCandidateExperiment, verifyCandidateExperimentComparison };