@tangle-network/agent-eval 0.119.1 → 0.120.0

package/dist/contract/index.d.ts

@@ -1387,6 +1387,10 @@ interface Scenario {
     kind: string;
     tags?: string[];
 }
+/** Redacted identity of a complete scenario payload retained in campaign results. */
+interface CampaignScenarioIdentity extends Pick<Scenario, 'id' | 'kind'> {
+    scenarioDigest: `sha256:${string}`;
+}
 /** Context handed to every dispatch invocation. Scoped — every
  *  trace/span carries the cellId, every artifact write lands under the cell's
  *  artifact root, the cost meter accumulates per cell. */
@@ -1964,7 +1968,11 @@ interface CampaignAggregates {
 interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
     /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
     manifestHash: string;
+    /** Canonical identity of the exact scenario payloads and replicate count. */
+    splitDigest: `sha256:${string}`;
     seed: number;
+    /** Replicates designed for every scenario in this campaign. */
+    reps: number;
     startedAt: string;
     endedAt: string;
     durationMs: number;
@@ -1978,13 +1986,14 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
     prUrl?: string;
     runDir: string;
     artifactsByPath: Record<string, string>;
-    /** Substrate strips the input scenarios to id+kind for the result manifest;
-     *  consumers needing full payload look it up via the original input. The
-     *  type parameter `TScenario` is propagated for downstream consumers that
-     *  want narrowed types when extending `CampaignResult`. */
-    scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
+    /** Redacted identities that let consumers verify the exact scenario payloads
+     *  without retaining customer task content in the result. */
+    scenarios: Array<CampaignScenarioIdentity & Pick<TScenario, 'id' | 'kind'>>;
 }
 
+/** Canonical identity of the exact scenario payloads and replicate count. */
+declare function campaignSplitDigest<TScenario extends Scenario>(scenarios: readonly TScenario[], reps: number): `sha256:${string}`;
+
 /**
  * `CampaignStorage` — the filesystem seam `runCampaign` writes through
  * (run/cell dirs, the resumability cache, per-cell artifacts, trace spans).
@@ -2319,8 +2328,8 @@ interface RunOptimizationResult<TArtifact, TScenario extends Scenario> {
  *
  * Hard-refuses unsafe configurations:
  *   - `tracing: 'off'` when a proposer is wired (improvement is unattributable)
- *   - `autoOnPromote: 'config'` — DEFERRED to Pass B; v0.40 only ships
- *     `'pr'` and `'none'`.
+ *   - `autoOnPromote: 'config'` — live mutation is unsupported without
+ *     isolated deployment, rollback, and independent validation.
  */
 
 type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptimizationOptions<TScenario, TArtifact> & {
@@ -2334,15 +2343,11 @@ type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptim
     /** What to do when the gate ships:
      *   - `'pr'`: open a PR via `openAutoPr`
      *   - `'none'`: just report — caller decides what to do with the winner
-     *  v0.40 does NOT support `'config'` (live-runtime self-mutation) —
-     *  deferred to Pass B behind safety stack. */
+     *  Live-runtime self-mutation is intentionally unsupported. */
     autoOnPromote: 'pr' | 'none';
     /** GH owner / repo for the auto-PR. Required when autoOnPromote === 'pr'. */
     ghOwner?: string;
     ghRepo?: string;
-    /** Optional render override — substrate writes a diff-shaped surface; pass
-     *  a function to format the promoted surface differently. */
-    renderPromotedDiff?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => string;
     /** Placebo control. When supplied AND the winner differs from baseline, the
      *  loop scores a THIRD holdout arm: the winner surface with its content
      *  footprint-matched-blanked by this function (typically via `neutralizeText`).
@@ -2355,6 +2360,8 @@ type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptim
 interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extends RunOptimizationResult<TArtifact, TScenario> {
     baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
     winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
+    neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario>;
+    neutralizedSurface?: MutableSurface;
     gateResult: Awaited<ReturnType<Gate<TArtifact, TScenario>['decide']>>;
     /** Unified baseline→winner surface diff. Computed UNCONDITIONALLY (not only
      *  when `autoOnPromote === 'pr'`) so the diff that the gate decided on is
@@ -3557,9 +3564,8 @@ interface PowerPreflight {
  *      reads). The hosted `/v1/ingest/traces` endpoint receives the FULL loop,
  *      not just the `cost.*` spans `runCampaign` already emits per cell.
  *
- * The record is built from the substrate's own loop result + the per-call
- * `RunRecord`s the worker emitted — no new measurement, no recomputation that
- * could drift from what the gate actually saw.
+ * The record is built from the loop result and its settled cost receipts — no
+ * second usage collector can contradict what the measured cells recorded.
  */
 
 interface LoopProvenanceCandidate {
@@ -3569,6 +3575,8 @@ interface LoopProvenanceCandidate {
     surfaceHash: string;
     /** Full sha256 content hash — byte-identical-verifiable. */
     contentHash: string;
+    /** Exact scored rows that produced this candidate's search result. */
+    campaignDigest: `sha256:${string}`;
     /** Proposer label, when the proposer returned a `ProposedCandidate`. */
     label?: string;
     /** Proposer rationale — the "because Z". When the proposer returned a bare
@@ -3602,13 +3610,33 @@ interface LoopProvenanceBackend {
     totalOutputTokens: number;
     totalCostUsd: number;
 }
+interface LoopProvenanceEvidence {
+    search: {
+        splitDigest: `sha256:${string}`;
+        baselineCampaignDigest: `sha256:${string}`;
+    };
+    holdout: {
+        splitDigest: `sha256:${string}`;
+        baselineCampaignDigest: `sha256:${string}`;
+        winnerCampaignDigest: `sha256:${string}`;
+        neutralized?: {
+            contentHash: `sha256:${string}`;
+            campaignDigest: `sha256:${string}`;
+            composite: number;
+            lift: number;
+        };
+    };
+    costReceiptsDigest: `sha256:${string}`;
+}
 /**
  * The durable provenance record. Aligns to the hosted `EvalRunEvent` path but
  * ADDS the rationale + the explicit baseline→candidate diff (both omitted from
  * the bare hosted event) + backend provenance.
  */
 interface LoopProvenanceRecord {
-    schema: 'tangle.loop-provenance.v3';
+    schema: 'tangle.loop-provenance';
+    /** SHA-256 over the canonical record with this field omitted. */
+    recordDigest: `sha256:${string}`;
     runId: string;
     runDir: string;
     timestamp: string;
@@ -3622,6 +3650,8 @@ interface LoopProvenanceRecord {
     diff: string;
     /** Every candidate across every generation, with its rationale and structured cause. */
     candidates: LoopProvenanceCandidate[];
+    /** Exact campaign, split, surface, and receipt identities behind every summary. */
+    evidence: LoopProvenanceEvidence;
     /** Baseline composite on the search split that generated the candidates. */
     baselineSearchComposite: number;
     /** The gate verdict — decision + reasons + contributing gates + delta. */
@@ -3632,6 +3662,7 @@ interface LoopProvenanceRecord {
         contributingGates: Array<{
             name: string;
             passed: boolean;
+            detail: unknown;
         }>;
     };
     /** baseline-on-holdout composite mean. */
@@ -3784,17 +3815,6 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
      *  Default `mem://selfImprove-<timestamp>` (in-memory, non-durable). Pass a
      *  real path to persist the provenance record + spans. */
     runDir?: string;
-    /**
-     * Worker call records for backend provenance. The agent is opaque to the
-     * substrate (it returns an artifact, not token usage), so to capture an
-     * `assertRealBackend`-grade verdict + worker call count + model in the
-     * provenance record, the agent reports its per-call `RunRecord`s here.
-     * Called once after the loop; return the records the agent accumulated.
-     * When unset, backend provenance is derived from campaign cells (cost only;
-     * verdict will read `stub` without token usage — the honest signal that no
-     * token channel was wired).
-     */
-    collectWorkerRecords?: () => RunRecord[];
     /** Fires once the durable provenance record + OTel spans are emitted.
      *  Receives the structured record for inline assertions / custom routing. */
     onProvenance?: (record: LoopProvenanceRecord) => void;
@@ -4006,8 +4026,12 @@ declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults
 interface MeasuredComparisonFromSelfImproveResultOptions<TScenario extends Scenario, TArtifact> {
     result: SelfImproveResult<TScenario, TArtifact>;
     benchmark: AgentImprovementMeasuredComparison['benchmark'];
+    /** Canonical digest verified by the runtime that owns the baseline profile. */
     baselineProfileDigest: Sha256Digest;
+    /** Canonical digest verified by the runtime that owns candidate sealing. */
     candidateBundleDigest: Sha256Digest;
+    /** Exact baseline surface; contradictory recorded provenance is rejected. */
+    baselineSurface: MutableSurface;
 }
 /** Convert one paired self-improvement result into the portable Interface evidence record. */
 declare function measuredComparisonFromSelfImproveResult<TScenario extends Scenario, TArtifact>(options: MeasuredComparisonFromSelfImproveResultOptions<TScenario, TArtifact>): AgentImprovementMeasuredComparison;
@@ -4942,4 +4966,4 @@ interface FromOtelSpansOptions {
 }
 declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
 
-export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
+export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };