@tangle-network/agent-eval 0.126.2 → 0.126.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -1
- package/README.md +21 -1
- package/dist/benchmarks/index.js +4 -4
- package/dist/campaign/index.d.ts +29 -1
- package/dist/campaign/index.js +3 -3
- package/dist/{chunk-7AN2E7BU.js → chunk-3I74FLK6.js} +40 -24
- package/dist/chunk-3I74FLK6.js.map +1 -0
- package/dist/{chunk-W4L6C2XT.js → chunk-CGG5SLH3.js} +2 -2
- package/dist/{chunk-SDPM6554.js → chunk-K6IAZZ6L.js} +2 -2
- package/dist/{chunk-Y5CLI4PY.js → chunk-P6WN2KF5.js} +3 -3
- package/dist/{chunk-UCLVDLCH.js → chunk-ZVCHKKOP.js} +76 -8
- package/dist/chunk-ZVCHKKOP.js.map +1 -0
- package/dist/contract/index.d.ts +29 -1
- package/dist/contract/index.js +3 -3
- package/dist/index.js +5 -5
- package/dist/openapi.json +1 -1
- package/dist/{run-campaign-LVFKZCEU.js → run-campaign-FFRM3RH5.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-7AN2E7BU.js.map +0 -1
- package/dist/chunk-UCLVDLCH.js.map +0 -1
- /package/dist/{chunk-W4L6C2XT.js.map → chunk-CGG5SLH3.js.map} +0 -0
- /package/dist/{chunk-SDPM6554.js.map → chunk-K6IAZZ6L.js.map} +0 -0
- /package/dist/{chunk-Y5CLI4PY.js.map → chunk-P6WN2KF5.js.map} +0 -0
- /package/dist/{run-campaign-LVFKZCEU.js.map → run-campaign-FFRM3RH5.js.map} +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,7 +4,17 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
## [
|
|
7
|
+
## [0.126.3] - 2026-07-24 - execution-bound optimizer resume
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `runCampaign({ abortOnCellError: true })` stops scheduling after the first dispatch or judge error, aborts and drains active sibling cells, then rejects with the original error.
|
|
12
|
+
- Failed cells write `<cell>/failure-receipt.json` before cancellation, including the exact cell result, call IDs, and settled agent-plus-judge cost and token totals.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- Official GEPA, SkillOpt, and generic external optimizer resume identities include the caller's exact dispatch identity, so changed agent, retrieval, judge, model, or service behavior cannot restore incompatible optimizer state.
|
|
17
|
+
- `compareOptimizationMethods()` passes its final-scoring dispatch identity into method optimization and rejects conflicting identities.
|
|
8
18
|
|
|
9
19
|
## [0.126.2] - 2026-07-24 - fail-closed candidate ranking
|
|
10
20
|
|
package/README.md
CHANGED
|
@@ -90,6 +90,26 @@ console.log(
|
|
|
90
90
|
Each call runs every case, records the artifact, applies the same judge, and returns score distributions.
|
|
91
91
|
The surface is the value being changed, such as a prompt, skill, or serialized configuration.
|
|
92
92
|
|
|
93
|
+
### Stop after the first failed cell
|
|
94
|
+
|
|
95
|
+
`runCampaign()` normally records a dispatch or judge error on that cell and continues the remaining cases.
|
|
96
|
+
Set `abortOnCellError: true` when another failed cell would only waste time or money:
|
|
97
|
+
|
|
98
|
+
```ts
|
|
99
|
+
await runCampaign({
|
|
100
|
+
scenarios,
|
|
101
|
+
dispatch,
|
|
102
|
+
judges: [judge],
|
|
103
|
+
runDir: 'release-candidate',
|
|
104
|
+
abortOnCellError: true,
|
|
105
|
+
})
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The failed cell is written first to `<runDir>/<cell>/failure-receipt.json`.
|
|
109
|
+
That receipt contains the original error, the cell result, exact call IDs, and settled agent-plus-judge cost and token totals.
|
|
110
|
+
Active sibling cells are cancelled and allowed to finish recording their own receipts before the campaign rejects with the original cell error.
|
|
111
|
+
Leaving `abortOnCellError` unset preserves continue-on-error behavior.
|
|
112
|
+
|
|
93
113
|
## Adapt Another Text Optimizer
|
|
94
114
|
|
|
95
115
|
Use `externalTextOptimizationMethod()` when an existing package owns search and selection for a text prompt or named text components.
|
|
@@ -97,7 +117,7 @@ Its `run` callback receives the starting candidate plus serialized train and sel
|
|
|
97
117
|
The optimizer must score candidates through `context.evaluate()` so Agent Eval can enforce the evaluation limit and use the configured execution and judges.
|
|
98
118
|
Every optimizer-owned paid call must use `context.cost.runPaidCall()`.
|
|
99
119
|
Set `source` to the package version and revision, and set `evaluationId` to a commit, content hash, or other stable identity for the execution and scoring behavior.
|
|
100
|
-
Agent Eval derives the run identity from those values, the optimizer settings, the starting surface, the described data, and the seed.
|
|
120
|
+
Agent Eval derives the run identity from those values, the exact dispatch identity, the optimizer settings, the starting surface, the described data, and the seed.
|
|
101
121
|
The callback returns the selected candidate, whether compatible state was restored, and how optimizer spend was recorded.
|
|
102
122
|
|
|
103
123
|
See [Adapt A Third-Party Text Optimizer](./docs/campaign-proposers.md#adapt-a-third-party-text-optimizer) for a complete minimal adapter.
|
package/dist/benchmarks/index.js
CHANGED
|
@@ -16,10 +16,10 @@ import {
|
|
|
16
16
|
routing_exports,
|
|
17
17
|
runBenchmarkAdapter,
|
|
18
18
|
summarizeBenchmarkCampaign
|
|
19
|
-
} from "../chunk-
|
|
20
|
-
import "../chunk-
|
|
21
|
-
import "../chunk-
|
|
22
|
-
import "../chunk-
|
|
19
|
+
} from "../chunk-CGG5SLH3.js";
|
|
20
|
+
import "../chunk-P6WN2KF5.js";
|
|
21
|
+
import "../chunk-3I74FLK6.js";
|
|
22
|
+
import "../chunk-ZVCHKKOP.js";
|
|
23
23
|
import "../chunk-WGXIEX7P.js";
|
|
24
24
|
import "../chunk-ARU2PZFM.js";
|
|
25
25
|
import "../chunk-J5SQWP6Y.js";
|
package/dist/campaign/index.d.ts
CHANGED
|
@@ -2368,6 +2368,14 @@ interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
|
|
|
2368
2368
|
costTags?: Readonly<Record<string, string>>;
|
|
2369
2369
|
/** Max concurrent cells. Default 2. */
|
|
2370
2370
|
maxConcurrency?: number;
|
|
2371
|
+
/**
|
|
2372
|
+
* Stop after the first dispatch or judge error. The failed cell is persisted
|
|
2373
|
+
* before active sibling cells are aborted and drained, then the campaign
|
|
2374
|
+
* rejects with the exact error thrown by that dispatch or judge.
|
|
2375
|
+
* Default false preserves the normal behavior of returning failed cells and
|
|
2376
|
+
* continuing the remaining schedule.
|
|
2377
|
+
*/
|
|
2378
|
+
abortOnCellError?: boolean;
|
|
2371
2379
|
/**
|
|
2372
2380
|
* Per-cell dispatch deadline in ms. A `dispatch` that neither resolves nor
|
|
2373
2381
|
* rejects within this window is a hang (a stalled model request, an
|
|
@@ -2432,6 +2440,26 @@ interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
|
|
|
2432
2440
|
generation?: number;
|
|
2433
2441
|
}) => string | undefined;
|
|
2434
2442
|
}
|
|
2443
|
+
/** Durable `<cell>/failure-receipt.json` written before a failed cell can
|
|
2444
|
+
* trigger campaign-wide cancellation. The cell keeps its dispatch-only usage
|
|
2445
|
+
* fields for compatibility; `cost` covers every settled agent and judge call
|
|
2446
|
+
* attributed to this exact run attempt. */
|
|
2447
|
+
interface CampaignCellFailureReceipt<TArtifact = unknown> {
|
|
2448
|
+
schemaVersion: 1;
|
|
2449
|
+
runAttemptId: string;
|
|
2450
|
+
recordedAt: string;
|
|
2451
|
+
failure: {
|
|
2452
|
+
stage: 'dispatch' | 'judge';
|
|
2453
|
+
judge?: string;
|
|
2454
|
+
error: {
|
|
2455
|
+
name: string;
|
|
2456
|
+
message: string;
|
|
2457
|
+
stack?: string;
|
|
2458
|
+
};
|
|
2459
|
+
};
|
|
2460
|
+
cell: CampaignCellResult<TArtifact>;
|
|
2461
|
+
cost: CostLedgerSummary;
|
|
2462
|
+
}
|
|
2435
2463
|
/**
|
|
2436
2464
|
* Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records.
|
|
2437
2465
|
*/
|
|
@@ -6076,4 +6104,4 @@ declare function verifyCodeSurface(surface: CodeSurface, worktreeDir?: string):
|
|
|
6076
6104
|
* identity against the checkout at `worktreeRef`. */
|
|
6077
6105
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
6078
6106
|
|
|
6079
|
-
export { type AnalystArtifact, type AnalystScenario, type AnalyzeCrossSurfaceInteractionsInput, type AxisEvidence, type AxisVerdict, type BuildAnalystSurfaceDispatchOptions, type BuildEvidenceVectorOptions, type BuildLoopProvenanceArgs, type CampaignAggregates, type CampaignArtifactWriter, type CampaignBreakdown, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignRunPlan, type CampaignRunPlanCell, type CampaignScenarioIdentity, type CampaignStorage, type CampaignTokenUsage, type CampaignTraceWriter, type CodeSurface, type CodeSurfaceVerification, type CompareOptimizationMethodsOptions, type ComparisonCost, type ComponentSurface, type CostLedgerHandle, type CrossSurfaceAdditionDecision, type CrossSurfaceAdditionRejectionReason, type CrossSurfaceAttemptCompleteness, type CrossSurfaceBestSingleSelection, type CrossSurfaceBootstrapPolicy, type CrossSurfaceCandidate, type CrossSurfaceCandidateComparison, type CrossSurfaceCandidateEvidence, type CrossSurfaceCandidateOutcome, type CrossSurfaceCandidateSummary, type CrossSurfaceComponent, type CrossSurfaceComponentEvidence, type CrossSurfaceCompositionStep, type CrossSurfaceDistribution, type CrossSurfaceEligibility, type CrossSurfaceEvidenceBreakdown, type CrossSurfaceIneligibilityReason, type CrossSurfaceInteractionAwareSelection, type CrossSurfaceInteractionEffect, type CrossSurfaceInteractionPath, type CrossSurfaceInteractionReport, type CrossSurfaceInteractionTask, type CrossSurfaceNaiveStackSelection, type CrossSurfacePairCompatibility, type CrossSurfacePairEvidence, type CrossSurfacePairIncompatibilityReason, type CrossSurfacePairwiseEntry, type CrossSurfaceRankedSingle, type CrossSurfaceRelativeCost, type CrossSurfaceSelectionPolicy, type CrossSurfaceSelections, type CrossSurfaceTaskRow, type DefaultProductionGateOptions, type DimensionRegression, type DiscriminationScore, type DispatchContext, type DispatchFn, type EmitLoopProvenanceArgs, type EmitLoopProvenanceResult, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type EvidenceVector, type ExternalOptimizationExample, type ExternalTextEvaluationResponse, type ExternalTextOptimizationMethodConfig, type ExternalTextOptimizerContext, type ExternalTextOptimizerResult, type FailureModeRecallJudgeOptions, FileSearchLedger, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaAdaptiveEngineRun, type GepaEngineOptions, type GepaEngineRun, type GepaOptimizationMethodConfig, type GepaOptimizationRecipe, type GepaRunnerCommand, type GitWorktreeAdapterOptions, type HeldOutGateOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type JudgeAggregate, type JudgeConfig, type JudgeDimension, type JudgeScore, type LabelTrust, type LabeledScenarioRecord, type LabeledScenarioSampleArgs, type LabeledScenarioSource, type LabeledScenarioStore, LabeledScenarioStoreError, type LabeledScenarioWrite, type LlmJudgeDimension, type LlmJudgeOptions, type LoadEvalFixtureScenariosOptions, type LoopProvenanceArgsFromResult, type LoopProvenanceBackend, type LoopProvenanceCandidate, type LoopProvenanceEvidence, type LoopProvenanceOptimizationMethod, type LoopProvenanceRecord, type MutableSurface, type NeutralizationGateOptions, type ObjectiveSource, type OpenAICompatibleOptimizerModel, type OpenAutoPrOptions, type OpenAutoPrResult, type OpenSearchLedgerOptions, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodPairwise, type OptimizationMethodProvenance, type OptimizationMethodResult, type OptimizationMethodRunOptions, type OptimizationMethodScore, type OptimizationPackageSource, type OptimizationProposer, type OptimizationTokenUsage, type OptimizerConfig, type OptimizerModelBudget, type PairedHoldout, type ParetoParent, type ParetoSignificanceGateOptions, type PendingCostCallView, type PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PowerPreflight, type PowerPreflightOptions, type PremeasuredOptimizationBaseline, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, type PromotionObjective, type PromotionPolicy, type ProposalTrackContext, type ProposeContext, type ProposedCandidate, type RedactionStatus, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceScenario, type RolloutArgumentDiff, type RolloutArgumentDiffOptions, type RolloutCall, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunOptimizationOptions, type RunOptimizationResult, type RunProfileMatrixOptions, type RunProfileMatrixResult, SEARCH_LEDGER_SCHEMA, type Scenario, type ScenarioAggregate, type ScenarioRollup, type ScenarioSignal, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type ScoredRollout, type ScoredSurfaceOutcome, type SearchAccountingAudit, type SearchArtifactRef, type SearchAttemptAccounting, type SearchCandidateDecidedEvent, type SearchCandidateLineage, type SearchCandidateRegisteredEvent, type SearchCandidateSlot, type SearchCandidateSlotClosedEvent, type SearchCandidateSurface, type SearchCompletedEvent, type SearchCostAccounting, type SearchFailureReason, type SearchLedger, type SearchLedgerAppendResult, SearchLedgerConflictError, type SearchLedgerEntry, SearchLedgerError, type SearchLedgerEvent, type SearchLedgerHash, SearchLedgerIntegrityError, type SearchLedgerReplay, type SearchModelIdentity, type SearchOperationKind, type SearchOperationRecordedEvent, type SearchPlan, type SearchPlannedEvent, type SearchPlannedOperation, type SearchPlannedTask, type SearchSourceRef, type SearchSurfaceEffect, type SearchSurfaceEvidence, type SearchSurfaceKind, type SearchTaskAttemptedEvent, type SearchTaskOutcome, type SearchTokenAccounting, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SessionScript, type SingleRunLock, type SingleRunLockOptions, type SkillOptOptimizationMethodConfig, type SkillOptRunnerCommand, type SkillOptTrainerConfig, type SurfaceProposer, type TraceSpan, type TransientFailureOptions, type UngroundedLiteralReport, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, acquireSingleRunLock, analyzeCrossSurfaceInteractions, assertCampaignDesign, assertCampaignSplitIdentity, assertCodeSurfaceIdentity, assertComponentSurface, buildAnalystSurfaceDispatch, buildEvidenceVector, buildLoopProvenanceRecord, campaignBreakdown, campaignMeanComposite, campaignMeasurementDigest, campaignScenarioIdentity, campaignSplitDigest, campaignSplitDigestFromIdentities, canonicalDigest, classifyUngroundedLiterals, codeSurfaceIdentityMaterial, compareOptimizationMethods, compareRankKeys, componentSurfaceIdentityMaterial, composeGate, costFromLedgerSummary, createReferenceEquivalenceJudge, createRunCostLedger, defaultProductionGate, detectScale, dimensionRegressions, discoverEvalFixtures, emitLoopProvenance, externalTextOptimizationMethod, failureModeRecallJudge, fsCampaignStorage, gepaOptimizationMethod, gitWorktreeAdapter, heldOutGate, heldoutSignificance, inMemoryCampaignStorage, isProposedCandidate, isTransientTransportFailure, labelTrustRank, llmJudge, loadEvalFixture, loadEvalFixtureScenarios, loopProvenanceArgsFromResult, loopProvenanceSpans, makePlaybackDispatch, neutralizationGate, neutralizeText, openAutoPr, openSearchLedger, optimizationTokenUsageFromSummary, pairHoldout, paretoPolicy, paretoSignificanceGate, planCampaignRun, planEvalFixtureRun, powerPreflight, provenanceRecordPath, provenanceSpansPath, renderScoreboardMarkdown, renderSurfaceDiff, resolveRunDir, resolveWorktreePath, rolloutArgumentDiff, runCampaign, runEval, runImprovementLoop, runOptimization, runProfileMatrix, scoreDiscrimination, scoreUserStory, scoreboardSummary, selectDiscriminative, sequentialDecide, sequentialPairedGate, skillOptOptimizationMethod, surfaceContentHash, surfaceHash, tangleTracesRoot, userStoryScoreboard, validateSearchLedgerEvent, verifyCodeSurface, verifyLoopProvenanceRecord };
|
|
6107
|
+
export { type AnalystArtifact, type AnalystScenario, type AnalyzeCrossSurfaceInteractionsInput, type AxisEvidence, type AxisVerdict, type BuildAnalystSurfaceDispatchOptions, type BuildEvidenceVectorOptions, type BuildLoopProvenanceArgs, type CampaignAggregates, type CampaignArtifactWriter, type CampaignBreakdown, type CampaignCellFailureReceipt, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignRunPlan, type CampaignRunPlanCell, type CampaignScenarioIdentity, type CampaignStorage, type CampaignTokenUsage, type CampaignTraceWriter, type CodeSurface, type CodeSurfaceVerification, type CompareOptimizationMethodsOptions, type ComparisonCost, type ComponentSurface, type CostLedgerHandle, type CrossSurfaceAdditionDecision, type CrossSurfaceAdditionRejectionReason, type CrossSurfaceAttemptCompleteness, type CrossSurfaceBestSingleSelection, type CrossSurfaceBootstrapPolicy, type CrossSurfaceCandidate, type CrossSurfaceCandidateComparison, type CrossSurfaceCandidateEvidence, type CrossSurfaceCandidateOutcome, type CrossSurfaceCandidateSummary, type CrossSurfaceComponent, type CrossSurfaceComponentEvidence, type CrossSurfaceCompositionStep, type CrossSurfaceDistribution, type CrossSurfaceEligibility, type CrossSurfaceEvidenceBreakdown, type CrossSurfaceIneligibilityReason, type CrossSurfaceInteractionAwareSelection, type CrossSurfaceInteractionEffect, type CrossSurfaceInteractionPath, type CrossSurfaceInteractionReport, type CrossSurfaceInteractionTask, type CrossSurfaceNaiveStackSelection, type CrossSurfacePairCompatibility, type CrossSurfacePairEvidence, type CrossSurfacePairIncompatibilityReason, type CrossSurfacePairwiseEntry, type CrossSurfaceRankedSingle, type CrossSurfaceRelativeCost, type CrossSurfaceSelectionPolicy, type CrossSurfaceSelections, type CrossSurfaceTaskRow, type DefaultProductionGateOptions, type DimensionRegression, type DiscriminationScore, type DispatchContext, type DispatchFn, type EmitLoopProvenanceArgs, type EmitLoopProvenanceResult, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type EvidenceVector, type ExternalOptimizationExample, type ExternalTextEvaluationResponse, type ExternalTextOptimizationMethodConfig, type ExternalTextOptimizerContext, type ExternalTextOptimizerResult, type FailureModeRecallJudgeOptions, FileSearchLedger, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaAdaptiveEngineRun, type GepaEngineOptions, type GepaEngineRun, type GepaOptimizationMethodConfig, type GepaOptimizationRecipe, type GepaRunnerCommand, type GitWorktreeAdapterOptions, type HeldOutGateOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type JudgeAggregate, type JudgeConfig, type JudgeDimension, type JudgeScore, type LabelTrust, type LabeledScenarioRecord, type LabeledScenarioSampleArgs, type LabeledScenarioSource, type LabeledScenarioStore, LabeledScenarioStoreError, type LabeledScenarioWrite, type LlmJudgeDimension, type LlmJudgeOptions, type LoadEvalFixtureScenariosOptions, type LoopProvenanceArgsFromResult, type LoopProvenanceBackend, type LoopProvenanceCandidate, type LoopProvenanceEvidence, type LoopProvenanceOptimizationMethod, type LoopProvenanceRecord, type MutableSurface, type NeutralizationGateOptions, type ObjectiveSource, type OpenAICompatibleOptimizerModel, type OpenAutoPrOptions, type OpenAutoPrResult, type OpenSearchLedgerOptions, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodPairwise, type OptimizationMethodProvenance, type OptimizationMethodResult, type OptimizationMethodRunOptions, type OptimizationMethodScore, type OptimizationPackageSource, type OptimizationProposer, type OptimizationTokenUsage, type OptimizerConfig, type OptimizerModelBudget, type PairedHoldout, type ParetoParent, type ParetoSignificanceGateOptions, type PendingCostCallView, type PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PowerPreflight, type PowerPreflightOptions, type PremeasuredOptimizationBaseline, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, type PromotionObjective, type PromotionPolicy, type ProposalTrackContext, type ProposeContext, type ProposedCandidate, type RedactionStatus, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceScenario, type RolloutArgumentDiff, type RolloutArgumentDiffOptions, type RolloutCall, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunOptimizationOptions, type RunOptimizationResult, type RunProfileMatrixOptions, type RunProfileMatrixResult, SEARCH_LEDGER_SCHEMA, type Scenario, type ScenarioAggregate, type ScenarioRollup, type ScenarioSignal, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type ScoredRollout, type ScoredSurfaceOutcome, type SearchAccountingAudit, type SearchArtifactRef, type SearchAttemptAccounting, type SearchCandidateDecidedEvent, type SearchCandidateLineage, type SearchCandidateRegisteredEvent, type SearchCandidateSlot, type SearchCandidateSlotClosedEvent, type SearchCandidateSurface, type SearchCompletedEvent, type SearchCostAccounting, type SearchFailureReason, type SearchLedger, type SearchLedgerAppendResult, SearchLedgerConflictError, type SearchLedgerEntry, SearchLedgerError, type SearchLedgerEvent, type SearchLedgerHash, SearchLedgerIntegrityError, type SearchLedgerReplay, type SearchModelIdentity, type SearchOperationKind, type SearchOperationRecordedEvent, type SearchPlan, type SearchPlannedEvent, type SearchPlannedOperation, type SearchPlannedTask, type SearchSourceRef, type SearchSurfaceEffect, type SearchSurfaceEvidence, type SearchSurfaceKind, type SearchTaskAttemptedEvent, type SearchTaskOutcome, type SearchTokenAccounting, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SessionScript, type SingleRunLock, type SingleRunLockOptions, type SkillOptOptimizationMethodConfig, type SkillOptRunnerCommand, type SkillOptTrainerConfig, type SurfaceProposer, type TraceSpan, type TransientFailureOptions, type UngroundedLiteralReport, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, acquireSingleRunLock, analyzeCrossSurfaceInteractions, assertCampaignDesign, assertCampaignSplitIdentity, assertCodeSurfaceIdentity, assertComponentSurface, buildAnalystSurfaceDispatch, buildEvidenceVector, buildLoopProvenanceRecord, campaignBreakdown, campaignMeanComposite, campaignMeasurementDigest, campaignScenarioIdentity, campaignSplitDigest, campaignSplitDigestFromIdentities, canonicalDigest, classifyUngroundedLiterals, codeSurfaceIdentityMaterial, compareOptimizationMethods, compareRankKeys, componentSurfaceIdentityMaterial, composeGate, costFromLedgerSummary, createReferenceEquivalenceJudge, createRunCostLedger, defaultProductionGate, detectScale, dimensionRegressions, discoverEvalFixtures, emitLoopProvenance, externalTextOptimizationMethod, failureModeRecallJudge, fsCampaignStorage, gepaOptimizationMethod, gitWorktreeAdapter, heldOutGate, heldoutSignificance, inMemoryCampaignStorage, isProposedCandidate, isTransientTransportFailure, labelTrustRank, llmJudge, loadEvalFixture, loadEvalFixtureScenarios, loopProvenanceArgsFromResult, loopProvenanceSpans, makePlaybackDispatch, neutralizationGate, neutralizeText, openAutoPr, openSearchLedger, optimizationTokenUsageFromSummary, pairHoldout, paretoPolicy, paretoSignificanceGate, planCampaignRun, planEvalFixtureRun, powerPreflight, provenanceRecordPath, provenanceSpansPath, renderScoreboardMarkdown, renderSurfaceDiff, resolveRunDir, resolveWorktreePath, rolloutArgumentDiff, runCampaign, runEval, runImprovementLoop, runOptimization, runProfileMatrix, scoreDiscrimination, scoreUserStory, scoreboardSummary, selectDiscriminative, sequentialDecide, sequentialPairedGate, skillOptOptimizationMethod, surfaceContentHash, surfaceHash, tangleTracesRoot, userStoryScoreboard, validateSearchLedgerEvent, verifyCodeSurface, verifyLoopProvenanceRecord };
|
package/dist/campaign/index.js
CHANGED
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
userStoryScoreboard,
|
|
33
33
|
validateSearchLedgerEvent,
|
|
34
34
|
verifyCodeSurface
|
|
35
|
-
} from "../chunk-
|
|
35
|
+
} from "../chunk-P6WN2KF5.js";
|
|
36
36
|
import {
|
|
37
37
|
acquireSingleRunLock,
|
|
38
38
|
assertCodeSurfaceIdentity,
|
|
@@ -79,7 +79,7 @@ import {
|
|
|
79
79
|
surfaceContentHash,
|
|
80
80
|
surfaceHash,
|
|
81
81
|
verifyLoopProvenanceRecord
|
|
82
|
-
} from "../chunk-
|
|
82
|
+
} from "../chunk-3I74FLK6.js";
|
|
83
83
|
import {
|
|
84
84
|
SearchLedgerConflictError,
|
|
85
85
|
SearchLedgerError,
|
|
@@ -96,7 +96,7 @@ import {
|
|
|
96
96
|
resolveRunDir,
|
|
97
97
|
runCampaign,
|
|
98
98
|
tangleTracesRoot
|
|
99
|
-
} from "../chunk-
|
|
99
|
+
} from "../chunk-ZVCHKKOP.js";
|
|
100
100
|
import "../chunk-WGXIEX7P.js";
|
|
101
101
|
import "../chunk-ARU2PZFM.js";
|
|
102
102
|
import "../chunk-J5SQWP6Y.js";
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
runCampaign,
|
|
14
14
|
summarizeAgentReceiptIntegrity,
|
|
15
15
|
tryAcquireAtomicFileLock
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-ZVCHKKOP.js";
|
|
17
17
|
import {
|
|
18
18
|
clamp01,
|
|
19
19
|
combineAbortSignals,
|
|
@@ -2040,6 +2040,17 @@ function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
|
2040
2040
|
"compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided"
|
|
2041
2041
|
);
|
|
2042
2042
|
}
|
|
2043
|
+
const optimizationDispatchRef = opts.optimizationRunOptions?.dispatchRef;
|
|
2044
|
+
if (optimizationDispatchRef !== void 0 && (typeof optimizationDispatchRef !== "string" || optimizationDispatchRef.trim().length === 0 || optimizationDispatchRef.trim() !== optimizationDispatchRef)) {
|
|
2045
|
+
throw new Error(
|
|
2046
|
+
"compareOptimizationMethods: optimizationRunOptions.dispatchRef must be trimmed and non-empty when provided"
|
|
2047
|
+
);
|
|
2048
|
+
}
|
|
2049
|
+
if (opts.dispatchRef !== void 0 && optimizationDispatchRef !== void 0 && opts.dispatchRef !== optimizationDispatchRef) {
|
|
2050
|
+
throw new Error(
|
|
2051
|
+
"compareOptimizationMethods: dispatchRef must match optimizationRunOptions.dispatchRef when both are provided"
|
|
2052
|
+
);
|
|
2053
|
+
}
|
|
2043
2054
|
try {
|
|
2044
2055
|
surfaceContentHash(opts.baselineSurface);
|
|
2045
2056
|
} catch (cause) {
|
|
@@ -2211,6 +2222,7 @@ function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, b
|
|
|
2211
2222
|
seed,
|
|
2212
2223
|
runOptions: Object.freeze({
|
|
2213
2224
|
...opts.optimizationRunOptions ?? {},
|
|
2225
|
+
...opts.optimizationRunOptions?.dispatchRef === void 0 && opts.dispatchRef !== void 0 ? { dispatchRef: opts.dispatchRef } : {},
|
|
2214
2226
|
...signal ? { signal } : {}
|
|
2215
2227
|
}),
|
|
2216
2228
|
costLedger
|
|
@@ -3800,28 +3812,30 @@ function externalTextOptimizationMethod(config) {
|
|
|
3800
3812
|
)
|
|
3801
3813
|
);
|
|
3802
3814
|
const methodDir = `${input.runDir}/external/${safePathComponent(snapshot.name)}`;
|
|
3803
|
-
const
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
...snapshot.source.revision ? { revision: snapshot.source.revision } : {}
|
|
3811
|
-
},
|
|
3812
|
-
method: snapshot.name,
|
|
3813
|
-
evaluationId: snapshot.evaluationId,
|
|
3814
|
-
objective: snapshot.objective,
|
|
3815
|
-
background: snapshot.background ?? "",
|
|
3816
|
-
seed: input.seed,
|
|
3817
|
-
seedCandidate,
|
|
3818
|
-
trainSet,
|
|
3819
|
-
selectionSet,
|
|
3820
|
-
maxEvaluations: snapshot.maxEvaluations,
|
|
3821
|
-
maxOptimizerCostUsd: snapshot.maxOptimizerCostUsd,
|
|
3822
|
-
maxCandidateChars,
|
|
3823
|
-
maxEvidenceChars
|
|
3815
|
+
const runMaterial = {
|
|
3816
|
+
optimizer: {
|
|
3817
|
+
kind: snapshot.source.kind,
|
|
3818
|
+
package: snapshot.source.package,
|
|
3819
|
+
version: snapshot.source.version,
|
|
3820
|
+
...snapshot.source.sourceUrl ? { sourceUrl: snapshot.source.sourceUrl } : {},
|
|
3821
|
+
...snapshot.source.revision ? { revision: snapshot.source.revision } : {}
|
|
3824
3822
|
},
|
|
3823
|
+
method: snapshot.name,
|
|
3824
|
+
evaluationId: snapshot.evaluationId,
|
|
3825
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
3826
|
+
objective: snapshot.objective,
|
|
3827
|
+
background: snapshot.background ?? "",
|
|
3828
|
+
seed: input.seed,
|
|
3829
|
+
seedCandidate,
|
|
3830
|
+
trainSet,
|
|
3831
|
+
selectionSet,
|
|
3832
|
+
maxEvaluations: snapshot.maxEvaluations,
|
|
3833
|
+
maxOptimizerCostUsd: snapshot.maxOptimizerCostUsd,
|
|
3834
|
+
maxCandidateChars,
|
|
3835
|
+
maxEvidenceChars
|
|
3836
|
+
};
|
|
3837
|
+
const runId = externalOptimizerRunKey({
|
|
3838
|
+
material: runMaterial,
|
|
3825
3839
|
attemptId,
|
|
3826
3840
|
resumeEnabled: resume !== "never"
|
|
3827
3841
|
});
|
|
@@ -5496,6 +5510,7 @@ function gepaOptimizationMethod(config) {
|
|
|
5496
5510
|
runtime: runtimeIdentity,
|
|
5497
5511
|
method: name,
|
|
5498
5512
|
evaluationId: config.evaluationId,
|
|
5513
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
5499
5514
|
seed: input.seed,
|
|
5500
5515
|
recipe: snapshotJson(config.recipe, "GEPA run settings"),
|
|
5501
5516
|
engineModules: config.engineModules ?? [],
|
|
@@ -6090,7 +6105,7 @@ async function runImprovementLoop(opts) {
|
|
|
6090
6105
|
const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
|
|
6091
6106
|
const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
|
|
6092
6107
|
const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
|
|
6093
|
-
const { runCampaign: runCampaign2 } = await import("./run-campaign-
|
|
6108
|
+
const { runCampaign: runCampaign2 } = await import("./run-campaign-FFRM3RH5.js");
|
|
6094
6109
|
const holdoutDeferred = (opts.holdout ?? "measured") === "deferred";
|
|
6095
6110
|
const baselineOnHoldout = holdoutDeferred ? await runCampaign2({
|
|
6096
6111
|
...opts,
|
|
@@ -7033,6 +7048,7 @@ function skillOptOptimizationMethod(config) {
|
|
|
7033
7048
|
runtime: runtimeIdentity,
|
|
7034
7049
|
method: name,
|
|
7035
7050
|
evaluationId: config.evaluationId,
|
|
7051
|
+
dispatchRef: input.runOptions.dispatchRef ?? null,
|
|
7036
7052
|
seed: input.seed,
|
|
7037
7053
|
trainer: snapshotJson(config.trainer, "SkillOpt run settings"),
|
|
7038
7054
|
objective: config.objective,
|
|
@@ -7325,4 +7341,4 @@ export {
|
|
|
7325
7341
|
emitLoopProvenance,
|
|
7326
7342
|
skillOptOptimizationMethod
|
|
7327
7343
|
};
|
|
7328
|
-
//# sourceMappingURL=chunk-
|
|
7344
|
+
//# sourceMappingURL=chunk-3I74FLK6.js.map
|