@tangle-network/agent-eval 0.123.4 → 0.123.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +214 -147
- package/dist/campaign/index.js +4 -2
- package/dist/{chunk-QBRSJK47.js → chunk-J3LHTAAB.js} +47 -47
- package/dist/chunk-J3LHTAAB.js.map +1 -0
- package/dist/{chunk-SUN7QLPB.js → chunk-KKPPFIDS.js} +87 -87
- package/dist/{chunk-SUN7QLPB.js.map → chunk-KKPPFIDS.js.map} +1 -1
- package/dist/{chunk-LT4J7ULK.js → chunk-VPDOSN3L.js} +1731 -1282
- package/dist/chunk-VPDOSN3L.js.map +1 -0
- package/dist/contract/index.d.ts +3 -0
- package/dist/contract/index.js +2 -1
- package/dist/contract/index.js.map +1 -1
- package/dist/index.d.ts +115 -1
- package/dist/index.js +120 -3
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +1 -1
- package/docs/campaign-proposers.md +66 -0
- package/package.json +3 -2
- package/dist/chunk-LT4J7ULK.js.map +0 -1
- package/dist/chunk-QBRSJK47.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
8
8
|
|
|
9
9
|
### Added
|
|
10
10
|
|
|
11
|
+
- `selfImprove({ budget: { candidateConcurrency } })` exposes the existing `runOptimization()` control for scoring candidate campaigns in parallel; it remains opt-in and defaults to one candidate campaign at a time.
|
|
11
12
|
- `llmPolicyEditProposer()` and `projectPolicyEditHistory()` accept `scenarioOrder: 'input'` when controlled comparisons must preserve first-occurrence caller order; ranked evidence selection remains the default.
|
|
12
13
|
- `callLlmJson()` accepts `jsonPayloadMode: 'exact'` when callers must reject fenced, prose-wrapped, or multi-root responses instead of extracting a JSON value.
|
|
13
14
|
- `llmPolicyEditProposer({ redactCurrentSurfaceForModel })` can remove credentials and unrelated fields from the current surface sent to the model while applying validated edits to the complete original surface.
|
|
@@ -52,6 +53,14 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
|
|
|
52
53
|
- `runSkillOpt({ holdoutScenarios })` fails closed because those rows are adaptively reused.
|
|
53
54
|
Pass `selectionScenarios`; selection result fields now use `Selection` instead of `Holdout`, and `lift` is now `selectionLift`.
|
|
54
55
|
|
|
56
|
+
## [0.123.6] — 2026-07-23 — optimization and rollout exports
|
|
57
|
+
|
|
58
|
+
### Added
|
|
59
|
+
|
|
60
|
+
- `mintRolloutRows()` joins existing run records and traces into `tangle.rollout.v1` rows; `toSftRows()`, `toRewardRows()`, and `toJsonl()` serialize clean-success and reward-labeled training data while preserving missing-trace and realness-block evidence.
|
|
61
|
+
- `gepaOptimizationMethod()` delegates bounded single-engine and Omni-shaped `optimize_best_of()` then `optimize_anything()` recipes to GEPA through the optional `agent-eval-rpc[gepa]` bridge.
|
|
62
|
+
The caller chooses the train and selection fields sent to GEPA, final comparison cases remain inside `agent-eval`, and proposer spend without agent-eval receipts is reported as incomplete.
|
|
63
|
+
|
|
55
64
|
## [0.122.2] — 2026-07-17 — premeasured optimization continuation
|
|
56
65
|
|
|
57
66
|
### Added
|
package/dist/benchmarks/index.js
CHANGED
|
@@ -17,8 +17,8 @@ import {
|
|
|
17
17
|
runBenchmarkAdapter,
|
|
18
18
|
summarizeBenchmarkCampaign
|
|
19
19
|
} from "../chunk-JKDNAOF5.js";
|
|
20
|
-
import "../chunk-
|
|
21
|
-
import "../chunk-
|
|
20
|
+
import "../chunk-VPDOSN3L.js";
|
|
21
|
+
import "../chunk-KKPPFIDS.js";
|
|
22
22
|
import "../chunk-D5JZ7UDZ.js";
|
|
23
23
|
import "../chunk-MHPEGJHC.js";
|
|
24
24
|
import "../chunk-ARU2PZFM.js";
|
package/dist/campaign/index.d.ts
CHANGED
|
@@ -3905,6 +3905,219 @@ declare function dimensionRegressions(candidate: Map<string, Record<string, Judg
|
|
|
3905
3905
|
seed?: number;
|
|
3906
3906
|
}): DimensionRegression[];
|
|
3907
3907
|
|
|
3908
|
+
/**
|
|
3909
|
+
* Compare optimization methods on shared train, selection, and test data.
|
|
3910
|
+
* Optimizers receive only train and selection data. After every optimizer
|
|
3911
|
+
* finishes, their selected surfaces are measured on the same untouched test
|
|
3912
|
+
* data and compared with paired confidence intervals.
|
|
3913
|
+
*/
|
|
3914
|
+
|
|
3915
|
+
/** Per-method campaign settings. Each method receives its own spend account. */
|
|
3916
|
+
type OptimizationMethodRunOptions<TScenario extends Scenario, TArtifact> = Omit<RunCampaignOptions<TScenario, TArtifact>, 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed'>;
|
|
3917
|
+
/** Cost reported by a method or by final test scoring. */
|
|
3918
|
+
interface ComparisonCost {
|
|
3919
|
+
totalCostUsd: number;
|
|
3920
|
+
accountingComplete: boolean;
|
|
3921
|
+
incompleteReasons: string[];
|
|
3922
|
+
}
|
|
3923
|
+
/** Shared inputs for one optimization method. Final test data is absent. */
|
|
3924
|
+
interface OptimizationMethodInput<TScenario extends Scenario, TArtifact> {
|
|
3925
|
+
/** Surface every method starts from. */
|
|
3926
|
+
readonly baselineSurface: MutableSurface;
|
|
3927
|
+
/** Evidence used to author or fit candidates. */
|
|
3928
|
+
readonly trainScenarios: readonly TScenario[];
|
|
3929
|
+
/** Data used for candidate acceptance, early stopping, and model selection. */
|
|
3930
|
+
readonly selectionScenarios: readonly TScenario[];
|
|
3931
|
+
/** Runs one scenario with a candidate surface. */
|
|
3932
|
+
readonly dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
3933
|
+
/** Scores artifacts produced by `dispatchWithSurface`. */
|
|
3934
|
+
readonly judges: readonly JudgeConfig<TArtifact, TScenario>[];
|
|
3935
|
+
/** Method-specific artifacts are written below this directory. */
|
|
3936
|
+
readonly runDir: string;
|
|
3937
|
+
readonly seed: number;
|
|
3938
|
+
/** Shared defaults for every method. A method may override them explicitly. */
|
|
3939
|
+
readonly runOptions: Readonly<OptimizationMethodRunOptions<TScenario, TArtifact>>;
|
|
3940
|
+
}
|
|
3941
|
+
interface OptimizationMethodResult {
|
|
3942
|
+
/** Surface selected without using the final test partition. */
|
|
3943
|
+
winnerSurface: MutableSurface;
|
|
3944
|
+
/** Optimization spend. Excludes final test scoring. */
|
|
3945
|
+
cost: ComparisonCost;
|
|
3946
|
+
/** Optimization duration. Excludes final test scoring. */
|
|
3947
|
+
durationMs?: number;
|
|
3948
|
+
}
|
|
3949
|
+
/** A complete optimization method, including candidate generation and selection. */
|
|
3950
|
+
interface OptimizationMethod<TScenario extends Scenario = Scenario, TArtifact = unknown> {
|
|
3951
|
+
/** Unique, trimmed display name. Its normalized form must also be unique. */
|
|
3952
|
+
name: string;
|
|
3953
|
+
optimize: (input: OptimizationMethodInput<TScenario, TArtifact>) => Promise<OptimizationMethodResult>;
|
|
3954
|
+
}
|
|
3955
|
+
interface OptimizationMethodScore {
|
|
3956
|
+
name: string;
|
|
3957
|
+
/** Mean final-test composite of the baseline (identical across methods). */
|
|
3958
|
+
baselineComposite: number;
|
|
3959
|
+
/** Mean final-test composite of this method's selected surface. */
|
|
3960
|
+
winnerComposite: number;
|
|
3961
|
+
/** Mean per-scenario final-test lift (winner minus baseline). */
|
|
3962
|
+
lift: number;
|
|
3963
|
+
/** Simultaneous paired-bootstrap interval for per-scenario lift.
|
|
3964
|
+
* `low > 0` excludes zero after adjustment for all reported contrasts. */
|
|
3965
|
+
liftCi: {
|
|
3966
|
+
low: number;
|
|
3967
|
+
high: number;
|
|
3968
|
+
};
|
|
3969
|
+
/** Optimization spend reported by the method. Excludes final test scoring. */
|
|
3970
|
+
optimizationCost: ComparisonCost;
|
|
3971
|
+
/** Optimization duration reported by the method. Excludes final test scoring. */
|
|
3972
|
+
durationMs?: number;
|
|
3973
|
+
/** Paired final-test values used to compute lift and its interval. */
|
|
3974
|
+
scenarioScores: Array<{
|
|
3975
|
+
scenarioId: string;
|
|
3976
|
+
baselineComposite: number;
|
|
3977
|
+
winnerComposite: number;
|
|
3978
|
+
lift: number;
|
|
3979
|
+
}>;
|
|
3980
|
+
winnerSurface: MutableSurface;
|
|
3981
|
+
/** 1-based, by descending lift. */
|
|
3982
|
+
rank: number;
|
|
3983
|
+
}
|
|
3984
|
+
interface OptimizationMethodPairwise {
|
|
3985
|
+
/** Higher-ranked method. */
|
|
3986
|
+
a: string;
|
|
3987
|
+
b: string;
|
|
3988
|
+
/** Mean per-scenario untouched-test delta (a − b). */
|
|
3989
|
+
deltaMean: number;
|
|
3990
|
+
low: number;
|
|
3991
|
+
high: number;
|
|
3992
|
+
/** `a` if the CI clears 0, `b` if it is entirely negative, else `'tie'`. */
|
|
3993
|
+
favored: string;
|
|
3994
|
+
}
|
|
3995
|
+
interface OptimizationMethodComparison {
|
|
3996
|
+
/** Sorted by descending lift; `rank` set accordingly. */
|
|
3997
|
+
scores: OptimizationMethodScore[];
|
|
3998
|
+
best: OptimizationMethodScore;
|
|
3999
|
+
/** Best vs each other method, using simultaneous paired-bootstrap intervals. */
|
|
4000
|
+
pairwise: OptimizationMethodPairwise[];
|
|
4001
|
+
testScenarioIds: string[];
|
|
4002
|
+
/** Sum of the costs reported by every optimization method. */
|
|
4003
|
+
optimizationCost: ComparisonCost;
|
|
4004
|
+
/** Baseline and distinct winner scoring on the final test partition. */
|
|
4005
|
+
testCost: ComparisonCost;
|
|
4006
|
+
/** Optimization plus final test scoring. */
|
|
4007
|
+
totalCost: ComparisonCost;
|
|
4008
|
+
/** Caller-requested simultaneous coverage across all reported contrasts. */
|
|
4009
|
+
confidence: number;
|
|
4010
|
+
/** Bonferroni-adjusted confidence used for each bootstrap interval. */
|
|
4011
|
+
intervalConfidence: number;
|
|
4012
|
+
/** Method-vs-baseline plus all possible method-vs-method contrasts. */
|
|
4013
|
+
comparisonCount: number;
|
|
4014
|
+
/** Deterministic bootstrap and campaign seed. */
|
|
4015
|
+
seed: number;
|
|
4016
|
+
/** Bootstrap draws used for each interval. */
|
|
4017
|
+
resamples: number;
|
|
4018
|
+
/** Agent runs averaged within each test scenario before resampling scenarios. */
|
|
4019
|
+
reps: number;
|
|
4020
|
+
}
|
|
4021
|
+
interface CompareOptimizationMethodsOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch' | 'judges' | 'scenarios'> {
|
|
4022
|
+
methods: OptimizationMethod<TScenario, TArtifact>[];
|
|
4023
|
+
baselineSurface: MutableSurface;
|
|
4024
|
+
/** Evidence used by every optimizer to author or fit candidates. */
|
|
4025
|
+
trainScenarios: TScenario[];
|
|
4026
|
+
/** Candidate acceptance, early-stopping, and optimizer-selection data. */
|
|
4027
|
+
selectionScenarios: TScenario[];
|
|
4028
|
+
/** Untouched final comparison data. Never passed to an optimization method. */
|
|
4029
|
+
testScenarios: TScenario[];
|
|
4030
|
+
/** Scores a surface on a scenario. The methods and final test share this function. */
|
|
4031
|
+
dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
4032
|
+
judges: JudgeConfig<TArtifact, TScenario>[];
|
|
4033
|
+
/** Bootstrap resamples for the lift intervals. Default is at least 2000 and
|
|
4034
|
+
* rises when the requested simultaneous confidence needs finer tails. */
|
|
4035
|
+
resamples?: number;
|
|
4036
|
+
/** Shared defaults for each method's train and selection campaigns. */
|
|
4037
|
+
optimizationRunOptions?: OptimizationMethodRunOptions<TScenario, TArtifact>;
|
|
4038
|
+
/** Number of optimization methods to run concurrently. Default 1. */
|
|
4039
|
+
optimizationConcurrency?: number;
|
|
4040
|
+
/** Simultaneous confidence across method-vs-baseline and method-vs-method contrasts.
|
|
4041
|
+
* Each bootstrap interval is Bonferroni-adjusted. Default 0.95. */
|
|
4042
|
+
confidence?: number;
|
|
4043
|
+
/** Shared spend limit across baseline and winner scoring on the final test partition.
|
|
4044
|
+
* Each method owns its optimization budget through `optimizationRunOptions.costCeiling`. */
|
|
4045
|
+
costCeiling?: number;
|
|
4046
|
+
}
|
|
4047
|
+
/**
|
|
4048
|
+
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
4049
|
+
*/
|
|
4050
|
+
declare function compareOptimizationMethods<TScenario extends Scenario, TArtifact>(opts: CompareOptimizationMethodsOptions<TScenario, TArtifact>): Promise<OptimizationMethodComparison>;
|
|
4051
|
+
/** Keep the cost fields a custom optimization method must report. */
|
|
4052
|
+
declare function costFromLedgerSummary(summary: CostLedgerSummary): ComparisonCost;
|
|
4053
|
+
|
|
4054
|
+
/** One bounded GEPA engine invocation inside a GEPA recipe. */
|
|
4055
|
+
interface GepaEngineRun {
|
|
4056
|
+
/** GEPA engine name. GEPA validates names available in its Python runtime. */
|
|
4057
|
+
engine: string;
|
|
4058
|
+
/** Maximum callback evaluations this engine may consume. */
|
|
4059
|
+
maxEvaluations: number;
|
|
4060
|
+
/** Required cap for this engine's own model or CLI spend. */
|
|
4061
|
+
maxProposerCostUsd: number;
|
|
4062
|
+
/** GEPA engine-specific configuration. Passed through without interpretation. */
|
|
4063
|
+
engineConfig?: Record<string, unknown>;
|
|
4064
|
+
}
|
|
4065
|
+
/**
|
|
4066
|
+
* A direct mapping to a GEPA optimization recipe.
|
|
4067
|
+
*
|
|
4068
|
+
* `best-of-then-continue` calls GEPA's `optimize_best_of`, then calls its
|
|
4069
|
+
* `optimize_anything` once more with the winning candidate. This is the
|
|
4070
|
+
* published Omni shape when `explore` contains GEPA, AutoResearch, and
|
|
4071
|
+
* Meta-Harness. Its total callback limit is the sum of the bounded runs.
|
|
4072
|
+
*/
|
|
4073
|
+
type GepaOptimizationRecipe = {
|
|
4074
|
+
kind: 'engine';
|
|
4075
|
+
run: GepaEngineRun;
|
|
4076
|
+
} | {
|
|
4077
|
+
kind: 'best-of-then-continue';
|
|
4078
|
+
explore: readonly GepaEngineRun[];
|
|
4079
|
+
continueWith: GepaEngineRun;
|
|
4080
|
+
};
|
|
4081
|
+
/** The command that runs the optional Python GEPA bridge. */
|
|
4082
|
+
interface GepaRunnerCommand {
|
|
4083
|
+
/** Default: `python`. */
|
|
4084
|
+
command?: string;
|
|
4085
|
+
/** Default: `['-m', 'agent_eval_rpc.gepa_bridge']`. */
|
|
4086
|
+
args?: readonly string[];
|
|
4087
|
+
cwd?: string;
|
|
4088
|
+
env?: NodeJS.ProcessEnv;
|
|
4089
|
+
}
|
|
4090
|
+
interface GepaOptimizationMethodConfig<TScenario extends Scenario> {
|
|
4091
|
+
/** Unique comparison-method name. Default identifies the GEPA recipe. */
|
|
4092
|
+
name?: string;
|
|
4093
|
+
/** A direct GEPA recipe. */
|
|
4094
|
+
recipe: GepaOptimizationRecipe;
|
|
4095
|
+
/** Plain-language goal shown to the external optimizer. */
|
|
4096
|
+
objective: string;
|
|
4097
|
+
/** Optional bounded context about the surface and task. */
|
|
4098
|
+
background?: string;
|
|
4099
|
+
/** Reject external candidates longer than this. Default: 200,000 characters. */
|
|
4100
|
+
maxCandidateChars?: number;
|
|
4101
|
+
/** End the bridge process after this many milliseconds. Default: 30 minutes. */
|
|
4102
|
+
timeoutMs?: number;
|
|
4103
|
+
/**
|
|
4104
|
+
* Decide what the external optimizer may read for a train or selection case.
|
|
4105
|
+
* The returned value must be JSON-serializable. The final comparison cases
|
|
4106
|
+
* are not accepted by this API and cannot be serialized here.
|
|
4107
|
+
*/
|
|
4108
|
+
describeScenario?: (scenario: TScenario) => unknown;
|
|
4109
|
+
runner?: GepaRunnerCommand;
|
|
4110
|
+
}
|
|
4111
|
+
/**
|
|
4112
|
+
* Turn an optional GEPA installation into an `OptimizationMethod`.
|
|
4113
|
+
*
|
|
4114
|
+
* GEPA receives only serialized train and selection cases. The caller's final
|
|
4115
|
+
* test partition stays inside `compareOptimizationMethods`, which invokes this
|
|
4116
|
+
* method without a test-set field. The local callback routes every candidate
|
|
4117
|
+
* evaluation through the same dispatch and judges used by other methods.
|
|
4118
|
+
*/
|
|
4119
|
+
declare function gepaOptimizationMethod<TScenario extends Scenario, TArtifact>(config: GepaOptimizationMethodConfig<TScenario>): OptimizationMethod<TScenario, TArtifact>;
|
|
4120
|
+
|
|
3908
4121
|
/**
|
|
3909
4122
|
* Evidence grounding for reflective optimizers (GEPA-style revise loops).
|
|
3910
4123
|
*
|
|
@@ -4194,152 +4407,6 @@ interface ParameterSweepProposerOptions {
|
|
|
4194
4407
|
/** Config/parameter-level proposer for FAPO's middle escalation level. */
|
|
4195
4408
|
declare function parameterSweepProposer(opts: ParameterSweepProposerOptions): SurfaceProposer;
|
|
4196
4409
|
|
|
4197
|
-
/**
|
|
4198
|
-
* Compare optimization methods on shared train, selection, and test data.
|
|
4199
|
-
* Optimizers receive only train and selection data. After every optimizer
|
|
4200
|
-
* finishes, their selected surfaces are measured on the same untouched test
|
|
4201
|
-
* data and compared with paired confidence intervals.
|
|
4202
|
-
*/
|
|
4203
|
-
|
|
4204
|
-
/** Per-method campaign settings. Each method receives its own spend account. */
|
|
4205
|
-
type OptimizationMethodRunOptions<TScenario extends Scenario, TArtifact> = Omit<RunCampaignOptions<TScenario, TArtifact>, 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed'>;
|
|
4206
|
-
/** Cost reported by a method or by final test scoring. */
|
|
4207
|
-
interface ComparisonCost {
|
|
4208
|
-
totalCostUsd: number;
|
|
4209
|
-
accountingComplete: boolean;
|
|
4210
|
-
incompleteReasons: string[];
|
|
4211
|
-
}
|
|
4212
|
-
/** Shared inputs for one optimization method. Final test data is absent. */
|
|
4213
|
-
interface OptimizationMethodInput<TScenario extends Scenario, TArtifact> {
|
|
4214
|
-
/** Surface every method starts from. */
|
|
4215
|
-
readonly baselineSurface: MutableSurface;
|
|
4216
|
-
/** Evidence used to author or fit candidates. */
|
|
4217
|
-
readonly trainScenarios: readonly TScenario[];
|
|
4218
|
-
/** Data used for candidate acceptance, early stopping, and model selection. */
|
|
4219
|
-
readonly selectionScenarios: readonly TScenario[];
|
|
4220
|
-
/** Runs one scenario with a candidate surface. */
|
|
4221
|
-
readonly dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
4222
|
-
/** Scores artifacts produced by `dispatchWithSurface`. */
|
|
4223
|
-
readonly judges: readonly JudgeConfig<TArtifact, TScenario>[];
|
|
4224
|
-
/** Method-specific artifacts are written below this directory. */
|
|
4225
|
-
readonly runDir: string;
|
|
4226
|
-
readonly seed: number;
|
|
4227
|
-
/** Shared defaults for every method. A method may override them explicitly. */
|
|
4228
|
-
readonly runOptions: Readonly<OptimizationMethodRunOptions<TScenario, TArtifact>>;
|
|
4229
|
-
}
|
|
4230
|
-
interface OptimizationMethodResult {
|
|
4231
|
-
/** Surface selected without using the final test partition. */
|
|
4232
|
-
winnerSurface: MutableSurface;
|
|
4233
|
-
/** Optimization spend. Excludes final test scoring. */
|
|
4234
|
-
cost: ComparisonCost;
|
|
4235
|
-
/** Optimization duration. Excludes final test scoring. */
|
|
4236
|
-
durationMs?: number;
|
|
4237
|
-
}
|
|
4238
|
-
/** A complete optimization method, including candidate generation and selection. */
|
|
4239
|
-
interface OptimizationMethod<TScenario extends Scenario = Scenario, TArtifact = unknown> {
|
|
4240
|
-
/** Unique, trimmed display name. Its normalized form must also be unique. */
|
|
4241
|
-
name: string;
|
|
4242
|
-
optimize: (input: OptimizationMethodInput<TScenario, TArtifact>) => Promise<OptimizationMethodResult>;
|
|
4243
|
-
}
|
|
4244
|
-
interface OptimizationMethodScore {
|
|
4245
|
-
name: string;
|
|
4246
|
-
/** Mean final-test composite of the baseline (identical across methods). */
|
|
4247
|
-
baselineComposite: number;
|
|
4248
|
-
/** Mean final-test composite of this method's selected surface. */
|
|
4249
|
-
winnerComposite: number;
|
|
4250
|
-
/** Mean per-scenario final-test lift (winner minus baseline). */
|
|
4251
|
-
lift: number;
|
|
4252
|
-
/** Simultaneous paired-bootstrap interval for per-scenario lift.
|
|
4253
|
-
* `low > 0` excludes zero after adjustment for all reported contrasts. */
|
|
4254
|
-
liftCi: {
|
|
4255
|
-
low: number;
|
|
4256
|
-
high: number;
|
|
4257
|
-
};
|
|
4258
|
-
/** Optimization spend reported by the method. Excludes final test scoring. */
|
|
4259
|
-
optimizationCost: ComparisonCost;
|
|
4260
|
-
/** Optimization duration reported by the method. Excludes final test scoring. */
|
|
4261
|
-
durationMs?: number;
|
|
4262
|
-
/** Paired final-test values used to compute lift and its interval. */
|
|
4263
|
-
scenarioScores: Array<{
|
|
4264
|
-
scenarioId: string;
|
|
4265
|
-
baselineComposite: number;
|
|
4266
|
-
winnerComposite: number;
|
|
4267
|
-
lift: number;
|
|
4268
|
-
}>;
|
|
4269
|
-
winnerSurface: MutableSurface;
|
|
4270
|
-
/** 1-based, by descending lift. */
|
|
4271
|
-
rank: number;
|
|
4272
|
-
}
|
|
4273
|
-
interface OptimizationMethodPairwise {
|
|
4274
|
-
/** Higher-ranked method. */
|
|
4275
|
-
a: string;
|
|
4276
|
-
b: string;
|
|
4277
|
-
/** Mean per-scenario untouched-test delta (a − b). */
|
|
4278
|
-
deltaMean: number;
|
|
4279
|
-
low: number;
|
|
4280
|
-
high: number;
|
|
4281
|
-
/** `a` if the CI clears 0, `b` if it is entirely negative, else `'tie'`. */
|
|
4282
|
-
favored: string;
|
|
4283
|
-
}
|
|
4284
|
-
interface OptimizationMethodComparison {
|
|
4285
|
-
/** Sorted by descending lift; `rank` set accordingly. */
|
|
4286
|
-
scores: OptimizationMethodScore[];
|
|
4287
|
-
best: OptimizationMethodScore;
|
|
4288
|
-
/** Best vs each other method, using simultaneous paired-bootstrap intervals. */
|
|
4289
|
-
pairwise: OptimizationMethodPairwise[];
|
|
4290
|
-
testScenarioIds: string[];
|
|
4291
|
-
/** Sum of the costs reported by every optimization method. */
|
|
4292
|
-
optimizationCost: ComparisonCost;
|
|
4293
|
-
/** Baseline and distinct winner scoring on the final test partition. */
|
|
4294
|
-
testCost: ComparisonCost;
|
|
4295
|
-
/** Optimization plus final test scoring. */
|
|
4296
|
-
totalCost: ComparisonCost;
|
|
4297
|
-
/** Caller-requested simultaneous coverage across all reported contrasts. */
|
|
4298
|
-
confidence: number;
|
|
4299
|
-
/** Bonferroni-adjusted confidence used for each bootstrap interval. */
|
|
4300
|
-
intervalConfidence: number;
|
|
4301
|
-
/** Method-vs-baseline plus all possible method-vs-method contrasts. */
|
|
4302
|
-
comparisonCount: number;
|
|
4303
|
-
/** Deterministic bootstrap and campaign seed. */
|
|
4304
|
-
seed: number;
|
|
4305
|
-
/** Bootstrap draws used for each interval. */
|
|
4306
|
-
resamples: number;
|
|
4307
|
-
/** Agent runs averaged within each test scenario before resampling scenarios. */
|
|
4308
|
-
reps: number;
|
|
4309
|
-
}
|
|
4310
|
-
interface CompareOptimizationMethodsOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch' | 'judges' | 'scenarios'> {
|
|
4311
|
-
methods: OptimizationMethod<TScenario, TArtifact>[];
|
|
4312
|
-
baselineSurface: MutableSurface;
|
|
4313
|
-
/** Evidence used by every optimizer to author or fit candidates. */
|
|
4314
|
-
trainScenarios: TScenario[];
|
|
4315
|
-
/** Candidate acceptance, early-stopping, and optimizer-selection data. */
|
|
4316
|
-
selectionScenarios: TScenario[];
|
|
4317
|
-
/** Untouched final comparison data. Never passed to an optimization method. */
|
|
4318
|
-
testScenarios: TScenario[];
|
|
4319
|
-
/** Scores a surface on a scenario. The methods and final test share this function. */
|
|
4320
|
-
dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
4321
|
-
judges: JudgeConfig<TArtifact, TScenario>[];
|
|
4322
|
-
/** Bootstrap resamples for the lift intervals. Default is at least 2000 and
|
|
4323
|
-
* rises when the requested simultaneous confidence needs finer tails. */
|
|
4324
|
-
resamples?: number;
|
|
4325
|
-
/** Shared defaults for each method's train and selection campaigns. */
|
|
4326
|
-
optimizationRunOptions?: OptimizationMethodRunOptions<TScenario, TArtifact>;
|
|
4327
|
-
/** Number of optimization methods to run concurrently. Default 1. */
|
|
4328
|
-
optimizationConcurrency?: number;
|
|
4329
|
-
/** Simultaneous confidence across method-vs-baseline and method-vs-method contrasts.
|
|
4330
|
-
* Each bootstrap interval is Bonferroni-adjusted. Default 0.95. */
|
|
4331
|
-
confidence?: number;
|
|
4332
|
-
/** Shared spend limit across baseline and winner scoring on the final test partition.
|
|
4333
|
-
* Each method owns its optimization budget through `optimizationRunOptions.costCeiling`. */
|
|
4334
|
-
costCeiling?: number;
|
|
4335
|
-
}
|
|
4336
|
-
/**
|
|
4337
|
-
* Compare complete optimization methods on disjoint train, selection, and final test data.
|
|
4338
|
-
*/
|
|
4339
|
-
declare function compareOptimizationMethods<TScenario extends Scenario, TArtifact>(opts: CompareOptimizationMethodsOptions<TScenario, TArtifact>): Promise<OptimizationMethodComparison>;
|
|
4340
|
-
/** Keep the cost fields a custom optimization method must report. */
|
|
4341
|
-
declare function costFromLedgerSummary(summary: CostLedgerSummary): ComparisonCost;
|
|
4342
|
-
|
|
4343
4410
|
/**
|
|
4344
4411
|
* `runOptimization` — the improvement loop body. Runs N generations: the
|
|
4345
4412
|
* `SurfaceProposer` proposes K candidate surfaces per generation, each
|
|
@@ -7685,4 +7752,4 @@ declare function verifyCodeSurface(surface: CodeSurface, worktreeDir?: string):
|
|
|
7685
7752
|
* identity against the checkout at `worktreeRef`. */
|
|
7686
7753
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
7687
7754
|
|
|
7688
|
-
export { type AcceptedEdit, type AceProposerOptions, type AnalystArtifact, type AnalystScenario, type AnalyzeCrossSurfaceInteractionsInput, type AnalyzeOtlpTraceFileOptions, type ApplySkillPatchResult, type AxisEvidence, type AxisVerdict, type BuildAnalystSurfaceDispatchOptions, type BuildEvidenceVectorOptions, type BuildLoopProvenanceArgs, type BuiltinOptimizationMethodConfig, type CampaignAggregates, type CampaignArtifactWriter, type CampaignBreakdown, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignRunPlan, type CampaignRunPlanCell, type CampaignScenarioIdentity, type CampaignStorage, type CampaignTokenUsage, type CampaignTraceWriter, type CodeSurface, type CodeSurfaceVerification, type CompareOptimizationMethodsOptions, type ComparisonCost, type CompositeProposerOptions, type CostLedgerHandle, type CrossSurfaceAdditionDecision, type CrossSurfaceAdditionRejectionReason, type CrossSurfaceAttemptCompleteness, type CrossSurfaceBestSingleSelection, type CrossSurfaceBootstrapPolicy, type CrossSurfaceCandidate, type CrossSurfaceCandidateComparison, type CrossSurfaceCandidateEvidence, type CrossSurfaceCandidateOutcome, type CrossSurfaceCandidateSummary, type CrossSurfaceComponent, type CrossSurfaceComponentEvidence, type CrossSurfaceCompositionStep, type CrossSurfaceDistribution, type CrossSurfaceEligibility, type CrossSurfaceEvidenceBreakdown, type CrossSurfaceIneligibilityReason, type CrossSurfaceInteractionAwareSelection, type CrossSurfaceInteractionEffect, type CrossSurfaceInteractionPath, type CrossSurfaceInteractionReport, type CrossSurfaceInteractionTask, type CrossSurfaceNaiveStackSelection, type CrossSurfacePairCompatibility, type CrossSurfacePairEvidence, type CrossSurfacePairIncompatibilityReason, type CrossSurfacePairwiseEntry, type CrossSurfaceRankedSingle, type CrossSurfaceRelativeCost, type CrossSurfaceSelectionPolicy, type CrossSurfaceSelections, type CrossSurfaceTaskRow, DEFAULT_POLICY_EDIT_HISTORY_LIMITS, type DefaultProductionGateOptions, type DimensionRegression, type DiscriminationScore, type DispatchContext, type DispatchFn, type EmitLoopProvenanceArgs, type EmitLoopProvenanceResult, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type EvidenceVector, type EvolutionaryProposerOptions, type FailureModeRecallJudgeOptions, type FapoAttributionSignals, type FapoFailureCluster, type FapoOptimizationLevel, type FapoOptimizationMethodConfig, type FapoProposerOptions, type FapoReviewInput, type FapoReviewIssue, type FapoReviewResult, type FapoScopeContract, FileSearchLedger, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerConstraints, type GepaProposerOptions, type GitWorktreeAdapterOptions, type Governor, type GovernorContext, type GovernorOp, type HaloProposerOptions, type HeldOutGateOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type HeuristicGovernorOptions, type JsonPolicyEditTargetSurface, type JsonPrimitive, type JsonValue, type JudgeAggregate, type JudgeConfig, type JudgeDimension, type JudgeScore, type LabelTrust, type LabeledScenarioRecord, type LabeledScenarioSampleArgs, type LabeledScenarioSource, type LabeledScenarioStore, LabeledScenarioStoreError, type LabeledScenarioWrite, Lineage, type LineageEdge, type LineageGraph, type LineageNode, type LineageNodeInput, type LineageStore, LineageStoreConflictError, type LlmJudgeDimension, type LlmJudgeOptions, type LlmPolicyEditProposerOptions, type LoadEvalFixtureScenariosOptions, type LoopProvenanceArgsFromResult, type LoopProvenanceBackend, type LoopProvenanceCandidate, type LoopProvenanceEvidence, type LoopProvenanceRecord, type MemoryCurationProposerOptions, type MutableSurface, type Mutator, type NeutralizationGateOptions, type ObjectiveSource, type OpenAutoPrOptions, type OpenAutoPrResult, type OpenSearchLedgerOptions, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodPairwise, type OptimizationMethodResult, type OptimizationMethodRunOptions, type OptimizationMethodScore, type OptimizationProposer, type OptimizerConfig, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, type PairedHoldout, type ParameterCandidate, type ParameterChange, type ParameterSweepProposerOptions, type ParetoParent, type ParetoSignificanceGateOptions, type PendingCostCallView, type PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PolicyEditAuthorScenarioOrder, type PolicyEditAuthorScenarioRow, type PolicyEditCandidateRecord, type PolicyEditCandidateSummary, type PolicyEditFindingInput, type PolicyEditFindingSource, type PolicyEditHistoryCandidateContext, type PolicyEditHistoryGenerationContext, type PolicyEditHistoryProjectionOptions, type PolicyEditObjective, type PolicyEditOutcomeContext, type PolicyEditProposerOptions, type PowerPreflight, type PowerPreflightOptions, type PremeasuredOptimizationBaseline, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, type PromotionObjective, type PromotionPolicy, type ProposalTrackContext, type ProposeContext, type ProposePatchesArgs, type ProposedCandidate, type RedactionStatus, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceScenario, type RejectedEdit, type RolloutArgumentDiff, type RolloutArgumentDiffOptions, type RolloutCall, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunLineageLoopOptions, type RunLineageLoopResult, type RunLineageLoopSeed, type RunLineageOptions, type RunLineageResult, type RunLineageSeed, type RunLineageStepResult, type RunOptimizationOptions, type RunOptimizationResult, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, SEARCH_LEDGER_SCHEMA, type Scenario, type ScenarioAggregate, type ScenarioRollup, type ScenarioSignal, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type ScoredRollout, type ScoredSurfaceOutcome, type SearchAccountingAudit, type SearchArtifactRef, type SearchAttemptAccounting, type SearchCandidateDecidedEvent, type SearchCandidateLineage, type SearchCandidateRegisteredEvent, type SearchCandidateSlot, type SearchCandidateSlotClosedEvent, type SearchCandidateSurface, type SearchCompletedEvent, type SearchCostAccounting, type SearchFailureReason, type SearchLedger, type SearchLedgerAppendResult, SearchLedgerConflictError, type SearchLedgerEntry, SearchLedgerError, type SearchLedgerEvent, type SearchLedgerHash, SearchLedgerIntegrityError, type SearchLedgerReplay, type SearchModelIdentity, type SearchOperationKind, type SearchOperationRecordedEvent, type SearchPlan, type SearchPlannedEvent, type SearchPlannedOperation, type SearchPlannedTask, type SearchSourceRef, type SearchSurfaceEffect, type SearchSurfaceEvidence, type SearchSurfaceKind, type SearchTaskAttemptedEvent, type SearchTaskOutcome, type SearchTokenAccounting, type SelectPolicyEditAuthorRowsOptions, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SerializedJsonBudget, type SessionScript, type SingleRunLock, type SingleRunLockOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillOptProposer, type SkillOptProposerOptions, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, type SurfaceProposer, type SurfaceScore, type TraceAnalystPriorFindings, type TraceAnalystProposerOptions, type TraceSpan, type TransientFailureOptions, type UngroundedLiteralReport, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceProposer, acquireSingleRunLock, analyzeCrossSurfaceInteractions, analyzeOtlpTraceFile, applySkillPatch, assertCampaignDesign, assertCampaignSplitIdentity, assertCodeSurfaceIdentity, assertPolicyEditAuthorContextBudget, buildAnalystSurfaceDispatch, buildEvidenceVector, buildLoopProvenanceRecord, callbackGovernor, campaignBreakdown, campaignLineageStore, campaignMeanComposite, campaignMeasurementDigest, campaignScenarioIdentity, campaignSplitDigest, campaignSplitDigestFromIdentities, canonicalDigest, classifyUngroundedLiterals, codeSurfaceIdentityMaterial, compareOptimizationMethods, composeGate, compositeProposer, costFromLedgerSummary, countSentenceEdits, createReferenceEquivalenceJudge, createRunCostLedger, defaultProductionGate, detectScale, dimensionRegressions, discoverEvalFixtures, emitLoopProvenance, evolutionaryProposer, extractFapoAttributionSignals, extractH2Sections, failureModeRecallJudge, fapoEscalationMethod, fapoProposer, fsCampaignStorage, fsLineageStore, gepaParetoMethod, gepaProposer, gepaReflectionMethod, gitWorktreeAdapter, haloProposer, heldOutGate, heldoutSignificance, heuristicGovernor, inMemoryCampaignStorage, isProposedCandidate, isTransientTransportFailure, labelTrustRank, lineageNodeId, llmJudge, llmPolicyEditProposer, loadEvalFixture, loadEvalFixtureScenarios, loopProvenanceArgsFromResult, loopProvenanceSpans, makePlaybackDispatch, memLineageStore, memoryCurationProposer, neutralizationGate, neutralizeText, openAutoPr, openSearchLedger, pairHoldout, parameterSweepProposer, paretoPolicy, paretoSignificanceGate, parseSkillPatchResponse, patchEditCount, planCampaignRun, planEvalFixtureRun, policyEditProposer, powerPreflight, projectPolicyEditHistory, provenanceRecordPath, provenanceSpansPath, renderScoreboardMarkdown, renderSurfaceDiff, resolveRunDir, resolveWorktreePath, rolloutArgumentDiff, runCampaign, runEval, runImprovementLoop, runLineage, runLineageLoop, runOptimization, runProfileMatrix, runSkillOpt, scoreDiscrimination, scoreUserStory, scoreboardSummary, selectDiscriminative, selectPolicyEditAuthorRows, sequentialDecide, sequentialPairedGate, skillOptMethod, skillOptProposer, surfaceContentHash, surfaceHash, tangleTracesRoot, traceAnalystProposer, userStoryScoreboard, validatePolicyEditCandidateRecord, validateSearchLedgerEvent, verifyCodeSurface, verifyLoopProvenanceRecord };
|
|
7755
|
+
export { type AcceptedEdit, type AceProposerOptions, type AnalystArtifact, type AnalystScenario, type AnalyzeCrossSurfaceInteractionsInput, type AnalyzeOtlpTraceFileOptions, type ApplySkillPatchResult, type AxisEvidence, type AxisVerdict, type BuildAnalystSurfaceDispatchOptions, type BuildEvidenceVectorOptions, type BuildLoopProvenanceArgs, type BuiltinOptimizationMethodConfig, type CampaignAggregates, type CampaignArtifactWriter, type CampaignBreakdown, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignRunPlan, type CampaignRunPlanCell, type CampaignScenarioIdentity, type CampaignStorage, type CampaignTokenUsage, type CampaignTraceWriter, type CodeSurface, type CodeSurfaceVerification, type CompareOptimizationMethodsOptions, type ComparisonCost, type CompositeProposerOptions, type CostLedgerHandle, type CrossSurfaceAdditionDecision, type CrossSurfaceAdditionRejectionReason, type CrossSurfaceAttemptCompleteness, type CrossSurfaceBestSingleSelection, type CrossSurfaceBootstrapPolicy, type CrossSurfaceCandidate, type CrossSurfaceCandidateComparison, type CrossSurfaceCandidateEvidence, type CrossSurfaceCandidateOutcome, type CrossSurfaceCandidateSummary, type CrossSurfaceComponent, type CrossSurfaceComponentEvidence, type CrossSurfaceCompositionStep, type CrossSurfaceDistribution, type CrossSurfaceEligibility, type CrossSurfaceEvidenceBreakdown, type CrossSurfaceIneligibilityReason, type CrossSurfaceInteractionAwareSelection, type CrossSurfaceInteractionEffect, type CrossSurfaceInteractionPath, type CrossSurfaceInteractionReport, type CrossSurfaceInteractionTask, type CrossSurfaceNaiveStackSelection, type CrossSurfacePairCompatibility, type CrossSurfacePairEvidence, type CrossSurfacePairIncompatibilityReason, type CrossSurfacePairwiseEntry, type CrossSurfaceRankedSingle, type CrossSurfaceRelativeCost, type CrossSurfaceSelectionPolicy, type CrossSurfaceSelections, type CrossSurfaceTaskRow, DEFAULT_POLICY_EDIT_HISTORY_LIMITS, type DefaultProductionGateOptions, type DimensionRegression, type DiscriminationScore, type DispatchContext, type DispatchFn, type EmitLoopProvenanceArgs, type EmitLoopProvenanceResult, type EvalFixture, type EvalFixtureFile, type EvalFixtureLoadOptions, type EvalFixtureRunPlan, type EvalFixtureScenario, type EvalFixtureValidationMode, type EvidenceVector, type EvolutionaryProposerOptions, type FailureModeRecallJudgeOptions, type FapoAttributionSignals, type FapoFailureCluster, type FapoOptimizationLevel, type FapoOptimizationMethodConfig, type FapoProposerOptions, type FapoReviewInput, type FapoReviewIssue, type FapoReviewResult, type FapoScopeContract, FileSearchLedger, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaEngineRun, type GepaOptimizationMethodConfig, type GepaOptimizationRecipe, type GepaProposerConstraints, type GepaProposerOptions, type GepaRunnerCommand, type GitWorktreeAdapterOptions, type Governor, type GovernorContext, type GovernorOp, type HaloProposerOptions, type HeldOutGateOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, type HeuristicGovernorOptions, type JsonPolicyEditTargetSurface, type JsonPrimitive, type JsonValue, type JudgeAggregate, type JudgeConfig, type JudgeDimension, type JudgeScore, type LabelTrust, type LabeledScenarioRecord, type LabeledScenarioSampleArgs, type LabeledScenarioSource, type LabeledScenarioStore, LabeledScenarioStoreError, type LabeledScenarioWrite, Lineage, type LineageEdge, type LineageGraph, type LineageNode, type LineageNodeInput, type LineageStore, LineageStoreConflictError, type LlmJudgeDimension, type LlmJudgeOptions, type LlmPolicyEditProposerOptions, type LoadEvalFixtureScenariosOptions, type LoopProvenanceArgsFromResult, type LoopProvenanceBackend, type LoopProvenanceCandidate, type LoopProvenanceEvidence, type LoopProvenanceRecord, type MemoryCurationProposerOptions, type MutableSurface, type Mutator, type NeutralizationGateOptions, type ObjectiveSource, type OpenAutoPrOptions, type OpenAutoPrResult, type OpenSearchLedgerOptions, type OptimizationMethod, type OptimizationMethodComparison, type OptimizationMethodInput, type OptimizationMethodPairwise, type OptimizationMethodResult, type OptimizationMethodRunOptions, type OptimizationMethodScore, type OptimizationProposer, type OptimizerConfig, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, type PairedHoldout, type ParameterCandidate, type ParameterChange, type ParameterSweepProposerOptions, type ParetoParent, type ParetoSignificanceGateOptions, type PendingCostCallView, type PlanCampaignRunOptions, type PlanEvalFixtureRunOptions, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type PolicyEditAuthorScenarioOrder, type PolicyEditAuthorScenarioRow, type PolicyEditCandidateRecord, type PolicyEditCandidateSummary, type PolicyEditFindingInput, type PolicyEditFindingSource, type PolicyEditHistoryCandidateContext, type PolicyEditHistoryGenerationContext, type PolicyEditHistoryProjectionOptions, type PolicyEditObjective, type PolicyEditOutcomeContext, type PolicyEditProposerOptions, type PowerPreflight, type PowerPreflightOptions, type PremeasuredOptimizationBaseline, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, type PromotionObjective, type PromotionPolicy, type ProposalTrackContext, type ProposeContext, type ProposePatchesArgs, type ProposedCandidate, type RedactionStatus, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceScenario, type RejectedEdit, type RolloutArgumentDiff, type RolloutArgumentDiffOptions, type RolloutCall, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunLineageLoopOptions, type RunLineageLoopResult, type RunLineageLoopSeed, type RunLineageOptions, type RunLineageResult, type RunLineageSeed, type RunLineageStepResult, type RunOptimizationOptions, type RunOptimizationResult, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, SEARCH_LEDGER_SCHEMA, type Scenario, type ScenarioAggregate, type ScenarioRollup, type ScenarioSignal, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type ScoredRollout, type ScoredSurfaceOutcome, type SearchAccountingAudit, type SearchArtifactRef, type SearchAttemptAccounting, type SearchCandidateDecidedEvent, type SearchCandidateLineage, type SearchCandidateRegisteredEvent, type SearchCandidateSlot, type SearchCandidateSlotClosedEvent, type SearchCandidateSurface, type SearchCompletedEvent, type SearchCostAccounting, type SearchFailureReason, type SearchLedger, type SearchLedgerAppendResult, SearchLedgerConflictError, type SearchLedgerEntry, SearchLedgerError, type SearchLedgerEvent, type SearchLedgerHash, SearchLedgerIntegrityError, type SearchLedgerReplay, type SearchModelIdentity, type SearchOperationKind, type SearchOperationRecordedEvent, type SearchPlan, type SearchPlannedEvent, type SearchPlannedOperation, type SearchPlannedTask, type SearchSourceRef, type SearchSurfaceEffect, type SearchSurfaceEvidence, type SearchSurfaceKind, type SearchTaskAttemptedEvent, type SearchTaskOutcome, type SearchTokenAccounting, type SelectPolicyEditAuthorRowsOptions, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SerializedJsonBudget, type SessionScript, type SingleRunLock, type SingleRunLockOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillOptProposer, type SkillOptProposerOptions, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, type SurfaceProposer, type SurfaceScore, type TraceAnalystPriorFindings, type TraceAnalystProposerOptions, type TraceSpan, type TransientFailureOptions, type UngroundedLiteralReport, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceProposer, acquireSingleRunLock, analyzeCrossSurfaceInteractions, analyzeOtlpTraceFile, applySkillPatch, assertCampaignDesign, assertCampaignSplitIdentity, assertCodeSurfaceIdentity, assertPolicyEditAuthorContextBudget, buildAnalystSurfaceDispatch, buildEvidenceVector, buildLoopProvenanceRecord, callbackGovernor, campaignBreakdown, campaignLineageStore, campaignMeanComposite, campaignMeasurementDigest, campaignScenarioIdentity, campaignSplitDigest, campaignSplitDigestFromIdentities, canonicalDigest, classifyUngroundedLiterals, codeSurfaceIdentityMaterial, compareOptimizationMethods, composeGate, compositeProposer, costFromLedgerSummary, countSentenceEdits, createReferenceEquivalenceJudge, createRunCostLedger, defaultProductionGate, detectScale, dimensionRegressions, discoverEvalFixtures, emitLoopProvenance, evolutionaryProposer, extractFapoAttributionSignals, extractH2Sections, failureModeRecallJudge, fapoEscalationMethod, fapoProposer, fsCampaignStorage, fsLineageStore, gepaOptimizationMethod, gepaParetoMethod, gepaProposer, gepaReflectionMethod, gitWorktreeAdapter, haloProposer, heldOutGate, heldoutSignificance, heuristicGovernor, inMemoryCampaignStorage, isProposedCandidate, isTransientTransportFailure, labelTrustRank, lineageNodeId, llmJudge, llmPolicyEditProposer, loadEvalFixture, loadEvalFixtureScenarios, loopProvenanceArgsFromResult, loopProvenanceSpans, makePlaybackDispatch, memLineageStore, memoryCurationProposer, neutralizationGate, neutralizeText, openAutoPr, openSearchLedger, pairHoldout, parameterSweepProposer, paretoPolicy, paretoSignificanceGate, parseSkillPatchResponse, patchEditCount, planCampaignRun, planEvalFixtureRun, policyEditProposer, powerPreflight, projectPolicyEditHistory, provenanceRecordPath, provenanceSpansPath, renderScoreboardMarkdown, renderSurfaceDiff, resolveRunDir, resolveWorktreePath, rolloutArgumentDiff, runCampaign, runEval, runImprovementLoop, runLineage, runLineageLoop, runOptimization, runProfileMatrix, runSkillOpt, scoreDiscrimination, scoreUserStory, scoreboardSummary, selectDiscriminative, selectPolicyEditAuthorRows, sequentialDecide, sequentialPairedGate, skillOptMethod, skillOptProposer, surfaceContentHash, surfaceHash, tangleTracesRoot, traceAnalystProposer, userStoryScoreboard, validatePolicyEditCandidateRecord, validateSearchLedgerEvent, verifyCodeSurface, verifyLoopProvenanceRecord };
|
package/dist/campaign/index.js
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
fapoEscalationMethod,
|
|
29
29
|
fapoProposer,
|
|
30
30
|
fsLineageStore,
|
|
31
|
+
gepaOptimizationMethod,
|
|
31
32
|
gepaParetoMethod,
|
|
32
33
|
gepaReflectionMethod,
|
|
33
34
|
gitWorktreeAdapter,
|
|
@@ -70,7 +71,7 @@ import {
|
|
|
70
71
|
userStoryScoreboard,
|
|
71
72
|
validateSearchLedgerEvent,
|
|
72
73
|
verifyCodeSurface
|
|
73
|
-
} from "../chunk-
|
|
74
|
+
} from "../chunk-VPDOSN3L.js";
|
|
74
75
|
import {
|
|
75
76
|
assertCodeSurfaceIdentity,
|
|
76
77
|
buildEvidenceVector,
|
|
@@ -111,7 +112,7 @@ import {
|
|
|
111
112
|
surfaceContentHash,
|
|
112
113
|
surfaceHash,
|
|
113
114
|
verifyLoopProvenanceRecord
|
|
114
|
-
} from "../chunk-
|
|
115
|
+
} from "../chunk-KKPPFIDS.js";
|
|
115
116
|
import {
|
|
116
117
|
SearchLedgerConflictError,
|
|
117
118
|
SearchLedgerError,
|
|
@@ -207,6 +208,7 @@ export {
|
|
|
207
208
|
fapoProposer,
|
|
208
209
|
fsCampaignStorage,
|
|
209
210
|
fsLineageStore,
|
|
211
|
+
gepaOptimizationMethod,
|
|
210
212
|
gepaParetoMethod,
|
|
211
213
|
gepaProposer,
|
|
212
214
|
gepaReflectionMethod,
|
|
@@ -8,6 +8,51 @@ import {
|
|
|
8
8
|
toolSpans
|
|
9
9
|
} from "./chunk-ZET2UAYW.js";
|
|
10
10
|
|
|
11
|
+
// src/trajectory.ts
|
|
12
|
+
async function buildTrajectory(store, runId) {
|
|
13
|
+
const spans = await store.spans({ runId });
|
|
14
|
+
const events = await store.events({ runId });
|
|
15
|
+
const childrenOf = /* @__PURE__ */ new Map();
|
|
16
|
+
for (const s of spans) {
|
|
17
|
+
const arr = childrenOf.get(s.parentSpanId) ?? [];
|
|
18
|
+
arr.push(s);
|
|
19
|
+
childrenOf.set(s.parentSpanId, arr);
|
|
20
|
+
}
|
|
21
|
+
for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt);
|
|
22
|
+
const eventsBySpan = /* @__PURE__ */ new Map();
|
|
23
|
+
for (const e of events) {
|
|
24
|
+
if (!e.spanId) continue;
|
|
25
|
+
const arr = eventsBySpan.get(e.spanId) ?? [];
|
|
26
|
+
arr.push(e);
|
|
27
|
+
eventsBySpan.set(e.spanId, arr);
|
|
28
|
+
}
|
|
29
|
+
const steps = [];
|
|
30
|
+
const walk = (spanId, depth) => {
|
|
31
|
+
const kids = childrenOf.get(spanId) ?? [];
|
|
32
|
+
for (const child of kids) {
|
|
33
|
+
steps.push({
|
|
34
|
+
index: steps.length,
|
|
35
|
+
span: child,
|
|
36
|
+
depth,
|
|
37
|
+
events: eventsBySpan.get(child.spanId) ?? []
|
|
38
|
+
});
|
|
39
|
+
walk(child.spanId, depth + 1);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
walk(void 0, 0);
|
|
43
|
+
const llmTurns = steps.filter((s) => s.span.kind === "llm").length;
|
|
44
|
+
const toolCalls = steps.filter((s) => s.span.kind === "tool").length;
|
|
45
|
+
const judgeVerdicts = steps.filter((s) => s.span.kind === "judge").length;
|
|
46
|
+
const retrievals = steps.filter((s) => s.span.kind === "retrieval").length;
|
|
47
|
+
let totalDurationMs = 0;
|
|
48
|
+
if (steps.length > 0) {
|
|
49
|
+
const starts = spans.map((s) => s.startedAt);
|
|
50
|
+
const ends = spans.map((s) => s.endedAt ?? s.startedAt);
|
|
51
|
+
totalDurationMs = Math.max(...ends) - Math.min(...starts);
|
|
52
|
+
}
|
|
53
|
+
return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs };
|
|
54
|
+
}
|
|
55
|
+
|
|
11
56
|
// src/failure-taxonomy.ts
|
|
12
57
|
var DEFAULT_RULES = [
|
|
13
58
|
// Outcome already named? Respect it.
|
|
@@ -415,51 +460,6 @@ async function computeToolUseMetrics(store, runId, options = {}) {
|
|
|
415
460
|
};
|
|
416
461
|
}
|
|
417
462
|
|
|
418
|
-
// src/trajectory.ts
|
|
419
|
-
async function buildTrajectory(store, runId) {
|
|
420
|
-
const spans = await store.spans({ runId });
|
|
421
|
-
const events = await store.events({ runId });
|
|
422
|
-
const childrenOf = /* @__PURE__ */ new Map();
|
|
423
|
-
for (const s of spans) {
|
|
424
|
-
const arr = childrenOf.get(s.parentSpanId) ?? [];
|
|
425
|
-
arr.push(s);
|
|
426
|
-
childrenOf.set(s.parentSpanId, arr);
|
|
427
|
-
}
|
|
428
|
-
for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt);
|
|
429
|
-
const eventsBySpan = /* @__PURE__ */ new Map();
|
|
430
|
-
for (const e of events) {
|
|
431
|
-
if (!e.spanId) continue;
|
|
432
|
-
const arr = eventsBySpan.get(e.spanId) ?? [];
|
|
433
|
-
arr.push(e);
|
|
434
|
-
eventsBySpan.set(e.spanId, arr);
|
|
435
|
-
}
|
|
436
|
-
const steps = [];
|
|
437
|
-
const walk = (spanId, depth) => {
|
|
438
|
-
const kids = childrenOf.get(spanId) ?? [];
|
|
439
|
-
for (const child of kids) {
|
|
440
|
-
steps.push({
|
|
441
|
-
index: steps.length,
|
|
442
|
-
span: child,
|
|
443
|
-
depth,
|
|
444
|
-
events: eventsBySpan.get(child.spanId) ?? []
|
|
445
|
-
});
|
|
446
|
-
walk(child.spanId, depth + 1);
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
|
-
walk(void 0, 0);
|
|
450
|
-
const llmTurns = steps.filter((s) => s.span.kind === "llm").length;
|
|
451
|
-
const toolCalls = steps.filter((s) => s.span.kind === "tool").length;
|
|
452
|
-
const judgeVerdicts = steps.filter((s) => s.span.kind === "judge").length;
|
|
453
|
-
const retrievals = steps.filter((s) => s.span.kind === "retrieval").length;
|
|
454
|
-
let totalDurationMs = 0;
|
|
455
|
-
if (steps.length > 0) {
|
|
456
|
-
const starts = spans.map((s) => s.startedAt);
|
|
457
|
-
const ends = spans.map((s) => s.endedAt ?? s.startedAt);
|
|
458
|
-
totalDurationMs = Math.max(...ends) - Math.min(...starts);
|
|
459
|
-
}
|
|
460
|
-
return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs };
|
|
461
|
-
}
|
|
462
|
-
|
|
463
463
|
// src/baseline.ts
|
|
464
464
|
function compareToBaseline(samples, options = {}) {
|
|
465
465
|
const effectThreshold = options.effectThreshold ?? 0.5;
|
|
@@ -611,12 +611,12 @@ function normalCdf(x) {
|
|
|
611
611
|
}
|
|
612
612
|
|
|
613
613
|
export {
|
|
614
|
+
buildTrajectory,
|
|
614
615
|
DEFAULT_RULES,
|
|
615
616
|
classifyFailure,
|
|
616
617
|
computeToolUseMetrics,
|
|
617
|
-
buildTrajectory,
|
|
618
618
|
compareToBaseline,
|
|
619
619
|
iqr,
|
|
620
620
|
welchsTTest
|
|
621
621
|
};
|
|
622
|
-
//# sourceMappingURL=chunk-
|
|
622
|
+
//# sourceMappingURL=chunk-J3LHTAAB.js.map
|