@tangle-network/agent-eval 0.133.3 → 0.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/dist/analyst/index.d.ts +11 -35
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +4 -53
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-BClW9OSe.d.ts → analyze-runs-DMo3Lb_y.d.ts} +4 -4
- package/dist/{analyze-runs-BClW9OSe.d.ts.map → analyze-runs-DMo3Lb_y.d.ts.map} +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-BP9sgMia.js → benchmarks-v5piCeDl.js} +3 -3
- package/dist/{benchmarks-BP9sgMia.js.map → benchmarks-v5piCeDl.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -4
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign--V4ffEKR.js → campaign-DEC_7DLn.js} +2 -2
- package/dist/{campaign--V4ffEKR.js.map → campaign-DEC_7DLn.js.map} +1 -1
- package/dist/{client-Du7B81wW.d.ts → client-BIyh1RCr.d.ts} +3 -3
- package/dist/{client-Du7B81wW.d.ts.map → client-BIyh1RCr.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +12 -11
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +4 -11
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-Cl3pHo4n.d.ts → default-registry-Brxr728w.d.ts} +4 -268
- package/dist/default-registry-Brxr728w.d.ts.map +1 -0
- package/dist/{default-registry-D3T9XbuY.js → default-registry-IjYs7T8l.js} +4 -61
- package/dist/default-registry-IjYs7T8l.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-DOqvIJ8I.d.ts → index-BoJNQR6n.d.ts} +4 -3
- package/dist/index-BoJNQR6n.d.ts.map +1 -0
- package/dist/{index-B5MNN1f1.d.ts → index-C21xKtxu.d.ts} +4 -4
- package/dist/{index-B5MNN1f1.d.ts.map → index-C21xKtxu.d.ts.map} +1 -1
- package/dist/index.d.ts +12 -11
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +109 -11
- package/dist/index.js.map +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/proposal-findings-DCawte-y.js +164 -0
- package/dist/proposal-findings-DCawte-y.js.map +1 -0
- package/dist/{release-report-DKBtegGt.d.ts → release-report-CuULWKyk.d.ts} +2 -2
- package/dist/{release-report-DKBtegGt.d.ts.map → release-report-CuULWKyk.d.ts.map} +1 -1
- package/dist/reporting.d.ts +2 -2
- package/dist/{researcher-BtD5U1Up.d.ts → researcher-DVtruQ9U.d.ts} +2 -2
- package/dist/{researcher-BtD5U1Up.d.ts.map → researcher-DVtruQ9U.d.ts.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +14 -1
- package/dist/rl.js.map +1 -1
- package/dist/{semantic-concept-judge-BypLt6Fw.js → semantic-concept-judge-C0P1VTXD.js} +2 -3
- package/dist/{semantic-concept-judge-BypLt6Fw.js.map → semantic-concept-judge-C0P1VTXD.js.map} +1 -1
- package/dist/{skill-usage-BaaxFSJR.d.ts → skill-usage-BDQVPIG1.d.ts} +3 -2
- package/dist/skill-usage-BDQVPIG1.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-vvJ4bMNI.js → skillopt-optimization-method-BY6vKLJB.js} +51 -30
- package/dist/skillopt-optimization-method-BY6vKLJB.js.map +1 -0
- package/dist/{skillopt-optimization-method-Dxr8pdZd.d.ts → skillopt-optimization-method-DJ3l4w8W.d.ts} +10 -13
- package/dist/skillopt-optimization-method-DJ3l4w8W.d.ts.map +1 -0
- package/dist/{summary-report-DyOhItws.d.ts → summary-report-DGp0-_XO.d.ts} +59 -3
- package/dist/summary-report-DGp0-_XO.d.ts.map +1 -0
- package/dist/types-DVjczBM9.d.ts +276 -0
- package/dist/types-DVjczBM9.d.ts.map +1 -0
- package/dist/{types-BokuXvOG.d.ts → types-DiWLru6Z.d.ts} +20 -37
- package/dist/types-DiWLru6Z.d.ts.map +1 -0
- package/docs/campaign-proposers.md +5 -0
- package/package.json +1 -1
- package/dist/default-registry-Cl3pHo4n.d.ts.map +0 -1
- package/dist/default-registry-D3T9XbuY.js.map +0 -1
- package/dist/index-DOqvIJ8I.d.ts.map +0 -1
- package/dist/run-score-iEEAWiBY.js +0 -41
- package/dist/run-score-iEEAWiBY.js.map +0 -1
- package/dist/skill-usage-BaaxFSJR.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-Dxr8pdZd.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-vvJ4bMNI.js.map +0 -1
- package/dist/summary-report-DyOhItws.d.ts.map +0 -1
- package/dist/types-BokuXvOG.d.ts.map +0 -1
|
@@ -3,11 +3,12 @@ import { a as RunRecord } from "./run-record-DcObtIGh.js";
|
|
|
3
3
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
4
4
|
import { n as LlmCallMetadata } from "./llm-client-BiK4HW0u.js";
|
|
5
5
|
import { A as ChatClient } from "./types-Cc3qbqzj.js";
|
|
6
|
-
import {
|
|
6
|
+
import { m as ProposalFinding } from "./types-DVjczBM9.js";
|
|
7
|
+
import { C as JudgeDimension, H as SurfaceProposer, N as ParetoParent, R as Scenario, S as JudgeConfig, _ as GateDecision, a as CampaignResult, b as GenerationRecord, c as CampaignTraceWriter, d as DispatchContext, f as DispatchFn, g as GateContribution, h as GateContext, i as CampaignCostMeter, j as MutableSurface, k as LabeledScenarioStore, o as CampaignScenarioIdentity, p as Gate, r as CampaignCellResult, v as GateResult, y as GenerationCandidate } from "./types-DiWLru6Z.js";
|
|
7
8
|
import { _ as PairedBootstrapResult } from "./statistics-D_4Snl-5.js";
|
|
8
9
|
import { a as DatasetScenario, t as Dataset } from "./dataset-BvtnC8Dc.js";
|
|
9
10
|
import { t as DetectRewardHackingInput } from "./reward-hacking-D-QqXvg-.js";
|
|
10
|
-
import { g as TraceSpanEvent, t as HostedClient } from "./client-
|
|
11
|
+
import { g as TraceSpanEvent, t as HostedClient } from "./client-BIyh1RCr.js";
|
|
11
12
|
import { z } from "zod";
|
|
12
13
|
//#region src/llm-judge.d.ts
|
|
13
14
|
/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
|
|
@@ -1315,7 +1316,7 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1315
1316
|
/** Dispatcher that takes the CURRENT surface + scenario → artifact. */
|
|
1316
1317
|
dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
|
|
1317
1318
|
/** The candidate-generation strategy. */
|
|
1318
|
-
proposer: SurfaceProposer
|
|
1319
|
+
proposer: SurfaceProposer<ProposalFinding>;
|
|
1319
1320
|
populationSize: number;
|
|
1320
1321
|
maxGenerations: number;
|
|
1321
1322
|
/** Candidate campaigns run at once. Default 1. Total concurrent cells are
|
|
@@ -1324,14 +1325,8 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1324
1325
|
/** DEPTH knob forwarded to the proposer's `propose()` — max iterations the
|
|
1325
1326
|
* agentic generator may take per candidate. */
|
|
1326
1327
|
maxImprovementShots?: number;
|
|
1327
|
-
/**
|
|
1328
|
-
|
|
1329
|
-
report?: unknown;
|
|
1330
|
-
/** Structured findings forwarded to `propose()` as `ctx.findings`. A
|
|
1331
|
-
* findings producer emits these from the
|
|
1332
|
-
* generation's traces; findings-grounded proposers consume them. Opaque here;
|
|
1333
|
-
* the proposer types its `TFindings`. Empty when no producer is wired. */
|
|
1334
|
-
findings?: unknown[];
|
|
1328
|
+
/** Search or observed-production findings forwarded to candidate generation. */
|
|
1329
|
+
findings?: ReadonlyArray<ProposalFinding>;
|
|
1335
1330
|
/** Per-generation findings producer. Runs once on the BASELINE campaign
|
|
1336
1331
|
* (as `generation: -1`, the baseline convention) before generation 0
|
|
1337
1332
|
* proposes — so even a single-generation run proposes with trace context —
|
|
@@ -1354,7 +1349,7 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1354
1349
|
/** Shared run spend account and receipt attribution phase. */
|
|
1355
1350
|
costLedger?: CostLedgerHandle;
|
|
1356
1351
|
costPhase?: string;
|
|
1357
|
-
}) => Promise<
|
|
1352
|
+
}) => Promise<ReadonlyArray<ProposalFinding>>;
|
|
1358
1353
|
/**
|
|
1359
1354
|
* Optional override for how the WINNER is selected among coverage-complete
|
|
1360
1355
|
* candidates (and how the incumbent bar is set). Returns a lexicographic rank
|
|
@@ -1384,6 +1379,8 @@ interface RunOptimizationResult<TArtifact, TScenario extends Scenario> {
|
|
|
1384
1379
|
campaign: CampaignResult<TArtifact, TScenario>;
|
|
1385
1380
|
}>;
|
|
1386
1381
|
}>;
|
|
1382
|
+
/** Frozen snapshot of the exact starting surface measured by `baselineCampaign`. */
|
|
1383
|
+
baselineSurface: MutableSurface;
|
|
1387
1384
|
winnerSurface: MutableSurface;
|
|
1388
1385
|
winnerSurfaceHash: string;
|
|
1389
1386
|
/** Proposer label for the promoted surface. Present when the winning
|
|
@@ -1742,4 +1739,4 @@ interface SkillOptOptimizationMethodConfig<TScenario extends Scenario, TArtifact
|
|
|
1742
1739
|
declare function skillOptOptimizationMethod<TScenario extends Scenario, TArtifact>(config: SkillOptOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1743
1740
|
//#endregion
|
|
1744
1741
|
export { Objective as $, optimizationTokenUsageFromSummary as $t, RunEvalOptions as A, CanarySeverity as At, OptimizerModelBudget as B, ComparisonCost as Bt, RunImprovementLoopOptions as C, ReferenceEquivalenceJudgeResult as Cn, scoreRedTeamOutput as Ct, RunOptimizationOptions as D, LlmJudgeDimension as Dn, CanaryKind as Dt, PremeasuredOptimizationBaseline as E, runReferenceEquivalenceJudge as En, CanaryEvaluation as Et, GepaOptimizationMethodConfig as F, ExternalTextOptimizerContext as Ft, ObjectiveSource as G, OptimizationMethodProvenance as Gt, AxisVerdict as H, OptimizationMethodComparison as Ht, GepaOptimizationRecipe as I, ExternalTextOptimizerResult as It, PromotionPolicy as J, OptimizationMethodScore as Jt, ParetoSignificanceGateOptions as K, OptimizationMethodResult as Kt, GepaRunnerCommand as L, ExternalOptimizationExample as Lt, GepaAdaptiveEngineRun as M, composeGate as Mt, GepaEngineOptions as N, externalTextOptimizationMethod as Nt, RunOptimizationResult as O, LlmJudgeOptions as On, CanaryOptions as Ot, GepaEngineRun as P, ExternalTextOptimizationMethodConfig as Pt, Direction as Q, costFromLedgerSummary as Qt, gepaOptimizationMethod as R, ExternalTextEvaluationResponse as Rt, verifyLoopProvenanceRecord as S, ReferenceEquivalenceJudgeOptions as Sn, redTeamReport as St, runImprovementLoop as T, createReferenceEquivalenceJudge as Tn, CanaryAlert as Tt, BuildEvidenceVectorOptions as U, OptimizationMethodInput as Ut, AxisEvidence as V, OptimizationMethod as Vt, EvidenceVector as W, OptimizationMethodPairwise as Wt, paretoPolicy as X, OptimizationTokenUsage as Xt, buildEvidenceVector as Y, OptimizationPackageSource as Yt, paretoSignificanceGate as Z, compareOptimizationMethods as Zt, emitLoopProvenance as _, OpenAutoPrResult as _n, RedTeamCategory as _t, BuildLoopProvenanceArgs as a, planCampaignRun as an, scalarScore as at, provenanceRecordPath as b, REFERENCE_EQUIVALENCE_JUDGE_VERSION as bn, RedTeamReport as bt, LoopProvenanceArgsFromResult as c, createRunCostLedger as cn, powerPreflight as ct, LoopProvenanceEvidence as d, assertCampaignDesign as dn, DefaultProductionGateCheck as dt, CampaignCellFailureReceipt as en, ParetoResult as et, LoopProvenanceOptimizationMethod as f, assertCampaignSplitIdentity as fn, DefaultProductionGateOptions as ft, canonicalDigest as g, OpenAutoPrOptions as gn, RedTeamCase as gt, campaignMeasurementDigest as h, campaignSplitDigestFromIdentities as hn, DEFAULT_RED_TEAM_CORPUS as ht, skillOptOptimizationMethod as i, RunCampaignOptions as in, paretoFrontierWithCrowding as it, runEval as j, runCanaries as jt, runOptimization as k, llmJudge as kn, CanaryReport as kt, LoopProvenanceBackend as l, fsCampaignStorage as ln, HeldOutGateOptions as lt, buildLoopProvenanceRecord as m, campaignSplitDigest as mn, defaultProductionGate as mt, SkillOptRunnerCommand as n, CampaignRunPlanCell as nn, dominates as nt, EmitLoopProvenanceArgs as o, runCampaign as on, PowerPreflight as ot, LoopProvenanceRecord as p, campaignScenarioIdentity as pn, DefaultProductionRewardHackingOptions as pt, PromotionObjective as q, OptimizationMethodRunOptions as qt, SkillOptTrainerConfig as r, PlanCampaignRunOptions as rn, paretoFrontier as rt, EmitLoopProvenanceResult as s, CampaignStorage as sn, PowerPreflightOptions as st, SkillOptOptimizationMethodConfig as t, CampaignRunPlan as tn, crowdingDistance as tt, LoopProvenanceCandidate as u, inMemoryCampaignStorage as un, heldOutGate as ut, loopProvenanceArgsFromResult as v, openAutoPr as vn, RedTeamFinding as vt, RunImprovementLoopResult as w, ReferenceEquivalenceScenario as wn, toolNamesForRun as wt, provenanceSpansPath as x, ReferenceEquivalenceJudgeInput as xn, redTeamDataset as xt, loopProvenanceSpans as y, REFERENCE_EQUIVALENCE_INPUT_LIMITS as yn, RedTeamPayload as yt, OpenAICompatibleOptimizerModel as z, CompareOptimizationMethodsOptions as zt };
|
|
1745
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
1742
|
+
//# sourceMappingURL=skillopt-optimization-method-DJ3l4w8W.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skillopt-optimization-method-DJ3l4w8W.d.ts","names":[],"sources":["../src/llm-judge.ts","../src/reference-equivalence-judge.ts","../src/campaign/auto-pr.ts","../src/campaign/coverage.ts","../src/campaign/external-optimizer-contracts.ts","../src/campaign/storage.ts","../src/campaign/run-campaign.ts","../src/campaign/presets/compare-optimization-methods.ts","../src/campaign/external-text-evaluation.ts","../src/campaign/external-text-optimization-contract.ts","../src/campaign/external-text-optimization.ts","../src/campaign/gates/compose.ts","../src/canary.ts","../src/red-team.ts","../src/campaign/gates/default-production-gate.ts","../src/campaign/gates/heldout-gate.ts","../src/campaign/gates/power-preflight.ts","../src/pareto.ts","../src/campaign/gates/promotion-policy.ts","../src/campaign/optimizer-model.ts","../src/campaign/gepa-optimization-method.ts","../src/campaign/presets/run-eval.ts","../src/campaign/presets/run-optimization.ts","../src/campaign/presets/run-improvement-loop.ts","../src/campaign/provenance.ts","../src/campaign/skillopt-optimization-method.ts"],"mappings":";;;;;;;;;;;;;;;KA6CY,6BAA6B;UAExB,gBAAgB,WAAW,kBAAkB,WAAW;;;;EAIvE,MAAM;;;EAGN,aAAa;;EAEb;;EAEA;EACA;EACA;;;EAGA,UAAU;;;;;EAKV;;EAEA,aAAa,UAAU;;;EAGvB,cAAc;IAAS,UAAU;IAAW,UAAU;;;EAEtD,aAAa;EACb;IAAmB;IAAc,QAAQ,EAAE;;;;;;;;;;;;;iBAoB7B,SAAS,qBAAqB,kBAAkB,WAAW,UACzE,cACA,gBACA,MAAM,gBAAgB,WAAW,aAChC,YAAY,WAAW;;;cC7Fb;cAEA;WACX;WACA;WACA;;UAGe,qCAAqC;EACpD;EACA;;UAGe;EACf;EACA;EACA;;UAGe;;EAEf,MAAM;;EAEN;;EAEA,SAAS;;EAET,aAAa;;UAGE,wCAAwC;EACvD;EACA;EACA;EACA;;;iBA0Bc,gCACd,SAAS,mCACR,oBAAoB;;iBA2CD,6BACpB,OAAO,gCACP,SAAS,mCACR,QAAQ;;;UCjGM,kBAAkB,WAAW,kBAAkB;;EAE9D,QAAQ,eAAe,WAAW;;;EAGlC,MAAM;;;EAGN;;EAEA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA,UAAU;IAAqB;IAAgB;IAAgB;;;UAGhD;EACf;EACA;EACA;EACA;;;;;iBAMc,WAAW,WAAW,kBAAkB,UACtD,SAAS,kBAAkB,WAAW,aACrC;;;;iBC1Ca,qBAAqB,kBAAkB,UACrD,oBAAoB,aACpB;;iBA2Bc,yBAAyB,kBAAkB,UACzD,UAAU,YACT,2BAA2B,KAAK;;iBAUnB,kCACd,oBAAoB,4BACpB;;iBAgBc,oBAAoB,kBAAkB,UACpD,oBAAoB,aACpB;;iBAOc,4BACd,oBAAoB,4BACpB,cACA;;;UC7Ee;EACf;EACA;EACA,MAAM,OAAO;;KAGH;KAEA,iCAAiC;UAE5B;EACf,WAAW;EACX;;UAUe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,SAAS;;EAET;;;;;;;;;;;;;;;;;;;UCrBe;;EAEf,UAAU;;EAEV,OAAO;;EAEP,KAAK;;EAEL,MAAM,cAAc,kBAAkB;;;EAGtC,OAAO,cAAc,iBAAiB;;;;;;;;;iBAUxB,qBAAqB;;;;iBA0CrB,2BAA2B;;iBAmC3B,oBAAoB;EAClC,SAAS;EACT;EACA;IACE;;;UC/Ea,mBAAmB,kBAAkB,UAAU;EAC9D,WAAW;EACX,UAAU,WAAW,WAAW;;EAEhC,SAAS;;;;;;EAMT;EACA,SAAS,YAAY,WAAW;;EAEhC;;;EAGA;;;EAGA;;;;EAIA,eAAe;EACf;EACA;;EAEA;;;EAGA,aAAa;;EAEb;;EAEA,WAAW,SAAS;;EAEpB;;;;;;;;EAQA;;;;;;;;;;EAUA;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA;;;;;;;;;;;EAWA;;EAEA,YAAY;;EAEZ,oBAAoB,gBAAgB,gBAAgB;;;;;;EAMpD,UAAU;;;;;;;;;;;;EAYV,iBAAiB;IACf,UAAU;IACV;IACA;;;;;;;UAQa,2BAA2B;EAC1C;EACA;EACA;EACA;IACE;IACA;IACA;MACE;MACA;MACA;;;EAGJ,MAAM,mBAAmB;EACzB,MAAM;;;;;iBAMc,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,eAAe,WAAW;UAyjBpB;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;;UAGQ,uBAAuB,kBAAkB,UAAU;EAClE,WAAW;EACX,WAAW,WAAW,WAAW;EACjC;EACA,SAAS,YAAY,WAAW;EAChC;EACA;EACA;EACA;;EAEA;EACA,UAAU;;;;;;iBAOI,gBAAgB,kBAAkB,UAAU,WAC1D,MAAM,uBAAuB,WAAW,aACvC;;;;KCxvBS,6BAA6B,kBAAkB,UAAU,aAAa,KAChF,mBAAmB,WAAW;;UAKf;EACf;EACA;EACA;;UAGe;EACf;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV,SAAS;;EAET;EACA;;EAEA;EACA;EACA;EACA;EACA,aAAa;;;UAIE,wBAAwB,kBAAkB,UAAU;;WAE1D,iBAAiB;;WAEjB,yBAAyB;;WAEzB,6BAA6B;;WAE7B,sBACP,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;;WAEJ,iBAAiB,YAAY,WAAW;;WAExC;WACA;;WAEA,YAAY,SAAS,6BAA6B,WAAW;;WAE7D,YAAY;;UAGN;;EAEf,eAAe;;EAEf,MAAM;;EAEN;;EAEA,aAAa;;;UAIE,mBAAmB,kBAAkB,WAAW,UAAU;;EAEzE;EACA,WACE,OAAO,wBAAwB,WAAW,eACvC,QAAQ;;UAGE;EACf;;EAEA;;EAEA;;EAEA;;;EAGA;IAAU;IAAa;;;EAEvB,kBAAkB;;EAElB;;EAEA,aAAa;;EAEb,gBAAgB;IACd;IACA;IACA;IACA;;EAEF,eAAe;;EAEf;;UAGe;;EAEf;EACA;;EAEA;EACA;EACA;;EAEA;;UAGe;;EAEf,QAAQ;EACR,MAAM;;EAEN,UAAU;EACV;;EAEA,kBAAkB;;EAElB,UAAU;;EAEV,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,kCAAkC,kBAAkB,UAAU,mBACrE,KAAK,mBAAmB,WAAW;EAC3C,SAAS,mBAAmB,WAAW;EACvC,iBAAiB;;EAEjB,gBAAgB;;EAEhB,oBAAoB;;EAEpB,eAAe;;EAEf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;EACb,QAAQ,YAAY,WAAW;;;EAG/B;;EAEA,yBAAyB,6BAA6B,WAAW;;EAEjE;;;EAGA;;EAEA;;;;;iBAMoB,2BAA2B,kBAAkB,UAAU,WAC3E,MAAM,kCAAkC,WAAW,aAClD,QAAQ;;iBAyrBK,sBAAsB,SAAS,oBAAoB;;iBAWnD,kCACd,SAAS,mBACT,mBAAmB,gBAClB;;;UC76Bc;EACf;EACA;;UAGe;EACf;EACA;IACE;IACA,YAAY;IACZ;IACA;;;;;UCba;WACN;WACA;WACA;WACA;WACA;WACA,eAAe;WACf,mBAAmB;WACnB,uBAAuB;WACvB;WACA;;WAEA;WACA;WACA;WACA,QAAQ;;WAER,MAAM;WACN,WACP,SAFa,kCAGV,QAAQ;;UAGE;EACf,eAAe;EACf;EACA;IACM;;IACA;;IACA;IAAkB;;;;;;;;;UAqBT,qCACf,kBAAkB,UAClB;EAEA;EACA,QAAQ,KAAK;EACb;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA,SAAS;EACT;EACA;EACA,oBAAoB,UAAU;EAC9B,oBAAoB,UAAU,WAAW,UAAU;EACnD,MAAM,SAAS,iCAAiC,QAAQ;;;;;;;;;;iBCP1C,+BAA+B,kBAAkB,UAAU,WACzE,QAAQ,qCAAqC,WAAW,aACvD,mBAAmB,WAAW;;;;;;iBCjEjB,YAAY,qBAAqB,kBAAkB,WAAW,aACzE,OAAO,MAAM,KAAK,WAAW,cAC/B,KAAK,WAAW;;;KCkBP;KAEA;UAEK;EACf,MAAM;EACN,UAAU;EACV;;;EAGA,UAAU;;UAGK;EACf,QAAQ;;EAER,QAAQ,OAAO;;EAEf,aAAa;;UAGE;EACf,MAAM;EACN;EACA;EACA;;UAGe;;;;;;;;;;EAUf;IACE;IACA;;IAEA;;;;;;;;;;;;;EAcF;IACE;IACA;IACA;IACA;;;;;;;;;;EAWF;IACE,WAAW,KAAK;IAChB;IACA;IACA;IACA;;;;;;;;iBASY,YAAY,MAAM,aAAa,OAAM,gBAAqB;;;KClG9D;UAUK;EACf,UAAU;;EAEV;;;;;EAKA;;EAEA;;EAEA;;UAGe,oBAAoB;EACnC,SAAS;;UAGM;EACf;EACA,UAAU;EACV;EACA;EACA;;UAGe;EACf,UAAU;EACV,oBAAoB,OAAO;EAC3B;;;cA0DW,yBAAyB;iBA6FtB,eAAe,aAAY,gBAAqB;;;;;iBAkBhD,mBACd,gBACA,qBACA,QAAQ,cACP;;iBA4Fa,cAAc,UAAU,mBAAmB;;;;;iBAqBrC,gBAAgB,OAAO,YAAY,gBAAgB;;;KC9T7D;KAOA,wCAAwC,KAClD;EAGA,SAAS,YAAY;;UAGN;;;EAGf,kBAAkB;;;;;EAKlB;;EAEA;;EAEA;;EAEA;;;;EAIA;;;;EAIA;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA,iBAAiB;;;EAGjB,aAAa;;EAEb,gBAAgB;;EAEhB,SAAS;;;;;EAKT,iBAAiB;;;;;iBAMH,sBAAsB,WAAW,kBAAkB,UACjE,SAAS,+BACR,KAAK,WAAW;;;UCnFF,mBAAmB,kBAAkB,WAAW;EAC/D,WAAW;;;EAGX;;EAEA;;;;EAIA;;EAEA;;EAEA;;;;;;iBAOc,YAAY,WAAW,kBAAkB,UACvD,SAAS,mBAAmB,aAC3B,KAAK,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;UCfF;;EAEf;;;EAGA;;EAEA;;EAEA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;;EAGA;EACA;EACA;;;EAGA;;EAEA;;;;;;iBAec,eAAe,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;KClEjD;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa;;;;;;;;;;iBA4B7E,YAAY,GAC1B,YAAY,KACZ,YAAY,UAAU,MACtB;EAAW,UAAU,QAAQ;IAC5B;EAAQ,WAAW;EAAG;;;;;;;;;;;;;;iBAyCT,iBAAiB,GAC/B,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;;iBA6BT,2BAA2B,GACzC,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;KC/Hb;EAAoB;;EAAwB;EAAmB;;UAE1D;;EAEf;EACA,QAAQ;;;EAGR,WAAW;;;;EAIX;;;;EAIA;;;KAIU;UAEK;EACf;EACA,QAAQ;EACR,WAAW;;;EAGX,WAAW;;EAEX;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGM;;EAEf,MAAM;;;EAGN;;;EAGA;IAAQ;IAAmB;;;;;;KAMjB,mBAAmB,IAAI,mBAAmB;UAErC;;;;EAIf;;EAEA;;EAEA;;EAEA;;;;;;;;iBASc,oBAAoB,WAAW,kBAAkB,UAC/D,KAAK,YAAY,WAAW,YAC5B,YAAY,sBACZ,OAAM,6BACL;;;;;;;;cAmFU,cAAc;UA2EV,sCAAsC;;EAErD,YAAY;;;EAGZ,SAAS;;EAET;;;;;;;iBAQc,uBAAuB,qBAAqB,kBAAkB,WAAW,UACvF,SAAS,gCACR,KAAK,WAAW;;;KCxRP,uBAAuB;;UAGlB;EACf;EACA;EACA;EACA,QAAQ;;;;;UC6CO;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;EAKA,eAAe;;;UAIA,sBAAsB;;EAErC;;;KAIU,wBAAwB;;;;;;;KAQxB;EAEN;EACA,KAAK;;EAGL;EACA,eAAe;;EAGf;EACA,eAAe;;EAEf;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA,eAAe;EACf;;EAGA;EACA,eAAe;EACf;;EAGA;EACA,kBAAkB;EAClB,cAAc;EACd;;;KAIM,oBAAoB;UAEf,6BAA6B,kBAAkB,UAAU;;EAExE;;EAEA,QAAQ;;EAER;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,YAAY;;;;;;EAMZ,oBAAoB,UAAU;;EAE9B,oBAAoB,UAAU,WAAW,UAAU;;EAEnD,SAAS;;;;;EAKT;EACA,SAAS;;;;;;;;;;iBAWK,uBAAuB,kBAAkB,UAAU,WACjE,QAAQ,6BAA6B,WAAW,aAC/C,mBAAmB,WAAW;;;UClLhB,eAAe,kBAAkB,UAAU,mBAClD,KAAK,mBAAmB,WAAW;EAC3C;;;;;iBAMoB,QAAQ,kBAAkB,UAAU,WACxD,MAAM,eAAe,WAAW,aAC/B,QAAQ,eAAe,WAAW;;;UC0BpB,gCAAgC,WAAW,kBAAkB;;EAE5E;;EAEA,UAAU,eAAe,WAAW;;UAGrB,2BAA2B,kBAAkB,UAAU,mBAC9D,KAAK,mBAAmB,WAAW;;EAE3C,iBAAiB;;;;;;;;;EASjB,sBAAsB,gCAAgC,WAAW;;EAEjE,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;;EAEb,UAAU,gBAAgB;EAC1B;EACA;;;EAGA;;;EAGA;;EAEA,WAAW,cAAc;;;;;;;;;;;EAWzB,qBAAqB;IACnB;IACA;IACA,YAAY;MACV;MACA,UAAU,eAAe,WAAW;MACpC;;IAEF,SAAS;;IAET,aAAa;IACb;QACI,QAAQ,cAAc;;;;;;;;;;;;;;;;;;EAkB5B,oBAAoB,UAAU,eAAe,WAAW;;KAG9C,uBACV,kBAAkB,UAClB,aACE,2BAA2B,WAAW;UAEzB,sBAAsB,WAAW,kBAAkB;EAClE,aAAa;IACX,QAAQ;IACR,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;;EAIxC,iBAAiB;EACjB,eAAe;EACf;;;;EAIA;;;;EAIA;EACA,kBAAkB,eAAe,WAAW;;EAE5C,MAAM;;;;;;EAMN,gBAAgB;;;;;iBAMI,gBAAgB,kBAAkB,UAAU,WAChE,MAAM,uBAAuB,WAAW,aACvC,QAAQ,sBAAsB,WAAW;;;KCpJhC,0BACV,kBAAkB,UAClB,aACE,uBAAuB,WAAW;;;EAGpC,kBAAkB;;;;;;;;;EASlB;;;;EAIA,MAAM,KAAK,WAAW;;;;;EAKtB;;EAEA;EACA;;;;;;;;EAQA,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;UAGlE,yBAAyB,WAAW,kBAAkB,kBAC7D,sBAAsB,WAAW;EACzC,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,uBAAuB,eAAe,WAAW;EACjD,qBAAqB;EACrB,YAAY,QAAQ,WAAW,KAAK,WAAW;;;;EAI/C;;;;;EAKA;EACA,WAAW,kBAAkB;;;;;iBAMT,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,0BAA0B,WAAW,aAC1C,QAAQ,yBAAyB,WAAW;;;UC1B9B;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,UAAU,YAAY;;EAEtB;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;EACA;EACA;;UAGe;EACf;IACE;IACA;;EAEF;IACE;IACA;IACA;IACA;MACE;MACA;MACA;MACA;;;EAGJ;;UAGe;EACf;EACA,MAAM;EACN;EACA,aAAa;;;;;;;UAQE;EACf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;EACA;;EAEA;;EAEA,YAAY;;EAEZ,qBAAqB;;EAErB,UAAU;;EAEV;;EAEA;IACE,UAAU;IACV;IACA;IACA,mBAAmB;;;;;;EAMrB;;EAEA;;EAEA;;;EAGA;;EAEA,SAAS;EACT;EACA;;UAGe,wBAAwB,WAAW,kBAAkB;EACpE;EACA;EACA;EACA,iBAAiB;EACjB,eAAe;EACf;EACA;;EAEA,wBAAwB,eAAe,WAAW;;EAElD,aAAa;IACX;IACA,YAAY;IACZ;;;IAGA,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;EAGxC,MAAM;;;;EAIN;EACA,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,qBAAqB;EACrB,uBAAuB,eAAe,WAAW;;EAEjD,cAAc,cAAc;EAC5B;EACA;EACA,qBAAqB;;UAGN,6BAA6B,WAAW,kBAAkB;EACzE;EACA;EACA;EACA,iBAAiB;EACjB,QAAQ,yBAAyB,WAAW;EAC5C,cAAc,cAAc;EAC5B;EACA;;;iBAIc,6BAA6B,WAAW,kBAAkB,UACxE,OAAO,6BAA6B,WAAW,aAC9C,wBAAwB,WAAW;;iBA4CtB,0BAA0B,WAAW,kBAAkB,UACrE,MAAM,wBAAwB,WAAW,aACxC;;iBAoRa,0BAA0B,WAAW,kBAAkB,UACrE,UAAU,eAAe,WAAW;;iBAoCtB,2BAA2B,QAAQ,uBAAuB;;iBAa1D,gBAAgB;;;;;;;;;;;;iBA2KhB,oBACd,QAAQ,sBACR;EAAQ;IACP;;iBAiJa,qBAAqB;;;;iBAMrB,oBAAoB;UAInB;EACf,QAAQ;EACR,OAAO;;EAEP;EACA;;UAGe,uBAAuB,WAAW,kBAAkB,kBAC3D,wBAAwB,WAAW;;EAE3C,SAAS;;;EAGT,eAAe;;;;;;;;;;;;iBA+EK,mBAAmB,WAAW,kBAAkB,UACpE,MAAM,uBAAuB,WAAW,aACvC,QAAQ;;;UC/8BM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;;;EAKA,YAAY;;KAGF,wBAAwB;UAEnB,iCAAiC,kBAAkB,UAAU;EAC5E;;EAEA;EACA;;EAEA;EACA,SAAS;;;;;EAKT,WAAW;;EAEX;;EAEA;EACA;;EAEA;EACA;EACA,oBAAoB,UAAU;EAC9B,oBAAoB,UAAU,WAAW,UAAU;EACnD,SAAS;EACT,SAAS;;;iBAIK,2BAA2B,kBAAkB,UAAU,WACrE,QAAQ,iCAAiC,WAAW,aACnD,mBAAmB,WAAW"}
|
|
@@ -1,7 +1,32 @@
|
|
|
1
1
|
import { a as RunRecord } from "./run-record-DcObtIGh.js";
|
|
2
2
|
import { n as FailureClusterReport } from "./failure-cluster-CqcvCcdR.js";
|
|
3
3
|
//#region src/held-out-gate.d.ts
|
|
4
|
-
type HeldOutGateRejectionCode = 'few_runs' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'indeterminate_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
4
|
+
type HeldOutGateRejectionCode = 'few_runs' | 'incomplete_coverage' | 'missing_split_scores' | 'missing_cost' | 'negative_delta' | 'indeterminate_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
5
|
+
/**
|
|
6
|
+
* How much of one split's DEALT work the comparison actually measured.
|
|
7
|
+
*
|
|
8
|
+
* `answered + unscoredPairs + candidateOnly + baselineOnly === dealt` — a
|
|
9
|
+
* complete, mutually exclusive partition of the item slots the split was dealt,
|
|
10
|
+
* so nothing can leave the comparison without appearing in exactly one bucket.
|
|
11
|
+
*/
|
|
12
|
+
interface SplitCoverage {
|
|
13
|
+
/** Work items the split was dealt: every item with a row on either arm. */
|
|
14
|
+
dealt: number;
|
|
15
|
+
/** Dealt items carrying a finite score on BOTH arms — the only ones any
|
|
16
|
+
* paired statistic can read. */
|
|
17
|
+
answered: number;
|
|
18
|
+
/** Dealt items with a row on both arms where at least one row carried no
|
|
19
|
+
* finite score: a run happened and produced nothing (crash, timeout, judge
|
|
20
|
+
* failure). Measured absence, not an absent item. */
|
|
21
|
+
unscoredPairs: number;
|
|
22
|
+
/** Dealt items only the candidate produced a row for. */
|
|
23
|
+
candidateOnly: number;
|
|
24
|
+
/** Dealt items only the baseline produced a row for. */
|
|
25
|
+
baselineOnly: number;
|
|
26
|
+
/** `answered / dealt`, or 0 when nothing was dealt. 1 means every dealt item
|
|
27
|
+
* was scored on both arms. */
|
|
28
|
+
coverage: number;
|
|
29
|
+
}
|
|
5
30
|
interface HeldOutGateConfig {
|
|
6
31
|
/** Minimum number of paired (candidate, baseline) holdout observations
|
|
7
32
|
* required before promotion. The exact small-sample test may require more
|
|
@@ -24,6 +49,23 @@ interface HeldOutGateConfig {
|
|
|
24
49
|
/** Optional deterministic seed for the bootstrap. Default undefined
|
|
25
50
|
* (Math.random). */
|
|
26
51
|
seed?: number;
|
|
52
|
+
/**
|
|
53
|
+
* Smallest acceptable `answered / dealt` fraction, required on BOTH splits.
|
|
54
|
+
* Default 1: every work item the comparison was dealt must carry a real score
|
|
55
|
+
* on both arms before the gate will decide anything.
|
|
56
|
+
*
|
|
57
|
+
* The default is 1 because a missing score is missing data, and a promotion
|
|
58
|
+
* computed over "the items that happened to answer" is a promotion computed
|
|
59
|
+
* over a set the candidate selected by failing (see the module header). One
|
|
60
|
+
* split below `minCoverage` rejects with `incomplete_coverage`.
|
|
61
|
+
*
|
|
62
|
+
* Lowering it is a real, defensible choice for a caller who accepts a known
|
|
63
|
+
* flake rate — but it must be DECLARED, not inherited: the shrunken
|
|
64
|
+
* denominator then still ships in `GateEvidence.holdoutCoverage` /
|
|
65
|
+
* `searchCoverage`, so the verdict can never be read without it. Must be in
|
|
66
|
+
* [0, 1]; anything else throws rather than clamping.
|
|
67
|
+
*/
|
|
68
|
+
minCoverage?: number;
|
|
27
69
|
/**
|
|
28
70
|
* Hard ceiling on the candidate's median per-task USD cost. When the
|
|
29
71
|
* candidate clears the quality gates (paired-delta + overfit-gap) but
|
|
@@ -45,6 +87,19 @@ interface GateEvidence {
|
|
|
45
87
|
unpairedCandidateRuns: number;
|
|
46
88
|
/** Baseline holdout rows with no candidate row at the same work identity. */
|
|
47
89
|
unpairedBaselineRuns: number;
|
|
90
|
+
/**
|
|
91
|
+
* How much of the dealt HOLDOUT work carried a score on both arms — the
|
|
92
|
+
* denominator behind `productiveRuns`, `medianPairedDelta`, `pairedCI` and
|
|
93
|
+
* `holdoutScore`. A `coverage` below 1 means those numbers describe a
|
|
94
|
+
* SUBSET of the items the comparison was dealt, and which items are in that
|
|
95
|
+
* subset was decided by whichever arm went dark.
|
|
96
|
+
*/
|
|
97
|
+
holdoutCoverage: SplitCoverage;
|
|
98
|
+
/**
|
|
99
|
+
* The same accounting for the SEARCH split — the denominator behind
|
|
100
|
+
* `searchScore`, and therefore behind both overfit gaps.
|
|
101
|
+
*/
|
|
102
|
+
searchCoverage: SplitCoverage;
|
|
48
103
|
/** Median of paired holdout deltas, or null when there are no pairs. */
|
|
49
104
|
medianPairedDelta: number | null;
|
|
50
105
|
/** Bootstrap CI on the median paired holdout delta, if computed. */
|
|
@@ -106,6 +161,7 @@ declare class HeldOutGate {
|
|
|
106
161
|
private readonly resamples;
|
|
107
162
|
private readonly seed?;
|
|
108
163
|
private readonly costPerTaskCeiling?;
|
|
164
|
+
private readonly minCoverage;
|
|
109
165
|
constructor(config: HeldOutGateConfig);
|
|
110
166
|
/** Decide whether `candidate` should replace `baseline`.
|
|
111
167
|
* Pairing is by `(experimentId, scenarioId, seed)`.
|
|
@@ -404,5 +460,5 @@ interface ResearchReport {
|
|
|
404
460
|
*/
|
|
405
461
|
declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
|
|
406
462
|
//#endregion
|
|
407
|
-
export { HeldOutGateConfig as C, HeldOutGate as S, paretoChart as _, ParetoPoint as a, GateDecision as b, ResearchReportCandidate as c, ResearchReportOptions as d, ResearchReportRecommendation as f, gainHistogram as g, SummaryTableRow as h, ParetoFigureSpec as i, ResearchReportDecision as l, SummaryTableOptions as m, GainDistributionFigureSpec as n, RESEARCH_REPORT_HARD_PAIR_FLOOR as o, SummaryTable as p, GainDistributionOptions as r, ResearchReport as s, GainDistributionBin as t, ResearchReportMethodology as u, researchReport as v, HeldOutGateRejectionCode as w, GateEvidence as x, summaryTable as y };
|
|
408
|
-
//# sourceMappingURL=summary-report-
|
|
463
|
+
export { HeldOutGateConfig as C, HeldOutGate as S, SplitCoverage as T, paretoChart as _, ParetoPoint as a, GateDecision as b, ResearchReportCandidate as c, ResearchReportOptions as d, ResearchReportRecommendation as f, gainHistogram as g, SummaryTableRow as h, ParetoFigureSpec as i, ResearchReportDecision as l, SummaryTableOptions as m, GainDistributionFigureSpec as n, RESEARCH_REPORT_HARD_PAIR_FLOOR as o, SummaryTable as p, GainDistributionOptions as r, ResearchReport as s, GainDistributionBin as t, ResearchReportMethodology as u, researchReport as v, HeldOutGateRejectionCode as w, GateEvidence as x, summaryTable as y };
|
|
464
|
+
//# sourceMappingURL=summary-report-DGp0-_XO.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summary-report-DGp0-_XO.d.ts","names":[],"sources":["../src/held-out-gate.ts","../src/summary-report.ts"],"mappings":";;;KAyEY;;;;;;;;UAiBK;;EAEf;;;EAGA;;;;EAIA;;EAEA;;EAEA;;;EAGA;;UAGe;;;;EAIf;;;EAGA;;;;EAIA;;;EAGA;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;;;;;;EAiBA;;;;;;;;;;;;;EAaA;;UAGe;;EAEf;;EAEA;;EAEA;;;;;;;;EAQA,iBAAiB;;;;;EAKjB,gBAAgB;;EAEhB;;EAEA;IAAY;IAAa;;;EAEzB;;EAEA;;EAEA;;EAEA;;EAEA;;;;;EAKA;;;EAGA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA,UAAU;;EAEV;;EAEA,eAAe;;;;;;;cAQJ;mBACM;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;mBACA;EAEjB,YAAY,QAAQ;;;;EAmCpB,SAAS,WAAW,aAAa,UAAU,cAAc;;;;UC9O1C;;;EAGf;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;EACf,MAAM;EACN;EACA;;EAEA;;;;;;;iBAQc,aAAa,MAAM,aAAa,OAAM,sBAA2B;UA6IhE;EACf;;EAEA;;EAEA;;EAEA;;;EAGA;;;EAGA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;IAAQ;IAAc;;;;;;;;iBAQR,YACd,MAAM,aACN;EACE;EACA,gBAAgB,eAAe;IAEhC;UAsDc;;EAEf;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA,MAAM;EACN;EACA;IAAM;IAAa;;;UAGJ;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;iBAQc,cACd,MAAM,aACN,qBACA,oBACA,OAAM,0BACL;KA8ES;;;;;;cAYC;UAEI;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;EASA;;;;;;;EAOA;IAAS;IAAa;;;;;;EAKtB;;;;;EAKA;;EAEA,gBAAgB,eAAe;;EAE/B,kBAAkB;;EAElB;;EAEA;;EAEA;;;;;;EAMA;;UAGe;EACf,UAAU;EACV;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IAAU;IAAa;;;;;;;EAMvB;EACA;;;;;;;EAOA;EACA;EACA,OAAO;EACP,UAAU;EACV;;UAGe;;;;;EAKf;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;;;;;;EAMA;EACA;EACA;IAAQ;IAAa;;EACrB;EACA,gBAAgB;EAChB,YAAY;EACZ,SAAS;EACT;IACE,QAAQ;IACR,OAAO;;EAET,aAAa;EACb,kBAAkB;EAClB;EACA;;;;;;;;;;;;;;;;;;;;;iBA6IoB,eACpB,MAAM,aACN,OAAM,wBACL,QAAQ"}
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { c as CostLedgerHandle } from "./cost-ledger-fGS_u_O1.js";
|
|
2
|
+
import { a as RunRecord, n as RunCostProvenance, u as RunTokenUsage } from "./run-record-DcObtIGh.js";
|
|
3
|
+
import { A as ChatClient, m as JudgeInput } from "./types-Cc3qbqzj.js";
|
|
4
|
+
import { t as TraceAnalysisStore } from "./store-CxJry_cs.js";
|
|
5
|
+
//#region src/analyst/types.d.ts
|
|
6
|
+
/**
|
|
7
|
+
* Unified envelope every analyst emits. Schema-versioned so renderers
|
|
8
|
+
* and time-series diffs survive future field additions.
|
|
9
|
+
*/
|
|
10
|
+
interface AnalystFinding {
|
|
11
|
+
schema_version: '1.0.0';
|
|
12
|
+
/**
|
|
13
|
+
* Stable hash over identity-defining fields (analyst_id + canonical
|
|
14
|
+
* claim + area + optional subject). Two findings from two runs that
|
|
15
|
+
* "are the same finding" share this id — that's what `diffFindings`
|
|
16
|
+
* uses to compute appeared/disappeared sets across runs.
|
|
17
|
+
*/
|
|
18
|
+
finding_id: string;
|
|
19
|
+
analyst_id: string;
|
|
20
|
+
produced_at: string;
|
|
21
|
+
severity: AnalystSeverity;
|
|
22
|
+
/**
|
|
23
|
+
* Coarse classification. Renderers group by this. Free-form so
|
|
24
|
+
* domain-specific analysts can introduce categories without a
|
|
25
|
+
* schema change ('agent-reasoning', 'verification', 'cost',
|
|
26
|
+
* 'tool-use', 'safety', 'latency', 'data-quality', ...).
|
|
27
|
+
*/
|
|
28
|
+
area: string;
|
|
29
|
+
claim: string;
|
|
30
|
+
rationale?: string;
|
|
31
|
+
evidence_refs: EvidenceRef[];
|
|
32
|
+
recommended_action?: string;
|
|
33
|
+
validation_plan?: string;
|
|
34
|
+
/** 0..1 — the analyst's own confidence. Not calibrated across analysts. */
|
|
35
|
+
confidence: number;
|
|
36
|
+
/**
|
|
37
|
+
* Optional subject the finding is about — leaf id, agent id, request
|
|
38
|
+
* id. Included in finding_id when present so per-subject findings
|
|
39
|
+
* diff cleanly across runs.
|
|
40
|
+
*/
|
|
41
|
+
subject?: string;
|
|
42
|
+
/** True when this finding was lifted from a judge result rather than observed
|
|
43
|
+
* directly in a trace or artifact. Descriptive only: proposal access is
|
|
44
|
+
* controlled by `ProposalFinding.proposal_origin`. */
|
|
45
|
+
derived_from_judge?: boolean;
|
|
46
|
+
/** Analyst-private extras; renderers ignore unless they know the analyst. */
|
|
47
|
+
metadata?: Record<string, unknown>;
|
|
48
|
+
}
|
|
49
|
+
type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
|
|
50
|
+
/** Data sources that candidate generation may intentionally learn from. */
|
|
51
|
+
type ProposalFindingOrigin = 'search' | 'production';
|
|
52
|
+
/** A finding explicitly admitted as candidate-generation input. */
|
|
53
|
+
type ProposalFinding = AnalystFinding & {
|
|
54
|
+
readonly proposal_origin: ProposalFindingOrigin;
|
|
55
|
+
};
|
|
56
|
+
interface EvidenceRef {
|
|
57
|
+
/**
|
|
58
|
+
* Where the evidence lives. `span` and `event` refer to OTLP trace
|
|
59
|
+
* elements; `artifact` to a file inside the run's artifact tree;
|
|
60
|
+
* `finding` to another AnalystFinding (cross-analyst chaining);
|
|
61
|
+
* `metric` to a named scalar reading the renderer knows how to read.
|
|
62
|
+
*/
|
|
63
|
+
kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
|
|
64
|
+
uri: string;
|
|
65
|
+
excerpt?: string;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* The discriminator the registry uses to pass the right input.
|
|
69
|
+
* `custom` is the escape hatch — analysts that need something else
|
|
70
|
+
* (e.g. an embedding cache, a partner SDK handle) read it from
|
|
71
|
+
* `AnalystRunInputs.custom[<analyst id>]`.
|
|
72
|
+
*/
|
|
73
|
+
type AnalystInputKind = 'trace-store' | 'artifact-dir' | 'run-record' | 'judge-input' | 'custom';
|
|
74
|
+
interface AnalystCost {
|
|
75
|
+
/** `deterministic` analysts MUST NOT call the LLM. */
|
|
76
|
+
kind: 'deterministic' | 'llm';
|
|
77
|
+
/** Optional declared upper bound; the registry can enforce a budget. */
|
|
78
|
+
est_usd_per_run?: number;
|
|
79
|
+
/** Models the analyst expects to use (informational). */
|
|
80
|
+
models?: string[];
|
|
81
|
+
/** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
|
|
82
|
+
settlement_timeout_ms?: number;
|
|
83
|
+
}
|
|
84
|
+
interface AnalystRequirements {
|
|
85
|
+
/** Min number of shots / samples the analyst needs to produce signal. */
|
|
86
|
+
min_shots?: number;
|
|
87
|
+
/** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */
|
|
88
|
+
capabilities?: string[];
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* What's passed to every analyst call. The registry resolves which
|
|
92
|
+
* field the analyst's `inputKind` selects and asserts it's present.
|
|
93
|
+
*/
|
|
94
|
+
interface AnalystRunInputs {
|
|
95
|
+
traceStore?: TraceAnalysisStore;
|
|
96
|
+
artifactDir?: string;
|
|
97
|
+
runRecord?: RunRecord;
|
|
98
|
+
judgeInput?: JudgeInput;
|
|
99
|
+
/** Keyed by analyst id; populated by callers that registered custom analysts. */
|
|
100
|
+
custom?: Record<string, unknown>;
|
|
101
|
+
}
|
|
102
|
+
interface AnalystContext {
|
|
103
|
+
runId: string;
|
|
104
|
+
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
105
|
+
correlationId: string;
|
|
106
|
+
/** Enforced wall-clock deadline (epoch ms). */
|
|
107
|
+
deadlineMs?: number;
|
|
108
|
+
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
109
|
+
budgetUsd?: number;
|
|
110
|
+
/** Shared paid-call account when the analyst runs inside a larger campaign. */
|
|
111
|
+
costLedger?: CostLedgerHandle;
|
|
112
|
+
/** Attribution phase used when writing to the shared paid-call account. */
|
|
113
|
+
costPhase?: string;
|
|
114
|
+
/**
|
|
115
|
+
* Shared chat client. Analysts that call an LLM go through this so
|
|
116
|
+
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
117
|
+
* direct-provider | mock) at the registry boundary without touching
|
|
118
|
+
* analyst code.
|
|
119
|
+
*/
|
|
120
|
+
chat?: ChatClient;
|
|
121
|
+
/**
|
|
122
|
+
* Findings from a prior run the operator wants the analyst to see as
|
|
123
|
+
* retrieval context. Kinds that take advantage of cross-run memory
|
|
124
|
+
* (failure-mode "I saw this cluster last run", knowledge-gap "the wiki
|
|
125
|
+
* page I asked for is still missing") render these into the actor's
|
|
126
|
+
* working set. Filtering is the operator's job: pass the slice that
|
|
127
|
+
* matches the analyst's id, or pass everything and let the kind
|
|
128
|
+
* filter. Empty / absent means no cross-run context.
|
|
129
|
+
*/
|
|
130
|
+
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
131
|
+
/**
|
|
132
|
+
* Findings emitted by analysts that completed earlier in this registry run.
|
|
133
|
+
* This is separate from `priorFindings`: upstream findings are dependency
|
|
134
|
+
* context for the current pass, while prior findings are cross-run memory.
|
|
135
|
+
* The registry populates this only when `RegistryRunOpts.chainFindings` is on.
|
|
136
|
+
*/
|
|
137
|
+
upstreamFindings?: ReadonlyArray<AnalystFinding>;
|
|
138
|
+
/**
|
|
139
|
+
* Report metered work independently of findings. This keeps an empty finding
|
|
140
|
+
* set from erasing token/cost telemetry. Multiple receipts are accumulated.
|
|
141
|
+
*/
|
|
142
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
143
|
+
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
144
|
+
tags?: Record<string, string>;
|
|
145
|
+
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
146
|
+
log?: (msg: string, fields?: Record<string, unknown>) => void;
|
|
147
|
+
/** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */
|
|
148
|
+
signal?: AbortSignal;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The minimal contract. Concrete analysts can refine `TInput` so
|
|
152
|
+
* implementations stay type-safe (e.g. a trace analyst's `TInput` is
|
|
153
|
+
* `TraceAnalysisStore`); the registry passes the right field from
|
|
154
|
+
* `AnalystRunInputs` based on `inputKind`.
|
|
155
|
+
*/
|
|
156
|
+
interface Analyst<TInput = unknown> {
|
|
157
|
+
/** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */
|
|
158
|
+
readonly id: string;
|
|
159
|
+
/** Human-readable. One sentence. */
|
|
160
|
+
readonly description: string;
|
|
161
|
+
readonly inputKind: AnalystInputKind;
|
|
162
|
+
readonly cost: AnalystCost;
|
|
163
|
+
readonly requires?: AnalystRequirements;
|
|
164
|
+
/** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */
|
|
165
|
+
readonly version: string;
|
|
166
|
+
analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
|
|
167
|
+
}
|
|
168
|
+
/** Metered work performed by one analyst call. */
|
|
169
|
+
interface AnalystUsageReceipt {
|
|
170
|
+
/** Number of model-usage records observed at the provider boundary. */
|
|
171
|
+
calls: number | null;
|
|
172
|
+
/** Null when the provider did not return token accounting. */
|
|
173
|
+
tokens: RunTokenUsage | null;
|
|
174
|
+
/** Observed, estimated, or explicitly uncaptured dollar cost. */
|
|
175
|
+
cost: RunCostProvenance;
|
|
176
|
+
/** Known lower bound when one or more calls have uncaptured cost. */
|
|
177
|
+
knownCostUsd?: number;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Compute the stable finding_id from the identity-defining fields.
|
|
181
|
+
* Default implementation hashes {analyst_id, area, subject, normalized claim}.
|
|
182
|
+
* Analysts that emit findings whose claim text varies per run (timestamps,
|
|
183
|
+
* counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,
|
|
184
|
+
* or (b) move the variable part into `rationale`/`metadata` and keep the
|
|
185
|
+
* `claim` static.
|
|
186
|
+
*/
|
|
187
|
+
declare function computeFindingId(input: {
|
|
188
|
+
analyst_id: string;
|
|
189
|
+
area: string;
|
|
190
|
+
subject?: string;
|
|
191
|
+
claim: string;
|
|
192
|
+
/** Override the claim for hashing — use when the displayed claim has run-specific bits. */
|
|
193
|
+
id_basis?: string;
|
|
194
|
+
}): string;
|
|
195
|
+
/**
|
|
196
|
+
* Convenience factory: produce a fully-formed AnalystFinding with the
|
|
197
|
+
* id computed automatically. Analyst code stays terse.
|
|
198
|
+
*/
|
|
199
|
+
declare function makeFinding(init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
|
|
200
|
+
id_basis?: string;
|
|
201
|
+
produced_at?: string;
|
|
202
|
+
}): AnalystFinding;
|
|
203
|
+
/** Build a finding whose source is explicitly allowed during candidate generation. */
|
|
204
|
+
declare function makeProposalFinding(init: Omit<ProposalFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
|
|
205
|
+
id_basis?: string;
|
|
206
|
+
produced_at?: string;
|
|
207
|
+
}): ProposalFinding;
|
|
208
|
+
interface AnalystRunSummary {
|
|
209
|
+
analyst_id: string;
|
|
210
|
+
status: 'ok' | 'skipped' | 'failed';
|
|
211
|
+
/** Why skipped — missing input, budget exceeded, capability unmet. */
|
|
212
|
+
reason?: string;
|
|
213
|
+
findings_count: number;
|
|
214
|
+
latency_ms: number;
|
|
215
|
+
/** Additive model usage and cost provenance for this analyst. */
|
|
216
|
+
usage: AnalystUsageReceipt;
|
|
217
|
+
/** When `status='failed'`: the error class + message, never the full stack. */
|
|
218
|
+
error?: {
|
|
219
|
+
class: string;
|
|
220
|
+
message: string;
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
interface AnalystRunResult {
|
|
224
|
+
run_id: string;
|
|
225
|
+
correlation_id: string;
|
|
226
|
+
started_at: string;
|
|
227
|
+
ended_at: string;
|
|
228
|
+
findings: AnalystFinding[];
|
|
229
|
+
per_analyst: AnalystRunSummary[];
|
|
230
|
+
/** Total LLM cost in USD across all analysts in this registry.run(). */
|
|
231
|
+
total_cost_usd: number;
|
|
232
|
+
/**
|
|
233
|
+
* Provenance for `total_cost_usd`. When uncaptured, the numeric field is only
|
|
234
|
+
* the known subtotal and must not be treated as the run's total spend.
|
|
235
|
+
*/
|
|
236
|
+
total_cost_provenance?: RunCostProvenance;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Events emitted by `AnalystRegistry.runStream(...)` in real time as
|
|
240
|
+
* the registry executes. UIs subscribe via `for await (const ev of
|
|
241
|
+
* registry.runStream(...))`; `registry.run(...)` is a thin collector
|
|
242
|
+
* over the same stream, so the two surfaces share their invariants.
|
|
243
|
+
*
|
|
244
|
+
* Per-finding events are intentionally omitted — analyzers are batch
|
|
245
|
+
* operations (an Ax actor returns the full `findings:json[]` at the
|
|
246
|
+
* end of the responder), so streaming inside one analyst would only
|
|
247
|
+
* emit partial JSON consumers can't render. The kind-completion event
|
|
248
|
+
* is the right granularity; subscribers wanting per-finding rendering
|
|
249
|
+
* iterate `event.findings` themselves.
|
|
250
|
+
*/
|
|
251
|
+
type AnalystRunEvent = {
|
|
252
|
+
type: 'run-started';
|
|
253
|
+
run_id: string;
|
|
254
|
+
correlation_id: string;
|
|
255
|
+
started_at: string;
|
|
256
|
+
/** The ordered list of analyst ids the registry will run. */
|
|
257
|
+
analyst_ids: ReadonlyArray<string>;
|
|
258
|
+
} | {
|
|
259
|
+
type: 'analyst-skipped';
|
|
260
|
+
summary: AnalystRunSummary;
|
|
261
|
+
} | {
|
|
262
|
+
type: 'analyst-started';
|
|
263
|
+
analyst_id: string;
|
|
264
|
+
started_at: string;
|
|
265
|
+
} | {
|
|
266
|
+
type: 'analyst-completed';
|
|
267
|
+
/** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */
|
|
268
|
+
summary: AnalystRunSummary;
|
|
269
|
+
findings: ReadonlyArray<AnalystFinding>;
|
|
270
|
+
} | {
|
|
271
|
+
type: 'run-completed';
|
|
272
|
+
result: AnalystRunResult;
|
|
273
|
+
};
|
|
274
|
+
//#endregion
|
|
275
|
+
export { makeFinding as _, AnalystInputKind as a, AnalystRunInputs as c, AnalystSeverity as d, AnalystUsageReceipt as f, computeFindingId as g, ProposalFindingOrigin as h, AnalystFinding as i, AnalystRunResult as l, ProposalFinding as m, AnalystContext as n, AnalystRequirements as o, EvidenceRef as p, AnalystCost as r, AnalystRunEvent as s, Analyst as t, AnalystRunSummary as u, makeProposalFinding as v };
|
|
276
|
+
//# sourceMappingURL=types-DVjczBM9.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-DVjczBM9.d.ts","names":[],"sources":["../src/analyst/types.ts"],"mappings":";;;;;;;;;UA4BiB;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;KAGD;;KAGA;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;UASM,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAiBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { S as PaidCallResult, T as RunPaidCallInput, a as CostChannel, c as CostLedgerHandle, f as CostLedgerSummary } from "./cost-ledger-fGS_u_O1.js";
|
|
2
2
|
import { u as RunTokenUsage } from "./run-record-DcObtIGh.js";
|
|
3
3
|
import { n as LlmCallMetadata } from "./llm-client-BiK4HW0u.js";
|
|
4
|
+
import { m as ProposalFinding } from "./types-DVjczBM9.js";
|
|
4
5
|
//#region src/campaign/types.d.ts
|
|
5
6
|
/** Stable identifier + kind tag for any scenario. Consumers
|
|
6
7
|
* extend with their per-domain payload (persona, task, requirement, ...). */
|
|
@@ -239,56 +240,38 @@ interface ScoredSurfaceOutcome {
|
|
|
239
240
|
scorableCells: number;
|
|
240
241
|
};
|
|
241
242
|
}
|
|
242
|
-
/**
|
|
243
|
-
*
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
interface ProposeContext<TFindings = unknown> {
|
|
249
|
-
currentSurface: MutableSurface;
|
|
250
|
-
history: GenerationRecord[];
|
|
251
|
-
findings: TFindings[];
|
|
243
|
+
/** Search state supplied to one candidate-generation call.
|
|
244
|
+
* Final evaluation data is not represented in this contract. */
|
|
245
|
+
interface ProposeContext<TFindings = ProposalFinding> {
|
|
246
|
+
readonly currentSurface: MutableSurface;
|
|
247
|
+
readonly history: ReadonlyArray<GenerationRecord>;
|
|
248
|
+
readonly findings: ReadonlyArray<TFindings>;
|
|
252
249
|
/** BREADTH: how many candidate surfaces to return this generation. */
|
|
253
|
-
populationSize: number;
|
|
254
|
-
generation: number;
|
|
255
|
-
signal: AbortSignal;
|
|
250
|
+
readonly populationSize: number;
|
|
251
|
+
readonly generation: number;
|
|
252
|
+
readonly signal: AbortSignal;
|
|
256
253
|
/** Present when a multi-track lineage requests this proposal. */
|
|
257
|
-
track?: ProposalTrackContext;
|
|
254
|
+
readonly track?: ProposalTrackContext;
|
|
258
255
|
/** Measured baseline for this optimization run. `runOptimization` always
|
|
259
256
|
* supplies it; optional for standalone proposer callers. */
|
|
260
|
-
baselineOutcome?: ScoredSurfaceOutcome;
|
|
257
|
+
readonly baselineOutcome?: ScoredSurfaceOutcome;
|
|
261
258
|
/** Measured result for `currentSurface`, the complete global incumbent every
|
|
262
259
|
* new candidate mutates. `runOptimization` always supplies it. */
|
|
263
|
-
incumbentOutcome?: ScoredSurfaceOutcome;
|
|
264
|
-
/** Optional analysis report produced before proposal. Opaque to the substrate:
|
|
265
|
-
* the proposer that consumes it owns the shape. */
|
|
266
|
-
report?: unknown;
|
|
267
|
-
/** Handle to all captured data — the proposer samples traces / artifacts /
|
|
268
|
-
* rewards here to ground its proposals. */
|
|
269
|
-
dataset?: LabeledScenarioStore;
|
|
260
|
+
readonly incumbentOutcome?: ScoredSurfaceOutcome;
|
|
270
261
|
/** DEPTH: max iterations the agentic generator may take per candidate.
|
|
271
262
|
* 1 = single-shot; >1 = it may iterate on its own change before handing it
|
|
272
263
|
* back to be measured. */
|
|
273
|
-
maxImprovementShots?: number;
|
|
264
|
+
readonly maxImprovementShots?: number;
|
|
274
265
|
/** GEPA Pareto frontier across ALL generations so far — the non-dominated
|
|
275
266
|
* surfaces by per-scenario objective vector. Empty/absent on generation 0
|
|
276
267
|
* (only the baseline is scored). A reflective proposer combines the
|
|
277
268
|
* complementary lessons of these parents (each excels on different
|
|
278
269
|
* scenarios) into a merged candidate. Proposers doing pure single-parent
|
|
279
270
|
* reflection may ignore it. See {@link ParetoParent}. */
|
|
280
|
-
paretoParents?: ParetoParent
|
|
271
|
+
readonly paretoParents?: ReadonlyArray<ParetoParent>;
|
|
281
272
|
/** Shared run spend account and receipt attribution phase. */
|
|
282
|
-
costLedger?: CostLedgerHandle;
|
|
283
|
-
costPhase?: string;
|
|
284
|
-
/** FIREWALL (non-negotiable): the held-out judge is write-only — its verdicts
|
|
285
|
-
* score the chosen output and gate promotion, and are NEVER an input to
|
|
286
|
-
* proposal/steering (else the optimizer games the acceptance axis = an
|
|
287
|
-
* oracle). This `never`-typed field makes that a compile-time tripwire: a
|
|
288
|
-
* proposer that tries to thread judge verdicts into the proposal will not type.
|
|
289
|
-
* Steering may consume TRACE-OBSERVABLE signals (what the agent did) via
|
|
290
|
-
* `findings`/`report`; it may NOT consume the judge's held-out verdict. */
|
|
291
|
-
judgeScores?: never;
|
|
273
|
+
readonly costLedger?: CostLedgerHandle;
|
|
274
|
+
readonly costPhase?: string;
|
|
292
275
|
}
|
|
293
276
|
/** A surface-improvement strategy. Given the current best
|
|
294
277
|
* surface, the history of what's been tried + scored, and any external
|
|
@@ -302,7 +285,7 @@ interface ProposeContext<TFindings = unknown> {
|
|
|
302
285
|
* behavior-fuzzing `MutationProposer` (`fuzz/types`), a scenario generator for
|
|
303
286
|
* a different loop.
|
|
304
287
|
*/
|
|
305
|
-
interface SurfaceProposer<TFindings =
|
|
288
|
+
interface SurfaceProposer<TFindings = ProposalFinding> {
|
|
306
289
|
kind: string;
|
|
307
290
|
/** Plan: propose N candidate surfaces for the next generation. A proposer
|
|
308
291
|
* may return bare `MutableSurface`s or `ProposedCandidate`s that carry the
|
|
@@ -312,7 +295,7 @@ interface SurfaceProposer<TFindings = unknown> {
|
|
|
312
295
|
/** Decide: stop early when the proposer judges the search converged or
|
|
313
296
|
* exhausted. Default (omitted) runs all `maxGenerations`. */
|
|
314
297
|
decide?(args: {
|
|
315
|
-
history: GenerationRecord
|
|
298
|
+
history: ReadonlyArray<GenerationRecord>;
|
|
316
299
|
}): {
|
|
317
300
|
stop: boolean;
|
|
318
301
|
reason?: string;
|
|
@@ -637,4 +620,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
637
620
|
}
|
|
638
621
|
//#endregion
|
|
639
622
|
export { LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, labelTrustRank as G, SurfaceProposer as H, ProposedCandidate as I, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, GateDecision as _, CampaignResult as a, GenerationRecord as b, CampaignTraceWriter as c, DispatchContext as d, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, CampaignScenarioIdentity as o, Gate as p, CampaignCellResult as r, CampaignTokenUsage as s, CampaignAggregates as t, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
640
|
-
//# sourceMappingURL=types-
|
|
623
|
+
//# sourceMappingURL=types-DiWLru6Z.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-DiWLru6Z.d.ts","names":[],"sources":["../src/campaign/types.ts"],"mappings":";;;;;;;UA+BiB;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;EAIV;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;iBAKc,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;WACjC,gBAAgB;WAChB,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAG/D;EACf;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;UAKX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;iBASI,eAAe,OAAO;;;;UAOrB,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;;EAGlC;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;EAC5B;;EAEA;;EAEA;;;EAGA,YAAY;;;;EAIZ;EACA;EACA;EACA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;UAGe;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -407,6 +407,11 @@ Concurrent processes cannot write the same compatible run at the same time.
|
|
|
407
407
|
|
|
408
408
|
Use `SurfaceProposer` when your code or runtime owns candidate creation.
|
|
409
409
|
The proposer receives the current surface, prior campaign history, findings, generation number, requested population size, and cancellation signal.
|
|
410
|
+
Every proposal finding must declare `proposal_origin: 'search' | 'production'`.
|
|
411
|
+
`runOptimization()` rejects unclassified findings before it calls the proposer.
|
|
412
|
+
Opaque reports and capture stores are not proposal input.
|
|
413
|
+
Convert analysis into explicit findings or close over a caller-owned knowledge source in your proposer.
|
|
414
|
+
Any caller-owned source must exclude final evaluation cases and results.
|
|
410
415
|
|
|
411
416
|
```ts
|
|
412
417
|
import type { SurfaceProposer } from '@tangle-network/agent-eval/campaign'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.134.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-Cl3pHo4n.d.ts","names":[],"sources":["../src/analyst/types.ts","../src/analyst/finding-signature.ts","../src/analyst/kind-factory.ts","../src/analyst/registry.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;;;;;;UA4BiB;EACf;;;;;;;EAOA;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;;;;;;EASA;;EAEA,WAAW;;KAGD;UAEK;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;UASM,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;;;iBAac,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;;;;;;iBAyBc,YACd,MAAM,KAAK;EACT;EACA;IAED;UAkBc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ;;;;cCxUD;cAEA,0BAAwB,EAAA;;;GAK1B,EAAA,KAAA;KAEC,qBAAqB,EAAE,aAAa;cAiBnC,yBAAuB,EAAA;;;;;;;;;;;;;;;;;GAKzB,EAAA,KAAA;KAEC,oBAAoB,EAAE,aAAa;;;;;;cAOlC;;iBAYG,2BAA2B,SAAS,oBAAoB;iBAQxD,gBACd,cACA,OAAO,aAAa,SAAS,mCAC5B;;;;;;;UCnCc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,aAAa,OAAO,uBAAuB;;EAE3C;IAAe;IAAkB;;;EAEjC;;EAEA;;EAEA;;EAEA,MAAM;;EAEN,eAAe,KAAK,mBAAmB,KAAK,mBAAmB;;EAE/D;;EAEA,UAAU;;;;;;;;UASK;EACf;EACA,UAAU,cAAc,KAAK;;UAGd;;EAEf,IAAI;;EAEJ;;EAEA;;;;;;;;EAQA;IAAa;IAAiB;IAAiB;IAAgB,mBAAmB;;;EAElF;;;;;;;;;;iBAWc,uBACd,MAAM,sBACN,MAAM,6BACL,QAAQ;;;;;;;;;;;;;;;iBAgRK,oBAAoB,OAAO;;iBAyB3B,uBAAuB,UAAU;;;UC3XhC;;EAEf,iBAAiB;IACf,SAAS;IACT,KAAK;IACL;aACS;;EAEX,gBAAgB;IACd,SAAS;IACT,SAAS;IACT,UAAU;IACV;aACS;;;;;;EAMX,SAAS;IACP,SAAS;IACT,OAAO;IACP;MACE,+BAA+B,QAAQ;;EAE3C,YAAY;IAAQ,QAAQ;aAA4B;;UAGzC;;EAEf;;EAEA,UAAU;;;;;;;EAOV,YAAY;IACV,SAAS;IACT;IACA;IACA;;;UAIa;;EAEf,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,QAAQ;;EAER,gBAAgB;;UAGD;;EAEf;;EAEA;;EAEA,SAAS;;EAET;;EAEA,SAAS;;EAET,aAAa;;EAEb;;EAEA,OAAO;;;;;;;;;EASP,gBAAgB,cAAc,kBAAkB,eAAe,cAAc;;;;;;EAM7E;;cAGW;mBACM;mBACA;EAEjB,YAAY,UAAS;EAIrB,SAAS,SAAS;EAmBlB,QAAQ;IACN;IACA;IACA;IACA,MAAM;;EAUF,IACJ,eACA,QAAQ,kBACR,UAAS,kBACR,QAAQ;;;;;;;;;;;;EAoBJ,UACL,eACA,QAAQ,kBACR,UAAS,kBACR,eAAe;UA0OV;UAaA;;;;UC5aO;;EAEf,KAAK;;EAEL;;EAEA,iBAAiB;;EAEjB;;EAEA,WAAW;;iBAGG,4BACd,OAAM,gCACL"}
|