@tangle-network/agent-eval 0.144.7 → 0.144.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/{agent-profile-CgDTo40f.d.ts → agent-profile-DPi7IZg7.d.ts} +3 -3
- package/dist/{agent-profile-CgDTo40f.d.ts.map → agent-profile-DPi7IZg7.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +9 -9
- package/dist/analyst/index.js +1 -1
- package/dist/{analyze-runs-BNNK7irB.js → analyze-runs-C30yljDJ.js} +2 -2
- package/dist/{analyze-runs-BNNK7irB.js.map → analyze-runs-C30yljDJ.js.map} +1 -1
- package/dist/{benchmark-command-BCafwNrf.js → benchmark-command-Dtym4pA-.js} +2 -2
- package/dist/{benchmark-command-BCafwNrf.js.map → benchmark-command-Dtym4pA-.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-CDSolHq7.js → benchmarks-Ce4Ae5fX.js} +3 -3
- package/dist/{benchmarks-CDSolHq7.js.map → benchmarks-Ce4Ae5fX.js.map} +1 -1
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +2 -2
- package/dist/{campaign-Tdy3h62h.js → campaign-FG3thH2m.js} +4 -4
- package/dist/{campaign-Tdy3h62h.js.map → campaign-FG3thH2m.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-DjXROWpx.d.ts → client-C9gzZE59.d.ts} +4 -4
- package/dist/{client-DjXROWpx.d.ts.map → client-C9gzZE59.d.ts.map} +1 -1
- package/dist/{completion-verifier-foUCLif_.d.ts → completion-verifier-DJA5BhPb.d.ts} +3 -3
- package/dist/{completion-verifier-foUCLif_.d.ts.map → completion-verifier-DJA5BhPb.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +4 -4
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/control.js +1 -1
- package/dist/{default-registry-BZhStdjl.d.ts → default-registry-Bf8Woqmq.d.ts} +6 -6
- package/dist/{default-registry-BZhStdjl.d.ts.map → default-registry-Bf8Woqmq.d.ts.map} +1 -1
- package/dist/{engine-nB64f48I.d.ts → engine-3hL-XqwJ.d.ts} +7 -7
- package/dist/{engine-nB64f48I.d.ts.map → engine-3hL-XqwJ.d.ts.map} +1 -1
- package/dist/{eval-campaign-DNjCvAm-.js → eval-campaign-B_7wcnav.js} +2 -2
- package/dist/{eval-campaign-DNjCvAm-.js.map → eval-campaign-B_7wcnav.js.map} +1 -1
- package/dist/{exact-types-Djvzosly.d.ts → exact-types-DSFFpLLI.d.ts} +2 -2
- package/dist/{exact-types-Djvzosly.d.ts.map → exact-types-DSFFpLLI.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/{feedback-trajectory-Rh280oXo.d.ts → feedback-trajectory-CacHpxsp.d.ts} +3 -3
- package/dist/{feedback-trajectory-Rh280oXo.d.ts.map → feedback-trajectory-CacHpxsp.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-CvXXlyz7.d.ts → index-BnEuDAK2.d.ts} +2 -2
- package/dist/{index-CvXXlyz7.d.ts.map → index-BnEuDAK2.d.ts.map} +1 -1
- package/dist/{index-Sh2I0DRc.d.ts → index-CBEMKO8d.d.ts} +9 -9
- package/dist/{index-Sh2I0DRc.d.ts.map → index-CBEMKO8d.d.ts.map} +1 -1
- package/dist/{index-C5HOo4ZF2.d.ts → index-CLKMJTJF2.d.ts} +5 -5
- package/dist/{index-C5HOo4ZF2.d.ts.map → index-CLKMJTJF2.d.ts.map} +1 -1
- package/dist/{index-CwDrUMe0.d.ts → index-CQTZ-4XN.d.ts} +2 -2
- package/dist/{index-CwDrUMe0.d.ts.map → index-CQTZ-4XN.d.ts.map} +1 -1
- package/dist/index.d.ts +22 -22
- package/dist/index.js +11 -11
- package/dist/{insight-report-C6h6F_4L.d.ts → insight-report-CgX_s0Ez.d.ts} +3 -3
- package/dist/{insight-report-C6h6F_4L.d.ts.map → insight-report-CgX_s0Ez.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/{mint-CGEkzPLf.js → mint-B2O60ACG.js} +2 -2
- package/dist/{mint-CGEkzPLf.js.map → mint-B2O60ACG.js.map} +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{promotion-policy-ChWhTDBH.d.ts → promotion-policy-Ckjhzg_4.d.ts} +2 -2
- package/dist/{promotion-policy-ChWhTDBH.d.ts.map → promotion-policy-Ckjhzg_4.d.ts.map} +1 -1
- package/dist/{propose-review-control-qgWLA6E9.js → propose-review-control-dSNPjFUH.js} +2 -2
- package/dist/{propose-review-control-qgWLA6E9.js.map → propose-review-control-dSNPjFUH.js.map} +1 -1
- package/dist/{release-report-Dy39cbFF.js → release-report-BUYmoKo2.js} +2 -2
- package/dist/{release-report-Dy39cbFF.js.map → release-report-BUYmoKo2.js.map} +1 -1
- package/dist/{release-report-CI8uisI1.d.ts → release-report-DA2BCu5a.d.ts} +3 -3
- package/dist/{release-report-CI8uisI1.d.ts.map → release-report-DA2BCu5a.d.ts.map} +1 -1
- package/dist/{replay-DyBLaKFc.js → replay-B3gACG_H.js} +2 -2
- package/dist/{replay-DyBLaKFc.js.map → replay-B3gACG_H.js.map} +1 -1
- package/dist/{replay-DFf-teiC.d.ts → replay-DbhZ4Ked.d.ts} +5 -5
- package/dist/{replay-DFf-teiC.d.ts.map → replay-DbhZ4Ked.d.ts.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +1 -1
- package/dist/{researcher-BoaxeCzP.d.ts → researcher-BchpD55R.d.ts} +3 -3
- package/dist/{researcher-BoaxeCzP.d.ts.map → researcher-BchpD55R.d.ts.map} +1 -1
- package/dist/{reward-hacking-B2tPL9a3.js → reward-hacking-BDToousL.js} +4 -3
- package/dist/reward-hacking-BDToousL.js.map +1 -0
- package/dist/{reward-hacking-Cf1PtEOz.d.ts → reward-hacking-BI0OMAlo.d.ts} +2 -2
- package/dist/{reward-hacking-Cf1PtEOz.d.ts.map → reward-hacking-BI0OMAlo.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +4 -4
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-2ECTXb2N.js → rollout-DRrksrcV.js} +2 -2
- package/dist/{rollout-2ECTXb2N.js.map → rollout-DRrksrcV.js.map} +1 -1
- package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts → rubric-predictive-validity-DvkjPYCe.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts.map → rubric-predictive-validity-DvkjPYCe.d.ts.map} +1 -1
- package/dist/{run-evidence-BDFFai9R.d.ts → run-evidence-Dhi3C81V.d.ts} +3 -3
- package/dist/{run-evidence-BDFFai9R.d.ts.map → run-evidence-Dhi3C81V.d.ts.map} +1 -1
- package/dist/{run-record-DqOw5X6_.js → run-record-BmSPWXJR.js} +2 -1
- package/dist/run-record-BmSPWXJR.js.map +1 -0
- package/dist/{run-record-DdSa93_W.d.ts → run-record-CF4Dwpxr.d.ts} +4 -1
- package/dist/run-record-CF4Dwpxr.d.ts.map +1 -0
- package/dist/{skill-usage-CJlWEUFt.d.ts → skill-usage-CO9OLRBx.d.ts} +6 -6
- package/dist/{skill-usage-CJlWEUFt.d.ts.map → skill-usage-CO9OLRBx.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts → skillopt-optimization-method-D8wysvU1.d.ts} +6 -6
- package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts.map → skillopt-optimization-method-D8wysvU1.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-CQwZ-ZX8.js → skillopt-optimization-method-Dtzp7QpP.js} +14 -4
- package/dist/skillopt-optimization-method-Dtzp7QpP.js.map +1 -0
- package/dist/{statistical-heldout-Dn9ruizm.d.ts → statistical-heldout-TQ-4CYiN.d.ts} +3 -3
- package/dist/{statistical-heldout-Dn9ruizm.d.ts.map → statistical-heldout-TQ-4CYiN.d.ts.map} +1 -1
- package/dist/{summary-report-DuUS_i7W.d.ts → summary-report-BNR7DWTj.d.ts} +2 -2
- package/dist/{summary-report-DuUS_i7W.d.ts.map → summary-report-BNR7DWTj.d.ts.map} +1 -1
- package/dist/{tool-groups-DIVBnJyl.d.ts → tool-groups-CZPGGlHf.d.ts} +3 -3
- package/dist/tool-groups-CZPGGlHf.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +4 -4
- package/dist/traces.js +1 -1
- package/dist/{types-DYuNHo9R.d.ts → types-BnjdJ70P.d.ts} +3 -3
- package/dist/{types-DYuNHo9R.d.ts.map → types-BnjdJ70P.d.ts.map} +1 -1
- package/dist/{types-DF_Udrp-.d.ts → types-BvDKaULh.d.ts} +2 -2
- package/dist/{types-DF_Udrp-.d.ts.map → types-BvDKaULh.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/campaign-proposers.md +5 -0
- package/docs/concepts.md +1 -0
- package/package.json +1 -1
- package/dist/reward-hacking-B2tPL9a3.js.map +0 -1
- package/dist/run-record-DdSa93_W.d.ts.map +0 -1
- package/dist/run-record-DqOw5X6_.js.map +0 -1
- package/dist/skillopt-optimization-method-CQwZ-ZX8.js.map +0 -1
- package/dist/tool-groups-DIVBnJyl.d.ts.map +0 -1
package/dist/control.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { $ as ControlRunResult, J as ControlActionOutcome, Q as ControlEvalResult, X as ControlContext, Y as ControlBudget, Z as ControlDecision, at as StopDecision, ct as runAgentControlLoop, dt as subjectiveEval, et as ControlRuntimeConfig, it as ControlStopPolicies, lt as stopOnNoProgress, nt as ControlSeverity, ot as allCriticalPassed, q as ControlActionFailureMode, rt as ControlStep, st as objectiveEval, tt as ControlRuntimeError, ut as stopOnRepeatedAction } from "./feedback-trajectory-
|
|
2
|
-
import { A as ActionExecutionPolicy, M as evaluateActionPolicy, _ as ProposeReviewReport, a as ProposeReviewControlAction, c as ProposeReviewControlState, g as ProposeReviewConfig, i as scoreFromEvals, j as ActionPolicyDecision, k as runProposeReview, n as RunEvidenceMetadata, o as ProposeReviewControlConfig, r as controlRunToRunRecord, s as ProposeReviewControlResult, t as ControlRunToRunRecordOptions, u as runProposeReviewAsControlLoop } from "./run-evidence-
|
|
1
|
+
import { $ as ControlRunResult, J as ControlActionOutcome, Q as ControlEvalResult, X as ControlContext, Y as ControlBudget, Z as ControlDecision, at as StopDecision, ct as runAgentControlLoop, dt as subjectiveEval, et as ControlRuntimeConfig, it as ControlStopPolicies, lt as stopOnNoProgress, nt as ControlSeverity, ot as allCriticalPassed, q as ControlActionFailureMode, rt as ControlStep, st as objectiveEval, tt as ControlRuntimeError, ut as stopOnRepeatedAction } from "./feedback-trajectory-CacHpxsp.js";
|
|
2
|
+
import { A as ActionExecutionPolicy, M as evaluateActionPolicy, _ as ProposeReviewReport, a as ProposeReviewControlAction, c as ProposeReviewControlState, g as ProposeReviewConfig, i as scoreFromEvals, j as ActionPolicyDecision, k as runProposeReview, n as RunEvidenceMetadata, o as ProposeReviewControlConfig, r as controlRunToRunRecord, s as ProposeReviewControlResult, t as ControlRunToRunRecordOptions, u as runProposeReviewAsControlLoop } from "./run-evidence-Dhi3C81V.js";
|
|
3
3
|
export { type ActionExecutionPolicy, type ActionPolicyDecision, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRunToRunRecordOptions, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type RunEvidenceMetadata, type StopDecision, allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };
|
package/dist/control.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, s as controlRunToRunRecord, u as objectiveEval, y as evaluateActionPolicy } from "./propose-review-control-
|
|
1
|
+
import { c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, s as controlRunToRunRecord, u as objectiveEval, y as evaluateActionPolicy } from "./propose-review-control-dSNPjFUH.js";
|
|
2
2
|
export { allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { I as TraceAnalystSpan, i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-
|
|
2
|
-
import { l as ExactCapableAnalyst } from "./exact-types-
|
|
3
|
-
import { D as AnalystRegistryOptions, E as AnalystRegistry } from "./completion-verifier-
|
|
4
|
-
import { a as TraceAnalystLimits, d as RawAnalystFinding, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-
|
|
5
|
-
import { t as TraceToolGroupName } from "./tool-groups-
|
|
1
|
+
import { I as TraceAnalystSpan, i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-BvDKaULh.js";
|
|
2
|
+
import { l as ExactCapableAnalyst } from "./exact-types-DSFFpLLI.js";
|
|
3
|
+
import { D as AnalystRegistryOptions, E as AnalystRegistry } from "./completion-verifier-DJA5BhPb.js";
|
|
4
|
+
import { a as TraceAnalystLimits, d as RawAnalystFinding, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-3hL-XqwJ.js";
|
|
5
|
+
import { t as TraceToolGroupName } from "./tool-groups-CZPGGlHf.js";
|
|
6
6
|
//#region src/trace-analyst/behavioral-metrics.d.ts
|
|
7
7
|
type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
|
|
8
8
|
interface SuboptimalSignal {
|
|
@@ -116,4 +116,4 @@ interface DefaultAnalystRegistryOptions {
|
|
|
116
116
|
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
117
117
|
//#endregion
|
|
118
118
|
export { createTraceAnalyst as a, runTraceAnalyst as c, deriveEfficiencyFindings as d, BehavioralMetrics as f, computeTraceMetrics as g, SuboptimalSignal as h, TraceAnalystDefinition as i, BehavioralAnalystOptions as l, SuboptimalCode as m, buildDefaultAnalystRegistry as n, renderPriorFindings as o, BehavioralTokenSequence as p, CreateTraceAnalystOptions as r, renderUpstreamFindings as s, DefaultAnalystRegistryOptions as t, behavioralAnalyst as u };
|
|
119
|
-
//# sourceMappingURL=default-registry-
|
|
119
|
+
//# sourceMappingURL=default-registry-Bf8Woqmq.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-
|
|
1
|
+
{"version":3,"file":"default-registry-Bf8Woqmq.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;;;iBAoFT,oBAAoB,gBAAgB,qBAAqB;;;UC9GxD;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;iBAe3B,uBAAuB,UAAU;;;UCnahC;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "./cost-ledger-Bv_e8XHY.js";
|
|
2
|
-
import { a as RunRecord } from "./run-record-
|
|
3
|
-
import { S as BoundedTraceAnalysisStoreOptions, p as EvidenceRef, w as TraceAnalysisStore } from "./types-
|
|
4
|
-
import { E as AnalystRegistry, r as CorrectnessChecker } from "./completion-verifier-
|
|
5
|
-
import { S as JudgeConfig } from "./types-
|
|
2
|
+
import { a as RunRecord } from "./run-record-CF4Dwpxr.js";
|
|
3
|
+
import { S as BoundedTraceAnalysisStoreOptions, p as EvidenceRef, w as TraceAnalysisStore } from "./types-BvDKaULh.js";
|
|
4
|
+
import { E as AnalystRegistry, r as CorrectnessChecker } from "./completion-verifier-DJA5BhPb.js";
|
|
5
|
+
import { S as JudgeConfig } from "./types-BnjdJ70P.js";
|
|
6
6
|
import { a as DatasetScenario } from "./dataset-C8xaLXdY.js";
|
|
7
|
-
import "./summary-report-
|
|
8
|
-
import { o as InsightReport, r as ExecutionInsight, t as CostProvenanceSummary } from "./insight-report-
|
|
7
|
+
import "./summary-report-BNR7DWTj.js";
|
|
8
|
+
import { o as InsightReport, r as ExecutionInsight, t as CostProvenanceSummary } from "./insight-report-CgX_s0Ez.js";
|
|
9
9
|
import { z } from "zod";
|
|
10
10
|
//#region src/contract/analyze-runs.d.ts
|
|
11
11
|
interface AnalyzeRunsOptions {
|
|
@@ -242,4 +242,4 @@ interface TraceAnalysisEngine {
|
|
|
242
242
|
declare function resolveTraceAnalystLimits(limits: Partial<TraceAnalystLimits> | undefined): TraceAnalystLimits;
|
|
243
243
|
//#endregion
|
|
244
244
|
export { toOpenAiTool as C, analyzeRuns as D, SummarizeExecutionOptions as E, summarizeExecution as O, makeEvalTools as S, ExecutionReport as T, TraceAnalysisToolDescriptor as _, TraceAnalystLimits as a, EvalToolDef as b, RAW_FINDING_SCHEMA_PROMPT as c, RawAnalystFinding as d, RawAnalystFindingSchema as f, TRACE_ANALYST_TOOL_NAMESPACE as g, BuildTraceAnalysisToolsOptions as h, TraceAnalysisEngineResult as i, RawAnalystEvidence as l, parseRawFinding as m, TraceAnalysisEngine as n, resolveTraceAnalystLimits as o, evidenceRefsFromRawFinding as p, TraceAnalysisEngineRequest as r, ANALYST_SEVERITIES as s, DEFAULT_TRACE_ANALYST_LIMITS as t, RawAnalystEvidenceSchema as u, buildTraceAnalysisToolDescriptors as v, AnalyzeRunsOptions as w, MakeEvalToolsConfig as x, traceAnalystFunctionGroup as y };
|
|
245
|
-
//# sourceMappingURL=engine-
|
|
245
|
+
//# sourceMappingURL=engine-3hL-XqwJ.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine-
|
|
1
|
+
{"version":3,"file":"engine-3hL-XqwJ.d.ts","names":[],"sources":["../src/contract/analyze-runs.ts","../src/eval-tools.ts","../src/trace-analyst/tools.ts","../src/analyst/finding-signature.ts","../src/analyst/engine.ts"],"mappings":";;;;;;;;;;UAoEiB;;EAEf,MAAM;;;EAGN;;;;;EAKA;EACA;;;EAGA,kBAAkB;;;EAGlB,UAAU;;;;EAIV;IACE;IACA,cAAc;;;;;EAKhB,cAAc;IAAQ;IAAe;IAAe;;;EAEpD;;;;EAIA;;;;;;;;;EASA,eAAe;;;EAGf;;UAGe;EACf,MAAM;EACN;;UAGe;EACf,WAAW;EACX,gBAAgB;;;iBAIF,mBAAmB,MAAM,4BAA4B;iBAgB/C,YAAY,MAAM,qBAAqB,QAAQ;;;;UCpHpD;EACf;EACA;EACA,YAAY;EACZ,UAAU,eAAe;IAAQ,SAAS;QAAkB;;UAG7C;;EAEf,SAAS,MAAM;;;;EAIf;IAAe,kBAAkB;;;;EAGjC,UAAU,KAAK;;;iBAID,aAAa,KAAK;EAChC;EACA;IAAY;IAAc;IAAqB,YAAY;;;;;;;;iBAa7C,cAAc,KAAK,sBAAsB;;;cCjD5C;UAEI,uCAAuC;EACtD,OAAO;;UAGQ,oCAAoC;EACnD,kBAAkB;;;iBAIJ,kCACd,SAAS,iCACR;iBAkHa,0BAA0B,SAAS;EACjD;EACA;EACA;EACA;EACA,WAAW;;;;cCtIA;cAEA,0BAAwB,EAAA;;;GAK1B,EAAA,KAAA;KAEC,qBAAqB,EAAE,aAAa;cAiBnC,yBAAuB,EAAA;;;;;;;;;;;;;;;;;GAKzB,EAAA,KAAA;KAEC,oBAAoB,EAAE,aAAa;;;;;;cAOlC;;iBAYG,2BAA2B,SAAS,oBAAoB;iBAQxD,gBACd,cACA,OAAO,aAAa,SAAS,mCAC5B;;;;UCtEc;;EAEf;;EAEA;;EAEA;;EAEA;;cAGW,8BAA8B,SAAS;UAOnC;EACf;EACA;EACA;EACA,gBAAgB;;;;;;;;;;;EAWhB,aAAa,SAAS;EACtB,QAAQ;EACR,YAAY;EACZ;EACA,WAAW;EACX,SAAS;EACT,OAAO,iBAAiB,SAAS;;;UAIlB;EACf;EACA,UAAU;;EAEV;;;;;EAKA;;EAEA;EACA,SAAS;;;;;;;;;UAUM;WACN;WACA;WACA;;WAEA;;;;;;;;WAQA,iBAAiB,SAAS;EACnC,QAAQ,SAAS,6BAA6B,QAAQ;;iBAGxC,0BACd,QAAQ,QAAQ,kCACf"}
|
|
@@ -3,7 +3,7 @@ import { buildAgentProfileCell, verifyAgentProfileCell } from "./profile-cell.js
|
|
|
3
3
|
import { a as assertLlmRoute } from "./llm-client-DzvMUsS_.js";
|
|
4
4
|
import { t as FileSystemRawProviderSink } from "./raw-provider-sink-BQd7mzyT.js";
|
|
5
5
|
import { t as TraceEmitter } from "./emitter-CPBAhxum.js";
|
|
6
|
-
import { s as validateRunRecord } from "./run-record-
|
|
6
|
+
import { s as validateRunRecord } from "./run-record-BmSPWXJR.js";
|
|
7
7
|
import { i as researchReport } from "./summary-report-Lf-5I7xh.js";
|
|
8
8
|
import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-MLzHOfV9.js";
|
|
9
9
|
//#region src/eval-campaign.ts
|
|
@@ -347,4 +347,4 @@ function defaultRunId(params) {
|
|
|
347
347
|
//#endregion
|
|
348
348
|
export { runEvalCampaign as t };
|
|
349
349
|
|
|
350
|
-
//# sourceMappingURL=eval-campaign-
|
|
350
|
+
//# sourceMappingURL=eval-campaign-B_7wcnav.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-campaign-DNjCvAm-.js","names":[],"sources":["../src/eval-campaign.ts"],"sourcesContent":["/**\n * EvalCampaign — opinionated matrix runner that wires the four\n * capture-integrity directives by construction.\n *\n * The canonical benchmark shape — matrix runner → for each\n * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the\n * run → analyze — has a bug class at the integration boundary: raw\n * events not captured, route silently wrong, integrity not asserted,\n * analyst never run. The directives in `SKILL.md § Capture integrity`\n * are the mitigations.\n *\n * `EvalCampaign` is the structural fix — consumers don't wire the\n * integrity surface themselves; the campaign owns it. Specifically:\n *\n * - calls `assertLlmRoute` once at preflight before any work runs\n * - constructs a per-run `TraceStore` and `RawProviderSink` via factories\n * - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`\n * - hands the runner an `LlmClientOptions` pre-wired with the sink and\n * trace context — the runner can't accidentally call an LLM without\n * capturing the raw HTTP envelope\n * - calls `assertRunCaptured` after every `endRun` and routes failures\n * through a configurable policy (`throw` / `mark_failed` / `log`)\n * - assembles per-run `RunRecord`s and runs `researchReport` at the end\n * so the campaign artifact is launch-decision-grade by default\n * - embeds the campaign fingerprint (a SHA-256 over the canonicalised\n * run set) and optional `preregistrationHash` in the report\n *\n * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`\n * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped\n * lives in the campaign. This is the inversion-of-control point — consumers\n * stop writing matrix runners and start writing scenario-runners.\n *\n * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):\n *\n * - Distributed/cluster execution (concurrency is local async)\n * - Adaptive sampling / sequential interim looks\n * - Resume from partial state across crashes\n * - LLM-call retry beyond what `LlmClient` already does\n */\n\nimport {\n type AgentProfileCell,\n type AgentProfileCellInput,\n buildAgentProfileCell,\n verifyAgentProfileCell,\n} from './agent-profile-cell'\nimport { assertLlmRoute, type LlmClientOptions, type LlmRouteRequirements } from './llm-client'\nimport { canonicalize, hashJson } from './pre-registration'\nimport type {\n JudgeScoresRecord,\n RunCostProvenance,\n RunJudgeMetadata,\n RunOutcome,\n RunRecord,\n RunSplitTag,\n RunTaskFailure,\n RunTokenUsage,\n} from './run-record'\nimport { validateRunRecord } from './run-record'\nimport { type ResearchReport, type ResearchReportOptions, researchReport } from './summary-report'\nimport type { RunCompleteHook } from './trace/emitter'\nimport { TraceEmitter } from './trace/emitter'\nimport {\n assertRunCaptured,\n RunIntegrityError,\n type RunIntegrityExpectations,\n type RunIntegrityReport,\n} from './trace/integrity'\nimport { FileSystemRawProviderSink, type RawProviderSink } from './trace/raw-provider-sink'\nimport type { TraceStore } from './trace/store'\n\n// ── Public types ─────────────────────────────────────────────────────────\n\nexport interface CampaignVariant<V> {\n id: string\n payload: V\n}\n\nexport interface CampaignScenario {\n scenarioId: string\n /** Free-form metadata propagated to runs and reports. */\n tags?: Record<string, string>\n}\n\nexport interface CampaignRunContext<V> {\n /** Stable run id. The campaign generates this; the runner does not. */\n runId: string\n /** Logical experiment id (campaignId by default; overridable per-run via opts). */\n experimentId: string\n variant: V\n variantId: string\n scenarioId: string\n scenarioTags: Record<string, string>\n seed: number\n splitTag: RunSplitTag\n /**\n * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired\n * (analyst auto-execution if configured, plus integrity check). The\n * runner MUST call `emitter.startRun` before doing any work and either\n * `emitter.endRun` or `emitter.abortRun` before returning.\n */\n emitter: TraceEmitter\n store: TraceStore\n rawSink: RawProviderSink\n /**\n * Pre-wired LLM client options — `rawSink` and `traceContext` are populated\n * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The\n * runner can spread additional fields if needed.\n */\n llmOpts: LlmClientOptions\n}\n\ninterface CampaignRunOutcomeFields {\n /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */\n pass: boolean\n /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */\n score: number\n /** Cost in USD, or null when the runner could not capture it. */\n costUsd: number | null\n /** Source of the cost amount. */\n costProvenance: RunCostProvenance\n tokenUsage: RunTokenUsage\n /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */\n model: string\n /** sha256 of the effective prompt sent to the model. */\n promptHash: string\n /** sha256 of the effective config (model, temperature, tools, judges, splits). */\n configHash: string\n /** Optional extra numeric metrics to land in `outcome.raw`. */\n raw?: Record<string, number>\n /** Optional judge metadata when a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /**\n * Optional per-judge / per-dim breakdown for ensemble-judged runs.\n * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.\n * Single-judge or scalar-only runs leave this unset.\n */\n judgeScores?: JudgeScoresRecord\n /**\n * Agent profile cell observed by the runner. When supplied, it overrides\n * `EvalCampaignOptions.agentProfile` for this run and must match the\n * outcome's `model` and `promptHash`.\n */\n agentProfile?: AgentProfileCell | AgentProfileCellInput\n}\n\n/** Campaign result with the same task-failure invariant as `RunRecord`. */\nexport type CampaignRunOutcome = CampaignRunOutcomeFields & RunTaskFailure\n\nexport type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>\n\nexport type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log'\n\nexport interface EvalCampaignOptions<V> {\n /**\n * Stable id for the campaign. Used as the default `experimentId` on\n * every run, and folded into the campaign fingerprint.\n */\n campaignId: string\n variants: CampaignVariant<V>[]\n scenarios: CampaignScenario[]\n /** Default `[0, 1, 2]`. */\n seeds?: number[]\n /** Default `'holdout'` — the split that anchors a launch decision. */\n splitTag?: RunSplitTag\n /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */\n commitSha: string\n /**\n * LLM client config. Augmented per-run with `rawSink` and `traceContext`\n * before being passed to the runner. The campaign asserts this config\n * matches `routeRequirements` once at preflight.\n */\n llmOpts: LlmClientOptions\n /**\n * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail\n * loud if the campaign would silently fall back to the public router or\n * run unauthenticated. Override with an empty object to disable.\n */\n routeRequirements?: LlmRouteRequirements\n /**\n * Per-run TraceStore factory. Common shape: a fresh store per run keyed\n * on `runId`. Implementations that share a store across the campaign\n * are valid — the campaign only writes through `emitter`.\n */\n storeFactory: (params: CampaignFactoryParams) => TraceStore\n /**\n * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`\n * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;\n * otherwise required. Forensic capture is non-negotiable in a campaign\n * run — pass `NoopRawProviderSink` explicitly if you want to opt out.\n */\n rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink\n /**\n * Filesystem root for default `rawSinkFactory`. Ignored if\n * `rawSinkFactory` is supplied.\n */\n workDir?: string\n /**\n * Extra `onRunComplete` hooks the campaign appends (after its own\n * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.\n */\n onRunComplete?: RunCompleteHook[]\n /**\n * Per-run integrity expectations. Defaults to:\n * `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.\n * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.\n */\n integrity?: RunIntegrityExpectations\n /** Behaviour when integrity fails. Default `'mark_failed'`. */\n onIntegrityFailure?: CampaignIntegrityPolicy\n /**\n * Per-run runner. Receives a fully-wired context; produces an outcome\n * the campaign converts into a `RunRecord`.\n */\n runner: CampaignRunner<V>\n /**\n * If set, the campaign computes `researchReport` at the end. `comparator`\n * is a `variantId`. Other fields are forwarded verbatim.\n */\n report?: { comparator?: string } & Omit<\n ResearchReportOptions,\n 'comparator' | 'preregistrationHash' | 'generatedAt'\n >\n /**\n * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).\n * Embedded in the campaign fingerprint and the research report.\n */\n preregistrationHash?: string\n /** Local concurrency. Default `1` (sequential). */\n concurrency?: number\n /**\n * Override the time source. Tests pass a mock to make wallMs deterministic.\n */\n now?: () => number\n /** Override the runId generator. Tests pin this. */\n runId?: (params: CampaignFactoryParams) => string\n /**\n * Agent profile cell for campaign runs. Static profiles can pass an object;\n * routers or variant-specific harnesses can pass a factory. The campaign\n * stamps the built cell onto every `RunRecord` and rejects profile/model or\n * profile/prompt contradictions.\n */\n agentProfile?:\n | AgentProfileCell\n | AgentProfileCellInput\n | ((\n params: CampaignFactoryParams & {\n variant: V\n scenarioTags: Record<string, string>\n },\n ) =>\n | AgentProfileCell\n | AgentProfileCellInput\n | Promise<AgentProfileCell | AgentProfileCellInput>)\n}\n\nexport interface CampaignFactoryParams {\n campaignId: string\n runId: string\n variantId: string\n scenarioId: string\n seed: number\n}\n\nexport interface FailedRun {\n runId: string\n variantId: string\n scenarioId: string\n seed: number\n reason: string\n error?: string\n}\n\nexport interface EvalCampaignResult {\n campaignId: string\n /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */\n campaignFingerprint: string\n preregistrationHash: string | null\n /** Successful runs only. Failed runs land in `failedRuns`. */\n runs: RunRecord[]\n /** Integrity reports for every successful run. */\n integrityReports: RunIntegrityReport[]\n failedRuns: FailedRun[]\n /** Computed when `report` is set on options. */\n report?: ResearchReport\n startedAt: string\n endedAt: string\n}\n\n// ── Implementation ───────────────────────────────────────────────────────\n\nconst DEFAULT_INTEGRITY: RunIntegrityExpectations = {\n llmSpansMin: 1,\n requireRawCoverageOfLlmSpans: true,\n requireOutcome: true,\n}\n\nconst DEFAULT_ROUTE: LlmRouteRequirements = {\n requireExplicitBaseUrl: true,\n requireAuth: true,\n}\n\nexport async function runEvalCampaign<V>(\n opts: EvalCampaignOptions<V>,\n): Promise<EvalCampaignResult> {\n // ── Preflight ──────────────────────────────────────────────────────\n assertLlmRoute(opts.llmOpts, opts.routeRequirements ?? DEFAULT_ROUTE)\n\n if (opts.variants.length === 0) {\n throw new Error('runEvalCampaign: variants must be non-empty.')\n }\n if (opts.scenarios.length === 0) {\n throw new Error('runEvalCampaign: scenarios must be non-empty.')\n }\n const variantIds = new Set<string>()\n for (const v of opts.variants) {\n if (variantIds.has(v.id)) {\n throw new Error(`runEvalCampaign: duplicate variant id \"${v.id}\".`)\n }\n variantIds.add(v.id)\n }\n const scenarioIds = new Set<string>()\n for (const s of opts.scenarios) {\n if (scenarioIds.has(s.scenarioId)) {\n throw new Error(`runEvalCampaign: duplicate scenarioId \"${s.scenarioId}\".`)\n }\n scenarioIds.add(s.scenarioId)\n }\n if (opts.report?.comparator && !variantIds.has(opts.report.comparator)) {\n throw new Error(\n `runEvalCampaign: report.comparator \"${opts.report.comparator}\" is not a configured variantId.`,\n )\n }\n if (!opts.commitSha) {\n throw new Error('runEvalCampaign: commitSha is required (every RunRecord needs it).')\n }\n\n const seeds = opts.seeds ?? [0, 1, 2]\n const splitTag: RunSplitTag = opts.splitTag ?? 'holdout'\n const concurrency = Math.max(1, opts.concurrency ?? 1)\n const integrity = { ...DEFAULT_INTEGRITY, ...(opts.integrity ?? {}) }\n const onIntegrityFailure: CampaignIntegrityPolicy = opts.onIntegrityFailure ?? 'mark_failed'\n const now = opts.now ?? (() => Date.now())\n const baseUrl = (opts.llmOpts.baseUrl ?? '').replace(/\\/+$/, '')\n const provider = opts.llmOpts.provider ?? null\n const preregistrationHash = opts.preregistrationHash ?? null\n\n const rawSinkFactory = opts.rawSinkFactory ?? defaultRawSinkFactory(opts.workDir)\n\n // ── Fingerprint ────────────────────────────────────────────────────\n const campaignFingerprint = await hashJson(\n canonicalize({\n campaignId: opts.campaignId,\n variants: opts.variants.map((v) => v.id).sort(),\n scenarios: opts.scenarios.map((s) => s.scenarioId).sort(),\n seeds: [...seeds].sort((a, b) => a - b),\n splitTag,\n comparator: opts.report?.comparator ?? null,\n baseUrl,\n provider,\n preregistrationHash,\n }),\n )\n\n // ── Plan the matrix ────────────────────────────────────────────────\n type Cell = { variant: CampaignVariant<V>; scenario: CampaignScenario; seed: number }\n const cells: Cell[] = []\n for (const variant of opts.variants) {\n for (const scenario of opts.scenarios) {\n for (const seed of seeds) {\n cells.push({ variant, scenario, seed })\n }\n }\n }\n\n const startedAt = new Date(now()).toISOString()\n const runs: RunRecord[] = []\n const integrityReports: RunIntegrityReport[] = []\n const failedRuns: FailedRun[] = []\n\n // ── Execute (bounded-concurrency worker pool) ──────────────────────\n // A genuine (non-CellExecutionError) error from any worker is a bug, not a\n // run-level failure. We must NOT let `Promise.all` reject mid-flight and\n // orphan the other workers' in-progress runs (emitters never finalized,\n // sinks/handles leak, partial work silently discarded). Instead: capture the\n // first genuine error, stop dispatching new cells so in-flight workers wind\n // down, finalize every still-open run, then re-throw the aggregated error.\n let cursor = 0\n let aborting = false\n const genuineErrors: unknown[] = []\n // Emitters for runs that are currently mid-flight, keyed by runId. A worker\n // registers its emitter before invoking the runner and removes it once the\n // run is finalized (via endRun/abortRun, success or failure). Anything left\n // here after the pool settles is an orphan we must abort.\n const openRuns = new Map<string, TraceEmitter>()\n\n async function worker(): Promise<void> {\n while (!aborting) {\n const i = cursor++\n if (i >= cells.length) return\n const cell = cells[i]!\n try {\n const result = await runOneCell(cell)\n runs.push(result.record)\n integrityReports.push(result.integrity)\n } catch (err) {\n if (err instanceof CellExecutionError) {\n failedRuns.push(err.failed)\n if (err.integrity) integrityReports.push(err.integrity)\n } else {\n // Genuine bug — not a runner failure, not an integrity failure.\n // Capture it and stop dispatching so peers wind down gracefully;\n // re-thrown after the pool settles. Do not surface here — that would\n // reject Promise.all and orphan the other workers' runs.\n genuineErrors.push(err)\n aborting = true\n return\n }\n }\n }\n }\n\n async function runOneCell(\n cell: Cell,\n ): Promise<{ record: RunRecord; integrity: RunIntegrityReport }> {\n const runId = (opts.runId ?? defaultRunId)({\n campaignId: opts.campaignId,\n runId: '', // unused by default generator\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n })\n const factoryParams: CampaignFactoryParams = {\n campaignId: opts.campaignId,\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n }\n const store = opts.storeFactory(factoryParams)\n const rawSink = rawSinkFactory(factoryParams)\n\n const emitter = new TraceEmitter(store, {\n runId,\n now: opts.now,\n onRunComplete: opts.onRunComplete,\n })\n // Track this run as open so a genuine error elsewhere in the pool can\n // finalize it instead of orphaning it. Removed in the finally below.\n openRuns.set(runId, emitter)\n\n const llmOpts: LlmClientOptions = {\n ...opts.llmOpts,\n rawSink,\n traceContext: { runId },\n }\n\n const ctx: CampaignRunContext<V> = {\n runId,\n experimentId: opts.campaignId,\n variant: cell.variant.payload,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n scenarioTags: cell.scenario.tags ?? {},\n seed: cell.seed,\n splitTag,\n emitter,\n store,\n rawSink,\n llmOpts,\n }\n\n try {\n const wallStart = now()\n let outcome: CampaignRunOutcome\n try {\n outcome = await opts.runner(ctx)\n } catch (err) {\n const message = err instanceof Error ? err.message : String(err)\n // The runner threw mid-execution. Abort the run so the emitter\n // finalizes. The only benign abortRun failure is \"the runner never\n // started the run\" (nothing to finalize). A store-write failure (disk\n // full, FS error) is a genuine diagnostic — surface it rather than\n // masking it as a plain runner failure.\n await finalizeAbort(emitter, runId, message)\n throw new CellExecutionError({\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n reason: 'runner_threw',\n error: message,\n })\n }\n const wallMs = now() - wallStart\n\n const integrityReport = await assertRunCaptured(store, runId, { ...integrity, rawSink })\n if (!integrityReport.ok) {\n switch (onIntegrityFailure) {\n case 'throw':\n throw new RunIntegrityError(integrityReport)\n case 'mark_failed':\n throw new CellExecutionError(\n {\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n reason: 'integrity_failed',\n error: integrityReport.issues.map((i) => i.code).join(', '),\n },\n integrityReport,\n )\n case 'log':\n // Caller wants the run admitted with a flagged report; fall through.\n break\n }\n }\n\n const recordOutcome: RunOutcome = {\n raw: outcome.raw ?? {},\n }\n if (splitTag === 'holdout') recordOutcome.holdoutScore = outcome.score\n else recordOutcome.searchScore = outcome.score\n if (outcome.judgeScores !== undefined) recordOutcome.judgeScores = outcome.judgeScores\n\n const record: RunRecord = {\n runId,\n experimentId: opts.campaignId,\n candidateId: cell.variant.id,\n seed: cell.seed,\n model: outcome.model,\n promptHash: outcome.promptHash,\n configHash: outcome.configHash,\n commitSha: opts.commitSha,\n wallMs,\n costUsd: outcome.costUsd,\n costProvenance: outcome.costProvenance,\n tokenUsage: outcome.tokenUsage,\n terminalOutcome: 'succeeded',\n judgeMetadata: outcome.judgeMetadata,\n outcome: recordOutcome,\n ...(outcome.failureClass ? { failureClass: outcome.failureClass } : {}),\n failureMode: outcome.failureMode,\n splitTag,\n scenarioId: cell.scenario.scenarioId,\n }\n const profileSource =\n outcome.agentProfile ??\n (typeof opts.agentProfile === 'function'\n ? await opts.agentProfile({\n campaignId: opts.campaignId,\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n variant: cell.variant.payload,\n scenarioTags: cell.scenario.tags ?? {},\n })\n : opts.agentProfile)\n if (profileSource !== undefined) {\n const agentProfile = await resolveAgentProfileCell(profileSource)\n assertAgentProfileMatchesRun(agentProfile, outcome.model, outcome.promptHash)\n record.agentProfile = agentProfile\n }\n return { record: validateRunRecord(record), integrity: integrityReport }\n } finally {\n // This run's worker has finished with it (success, run-level failure, or\n // genuine error). It is no longer the pool's job to finalize — drop it so\n // the post-settle sweep doesn't double-abort a finalized run.\n openRuns.delete(runId)\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, cells.length) }, () => worker())\n // allSettled (not all): a genuine error in one worker must not reject the\n // pool mid-flight and orphan the others. Each worker captures its own\n // genuine error into `genuineErrors` and returns; we re-throw below.\n await Promise.allSettled(workers)\n\n // Finalize any run still open after the pool wound down. With the\n // stop-dispatch flag these are the runs that were mid-flight in peer workers\n // when the first genuine error fired — abort them so their emitters finalize\n // (hooks fire, store records a terminal status) instead of leaking.\n for (const [runId, emitter] of openRuns) {\n await finalizeAbort(emitter, runId, 'campaign aborted: genuine error in a sibling run')\n }\n openRuns.clear()\n\n if (genuineErrors.length > 0) {\n throw genuineErrors.length === 1\n ? genuineErrors[0]\n : new AggregateError(\n genuineErrors,\n `runEvalCampaign: ${genuineErrors.length} runs failed with genuine (non-run-level) errors`,\n )\n }\n\n // ── Optional research report ───────────────────────────────────────\n let report: ResearchReport | undefined\n if (opts.report) {\n const reportOpts: ResearchReportOptions = {\n ...opts.report,\n comparator: opts.report.comparator,\n split: splitTag === 'dev' ? 'search' : splitTag,\n generatedAt: new Date(now()).toISOString(),\n preregistrationHash: preregistrationHash ?? undefined,\n }\n report = await researchReport(runs, reportOpts)\n }\n\n const endedAt = new Date(now()).toISOString()\n\n return {\n campaignId: opts.campaignId,\n campaignFingerprint,\n preregistrationHash,\n runs,\n integrityReports,\n failedRuns,\n report,\n startedAt,\n endedAt,\n }\n}\n\n// ── Internal ─────────────────────────────────────────────────────────────\n\nclass CellExecutionError extends Error {\n readonly failed: FailedRun\n readonly integrity?: RunIntegrityReport\n constructor(failed: FailedRun, integrity?: RunIntegrityReport) {\n super(`cell ${failed.variantId}/${failed.scenarioId}@${failed.seed} failed: ${failed.reason}`)\n this.failed = failed\n this.integrity = integrity\n }\n}\n\n/**\n * Abort a run whose owning work threw or was orphaned by a sibling's genuine\n * error, finalizing the emitter so hooks fire and the store records a terminal\n * status. Safe to call unconditionally: it only aborts runs that are still\n * `running`. Two no-op cases are intentional and benign:\n *\n * - the run was never started (absent from the store) — nothing to finalize\n * - the run is already terminal (`completed` / `failed` / `aborted`) —\n * finalizing again would overwrite the real outcome (e.g. flip a passed run\n * to `aborted` with `{ pass: false, notes: reason }`), so we leave it alone\n *\n * Any OTHER failure of the store read or the abort write (disk full, FS fault,\n * backend down) is a genuine diagnostic and propagates rather than being\n * swallowed.\n */\nexport async function finalizeAbort(\n emitter: TraceEmitter,\n runId: string,\n reason: string,\n): Promise<void> {\n const existing = await emitter.traceStore.getRun(runId)\n if (existing === undefined) return // run never started; nothing to abort\n if (existing.status !== 'running') return // already finalized; never overwrite a real outcome\n await emitter.abortRun(reason)\n}\n\nfunction defaultRawSinkFactory(workDir: string | undefined) {\n return (params: CampaignFactoryParams): RawProviderSink => {\n if (!workDir) {\n throw new Error(\n 'runEvalCampaign: rawSinkFactory not supplied and workDir not set. Pass either to enable raw provider capture, or pass `new NoopRawProviderSink()` via rawSinkFactory to opt out explicitly.',\n )\n }\n return new FileSystemRawProviderSink({\n dir: `${workDir}/raw-events/${params.runId}`,\n })\n }\n}\n\nasync function resolveAgentProfileCell(\n input: AgentProfileCell | AgentProfileCellInput,\n): Promise<AgentProfileCell> {\n if (isAgentProfileCell(input)) {\n if (!(await verifyAgentProfileCell(input))) {\n throw new Error(`runEvalCampaign: agentProfile.cellId does not match its content`)\n }\n return input\n }\n return buildAgentProfileCell(input)\n}\n\nfunction isAgentProfileCell(\n input: AgentProfileCell | AgentProfileCellInput,\n): input is AgentProfileCell {\n return 'schemaVersion' in input && 'cellId' in input\n}\n\nfunction assertAgentProfileMatchesRun(\n profile: AgentProfileCell,\n model: string,\n promptHash: string,\n): void {\n if (profile.model !== undefined && profile.model !== model) {\n throw new Error(\n `runEvalCampaign: agentProfile.model \"${profile.model}\" does not match outcome.model \"${model}\"`,\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== promptHash) {\n throw new Error(\n `runEvalCampaign: agentProfile.promptHash \"${profile.promptHash}\" does not match outcome.promptHash \"${promptHash}\"`,\n )\n }\n}\n\nfunction defaultRunId(params: CampaignFactoryParams): string {\n // Stable across re-runs: fingerprint of (campaignId, variantId, scenarioId, seed).\n // Caller can override via opts.runId for non-deterministic IDs.\n const base = `${params.campaignId}::${params.variantId}::${params.scenarioId}::${params.seed}`\n // Lightweight hex: we don't need crypto-grade here, just stability + uniqueness.\n let h1 = 0x811c9dc5\n let h2 = 0x12345678\n for (let i = 0; i < base.length; i++) {\n const c = base.charCodeAt(i)\n h1 = Math.imul(h1 ^ c, 0x01000193) >>> 0\n h2 = Math.imul(h2 ^ c, 0x9e3779b1) >>> 0\n }\n return `run-${h1.toString(16).padStart(8, '0')}${h2.toString(16).padStart(8, '0')}`\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmSA,MAAM,oBAA8C;CAClD,aAAa;CACb,8BAA8B;CAC9B,gBAAgB;AAClB;AAEA,MAAM,gBAAsC;CAC1C,wBAAwB;CACxB,aAAa;AACf;AAEA,eAAsB,gBACpB,MAC6B;CAE7B,eAAe,KAAK,SAAS,KAAK,qBAAqB,aAAa;CAEpE,IAAI,KAAK,SAAS,WAAW,GAC3B,MAAM,IAAI,MAAM,8CAA8C;CAEhE,IAAI,KAAK,UAAU,WAAW,GAC5B,MAAM,IAAI,MAAM,+CAA+C;CAEjE,MAAM,6BAAa,IAAI,IAAY;CACnC,KAAK,MAAM,KAAK,KAAK,UAAU;EAC7B,IAAI,WAAW,IAAI,EAAE,EAAE,GACrB,MAAM,IAAI,MAAM,0CAA0C,EAAE,GAAG,GAAG;EAEpE,WAAW,IAAI,EAAE,EAAE;CACrB;CACA,MAAM,8BAAc,IAAI,IAAY;CACpC,KAAK,MAAM,KAAK,KAAK,WAAW;EAC9B,IAAI,YAAY,IAAI,EAAE,UAAU,GAC9B,MAAM,IAAI,MAAM,0CAA0C,EAAE,WAAW,GAAG;EAE5E,YAAY,IAAI,EAAE,UAAU;CAC9B;CACA,IAAI,KAAK,QAAQ,cAAc,CAAC,WAAW,IAAI,KAAK,OAAO,UAAU,GACnE,MAAM,IAAI,MACR,uCAAuC,KAAK,OAAO,WAAW,iCAChE;CAEF,IAAI,CAAC,KAAK,WACR,MAAM,IAAI,MAAM,oEAAoE;CAGtF,MAAM,QAAQ,KAAK,SAAS;EAAC;EAAG;EAAG;CAAC;CACpC,MAAM,WAAwB,KAAK,YAAY;CAC/C,MAAM,cAAc,KAAK,IAAI,GAAG,KAAK,eAAe,CAAC;CACrD,MAAM,YAAY;EAAE,GAAG;EAAmB,GAAI,KAAK,aAAa,CAAC;CAAG;CACpE,MAAM,qBAA8C,KAAK,sBAAsB;CAC/E,MAAM,MAAM,KAAK,cAAc,KAAK,IAAI;CACxC,MAAM,WAAW,KAAK,QAAQ,WAAW,GAAA,CAAI,QAAQ,QAAQ,EAAE;CAC/D,MAAM,WAAW,KAAK,QAAQ,YAAY;CAC1C,MAAM,sBAAsB,KAAK,uBAAuB;CAExD,MAAM,iBAAiB,KAAK,kBAAkB,sBAAsB,KAAK,OAAO;CAGhF,MAAM,sBAAsB,MAAM,SAChC,aAAa;EACX,YAAY,KAAK;EACjB,UAAU,KAAK,SAAS,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK;EAC9C,WAAW,KAAK,UAAU,KAAK,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK;EACxD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;EACtC;EACA,YAAY,KAAK,QAAQ,cAAc;EACvC;EACA;EACA;CACF,CAAC,CACH;CAIA,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,WAAW,KAAK,UACzB,KAAK,MAAM,YAAY,KAAK,WAC1B,KAAK,MAAM,QAAQ,OACjB,MAAM,KAAK;EAAE;EAAS;EAAU;CAAK,CAAC;CAK5C,MAAM,YAAY,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;CAC9C,MAAM,OAAoB,CAAC;CAC3B,MAAM,mBAAyC,CAAC;CAChD,MAAM,aAA0B,CAAC;CASjC,IAAI,SAAS;CACb,IAAI,WAAW;CACf,MAAM,gBAA2B,CAAC;CAKlC,MAAM,2BAAW,IAAI,IAA0B;CAE/C,eAAe,SAAwB;EACrC,OAAO,CAAC,UAAU;GAChB,MAAM,IAAI;GACV,IAAI,KAAK,MAAM,QAAQ;GACvB,MAAM,OAAO,MAAM;GACnB,IAAI;IACF,MAAM,SAAS,MAAM,WAAW,IAAI;IACpC,KAAK,KAAK,OAAO,MAAM;IACvB,iBAAiB,KAAK,OAAO,SAAS;GACxC,SAAS,KAAK;IACZ,IAAI,eAAe,oBAAoB;KACrC,WAAW,KAAK,IAAI,MAAM;KAC1B,IAAI,IAAI,WAAW,iBAAiB,KAAK,IAAI,SAAS;IACxD,OAAO;KAKL,cAAc,KAAK,GAAG;KACtB,WAAW;KACX;IACF;GACF;EACF;CACF;CAEA,eAAe,WACb,MAC+D;EAC/D,MAAM,SAAS,KAAK,SAAS,aAAA,CAAc;GACzC,YAAY,KAAK;GACjB,OAAO;GACP,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,MAAM,KAAK;EACb,CAAC;EACD,MAAM,gBAAuC;GAC3C,YAAY,KAAK;GACjB;GACA,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,MAAM,KAAK;EACb;EACA,MAAM,QAAQ,KAAK,aAAa,aAAa;EAC7C,MAAM,UAAU,eAAe,aAAa;EAE5C,MAAM,UAAU,IAAI,aAAa,OAAO;GACtC;GACA,KAAK,KAAK;GACV,eAAe,KAAK;EACtB,CAAC;EAGD,SAAS,IAAI,OAAO,OAAO;EAE3B,MAAM,UAA4B;GAChC,GAAG,KAAK;GACR;GACA,cAAc,EAAE,MAAM;EACxB;EAEA,MAAM,MAA6B;GACjC;GACA,cAAc,KAAK;GACnB,SAAS,KAAK,QAAQ;GACtB,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,cAAc,KAAK,SAAS,QAAQ,CAAC;GACrC,MAAM,KAAK;GACX;GACA;GACA;GACA;GACA;EACF;EAEA,IAAI;GACF,MAAM,YAAY,IAAI;GACtB,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,KAAK,OAAO,GAAG;GACjC,SAAS,KAAK;IACZ,MAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAM/D,MAAM,cAAc,SAAS,OAAO,OAAO;IAC3C,MAAM,IAAI,mBAAmB;KAC3B;KACA,WAAW,KAAK,QAAQ;KACxB,YAAY,KAAK,SAAS;KAC1B,MAAM,KAAK;KACX,QAAQ;KACR,OAAO;IACT,CAAC;GACH;GACA,MAAM,SAAS,IAAI,IAAI;GAEvB,MAAM,kBAAkB,MAAM,kBAAkB,OAAO,OAAO;IAAE,GAAG;IAAW;GAAQ,CAAC;GACvF,IAAI,CAAC,gBAAgB,IACnB,QAAQ,oBAAR;IACE,KAAK,SACH,MAAM,IAAI,kBAAkB,eAAe;IAC7C,KAAK,eACH,MAAM,IAAI,mBACR;KACE;KACA,WAAW,KAAK,QAAQ;KACxB,YAAY,KAAK,SAAS;KAC1B,MAAM,KAAK;KACX,QAAQ;KACR,OAAO,gBAAgB,OAAO,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI;IAC5D,GACA,eACF;IACF,KAAK,OAEH;GACJ;GAGF,MAAM,gBAA4B,EAChC,KAAK,QAAQ,OAAO,CAAC,EACvB;GACA,IAAI,aAAa,WAAW,cAAc,eAAe,QAAQ;QAC5D,cAAc,cAAc,QAAQ;GACzC,IAAI,QAAQ,gBAAgB,KAAA,GAAW,cAAc,cAAc,QAAQ;GAE3E,MAAM,SAAoB;IACxB;IACA,cAAc,KAAK;IACnB,aAAa,KAAK,QAAQ;IAC1B,MAAM,KAAK;IACX,OAAO,QAAQ;IACf,YAAY,QAAQ;IACpB,YAAY,QAAQ;IACpB,WAAW,KAAK;IAChB;IACA,SAAS,QAAQ;IACjB,gBAAgB,QAAQ;IACxB,YAAY,QAAQ;IACpB,iBAAiB;IACjB,eAAe,QAAQ;IACvB,SAAS;IACT,GAAI,QAAQ,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;IACrE,aAAa,QAAQ;IACrB;IACA,YAAY,KAAK,SAAS;GAC5B;GACA,MAAM,gBACJ,QAAQ,iBACP,OAAO,KAAK,iBAAiB,aAC1B,MAAM,KAAK,aAAa;IACtB,YAAY,KAAK;IACjB;IACA,WAAW,KAAK,QAAQ;IACxB,YAAY,KAAK,SAAS;IAC1B,MAAM,KAAK;IACX,SAAS,KAAK,QAAQ;IACtB,cAAc,KAAK,SAAS,QAAQ,CAAC;GACvC,CAAC,IACD,KAAK;GACX,IAAI,kBAAkB,KAAA,GAAW;IAC/B,MAAM,eAAe,MAAM,wBAAwB,aAAa;IAChE,6BAA6B,cAAc,QAAQ,OAAO,QAAQ,UAAU;IAC5E,OAAO,eAAe;GACxB;GACA,OAAO;IAAE,QAAQ,kBAAkB,MAAM;IAAG,WAAW;GAAgB;EACzE,UAAU;GAIR,SAAS,OAAO,KAAK;EACvB;CACF;CAEA,MAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,SAAS,OAAO,CAAC;CAI1F,MAAM,QAAQ,WAAW,OAAO;CAMhC,KAAK,MAAM,CAAC,OAAO,YAAY,UAC7B,MAAM,cAAc,SAAS,OAAO,kDAAkD;CAExF,SAAS,MAAM;CAEf,IAAI,cAAc,SAAS,GACzB,MAAM,cAAc,WAAW,IAC3B,cAAc,KACd,IAAI,eACF,eACA,oBAAoB,cAAc,OAAO,iDAC3C;CAIN,IAAI;CACJ,IAAI,KAAK,QAQP,SAAS,MAAM,eAAe,MAAM;EANlC,GAAG,KAAK;EACR,YAAY,KAAK,OAAO;EACxB,OAAO,aAAa,QAAQ,WAAW;EACvC,aAAa,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;EACzC,qBAAqB,uBAAuB,KAAA;CAED,CAAC;CAGhD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;CAE5C,OAAO;EACL,YAAY,KAAK;EACjB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;AACF;AAIA,IAAM,qBAAN,cAAiC,MAAM;CACrC;CACA;CACA,YAAY,QAAmB,WAAgC;EAC7D,MAAM,QAAQ,OAAO,UAAU,GAAG,OAAO,WAAW,GAAG,OAAO,KAAK,WAAW,OAAO,QAAQ;EAC7F,KAAK,SAAS;EACd,KAAK,YAAY;CACnB;AACF;;;;;;;;;;;;;;;;AAiBA,eAAsB,cACpB,SACA,OACA,QACe;CACf,MAAM,WAAW,MAAM,QAAQ,WAAW,OAAO,KAAK;CACtD,IAAI,aAAa,KAAA,GAAW;CAC5B,IAAI,SAAS,WAAW,WAAW;CACnC,MAAM,QAAQ,SAAS,MAAM;AAC/B;AAEA,SAAS,sBAAsB,SAA6B;CAC1D,QAAQ,WAAmD;EACzD,IAAI,CAAC,SACH,MAAM,IAAI,MACR,6LACF;EAEF,OAAO,IAAI,0BAA0B,EACnC,KAAK,GAAG,QAAQ,cAAc,OAAO,QACvC,CAAC;CACH;AACF;AAEA,eAAe,wBACb,OAC2B;CAC3B,IAAI,mBAAmB,KAAK,GAAG;EAC7B,IAAI,CAAE,MAAM,uBAAuB,KAAK,GACtC,MAAM,IAAI,MAAM,iEAAiE;EAEnF,OAAO;CACT;CACA,OAAO,sBAAsB,KAAK;AACpC;AAEA,SAAS,mBACP,OAC2B;CAC3B,OAAO,mBAAmB,SAAS,YAAY;AACjD;AAEA,SAAS,6BACP,SACA,OACA,YACM;CACN,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,OACnD,MAAM,IAAI,MACR,wCAAwC,QAAQ,MAAM,kCAAkC,MAAM,EAChG;CAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,YAC7D,MAAM,IAAI,MACR,6CAA6C,QAAQ,WAAW,uCAAuC,WAAW,EACpH;AAEJ;AAEA,SAAS,aAAa,QAAuC;CAG3D,MAAM,OAAO,GAAG,OAAO,WAAW,IAAI,OAAO,UAAU,IAAI,OAAO,WAAW,IAAI,OAAO;CAExF,IAAI,KAAK;CACT,IAAI,KAAK;CACT,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,IAAI,KAAK,WAAW,CAAC;EAC3B,KAAK,KAAK,KAAK,KAAK,GAAG,QAAU,MAAM;EACvC,KAAK,KAAK,KAAK,KAAK,GAAG,UAAU,MAAM;CACzC;CACA,OAAO,OAAO,GAAG,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG;AAClF"}
|
|
1
|
+
{"version":3,"file":"eval-campaign-B_7wcnav.js","names":[],"sources":["../src/eval-campaign.ts"],"sourcesContent":["/**\n * EvalCampaign — opinionated matrix runner that wires the four\n * capture-integrity directives by construction.\n *\n * The canonical benchmark shape — matrix runner → for each\n * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the\n * run → analyze — has a bug class at the integration boundary: raw\n * events not captured, route silently wrong, integrity not asserted,\n * analyst never run. The directives in `SKILL.md § Capture integrity`\n * are the mitigations.\n *\n * `EvalCampaign` is the structural fix — consumers don't wire the\n * integrity surface themselves; the campaign owns it. Specifically:\n *\n * - calls `assertLlmRoute` once at preflight before any work runs\n * - constructs a per-run `TraceStore` and `RawProviderSink` via factories\n * - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`\n * - hands the runner an `LlmClientOptions` pre-wired with the sink and\n * trace context — the runner can't accidentally call an LLM without\n * capturing the raw HTTP envelope\n * - calls `assertRunCaptured` after every `endRun` and routes failures\n * through a configurable policy (`throw` / `mark_failed` / `log`)\n * - assembles per-run `RunRecord`s and runs `researchReport` at the end\n * so the campaign artifact is launch-decision-grade by default\n * - embeds the campaign fingerprint (a SHA-256 over the canonicalised\n * run set) and optional `preregistrationHash` in the report\n *\n * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`\n * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped\n * lives in the campaign. This is the inversion-of-control point — consumers\n * stop writing matrix runners and start writing scenario-runners.\n *\n * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):\n *\n * - Distributed/cluster execution (concurrency is local async)\n * - Adaptive sampling / sequential interim looks\n * - Resume from partial state across crashes\n * - LLM-call retry beyond what `LlmClient` already does\n */\n\nimport {\n type AgentProfileCell,\n type AgentProfileCellInput,\n buildAgentProfileCell,\n verifyAgentProfileCell,\n} from './agent-profile-cell'\nimport { assertLlmRoute, type LlmClientOptions, type LlmRouteRequirements } from './llm-client'\nimport { canonicalize, hashJson } from './pre-registration'\nimport type {\n JudgeScoresRecord,\n RunCostProvenance,\n RunJudgeMetadata,\n RunOutcome,\n RunRecord,\n RunSplitTag,\n RunTaskFailure,\n RunTokenUsage,\n} from './run-record'\nimport { validateRunRecord } from './run-record'\nimport { type ResearchReport, type ResearchReportOptions, researchReport } from './summary-report'\nimport type { RunCompleteHook } from './trace/emitter'\nimport { TraceEmitter } from './trace/emitter'\nimport {\n assertRunCaptured,\n RunIntegrityError,\n type RunIntegrityExpectations,\n type RunIntegrityReport,\n} from './trace/integrity'\nimport { FileSystemRawProviderSink, type RawProviderSink } from './trace/raw-provider-sink'\nimport type { TraceStore } from './trace/store'\n\n// ── Public types ─────────────────────────────────────────────────────────\n\nexport interface CampaignVariant<V> {\n id: string\n payload: V\n}\n\nexport interface CampaignScenario {\n scenarioId: string\n /** Free-form metadata propagated to runs and reports. */\n tags?: Record<string, string>\n}\n\nexport interface CampaignRunContext<V> {\n /** Stable run id. The campaign generates this; the runner does not. */\n runId: string\n /** Logical experiment id (campaignId by default; overridable per-run via opts). */\n experimentId: string\n variant: V\n variantId: string\n scenarioId: string\n scenarioTags: Record<string, string>\n seed: number\n splitTag: RunSplitTag\n /**\n * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired\n * (analyst auto-execution if configured, plus integrity check). The\n * runner MUST call `emitter.startRun` before doing any work and either\n * `emitter.endRun` or `emitter.abortRun` before returning.\n */\n emitter: TraceEmitter\n store: TraceStore\n rawSink: RawProviderSink\n /**\n * Pre-wired LLM client options — `rawSink` and `traceContext` are populated\n * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The\n * runner can spread additional fields if needed.\n */\n llmOpts: LlmClientOptions\n}\n\ninterface CampaignRunOutcomeFields {\n /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */\n pass: boolean\n /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */\n score: number\n /** Cost in USD, or null when the runner could not capture it. */\n costUsd: number | null\n /** Source of the cost amount. */\n costProvenance: RunCostProvenance\n tokenUsage: RunTokenUsage\n /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */\n model: string\n /** sha256 of the effective prompt sent to the model. */\n promptHash: string\n /** sha256 of the effective config (model, temperature, tools, judges, splits). */\n configHash: string\n /** Optional extra numeric metrics to land in `outcome.raw`. */\n raw?: Record<string, number>\n /** Optional judge metadata when a judge was used. */\n judgeMetadata?: RunJudgeMetadata\n /**\n * Optional per-judge / per-dim breakdown for ensemble-judged runs.\n * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.\n * Single-judge or scalar-only runs leave this unset.\n */\n judgeScores?: JudgeScoresRecord\n /**\n * Agent profile cell observed by the runner. When supplied, it overrides\n * `EvalCampaignOptions.agentProfile` for this run and must match the\n * outcome's `model` and `promptHash`.\n */\n agentProfile?: AgentProfileCell | AgentProfileCellInput\n}\n\n/** Campaign result with the same task-failure invariant as `RunRecord`. */\nexport type CampaignRunOutcome = CampaignRunOutcomeFields & RunTaskFailure\n\nexport type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>\n\nexport type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log'\n\nexport interface EvalCampaignOptions<V> {\n /**\n * Stable id for the campaign. Used as the default `experimentId` on\n * every run, and folded into the campaign fingerprint.\n */\n campaignId: string\n variants: CampaignVariant<V>[]\n scenarios: CampaignScenario[]\n /** Default `[0, 1, 2]`. */\n seeds?: number[]\n /** Default `'holdout'` — the split that anchors a launch decision. */\n splitTag?: RunSplitTag\n /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */\n commitSha: string\n /**\n * LLM client config. Augmented per-run with `rawSink` and `traceContext`\n * before being passed to the runner. The campaign asserts this config\n * matches `routeRequirements` once at preflight.\n */\n llmOpts: LlmClientOptions\n /**\n * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail\n * loud if the campaign would silently fall back to the public router or\n * run unauthenticated. Override with an empty object to disable.\n */\n routeRequirements?: LlmRouteRequirements\n /**\n * Per-run TraceStore factory. Common shape: a fresh store per run keyed\n * on `runId`. Implementations that share a store across the campaign\n * are valid — the campaign only writes through `emitter`.\n */\n storeFactory: (params: CampaignFactoryParams) => TraceStore\n /**\n * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`\n * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;\n * otherwise required. Forensic capture is non-negotiable in a campaign\n * run — pass `NoopRawProviderSink` explicitly if you want to opt out.\n */\n rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink\n /**\n * Filesystem root for default `rawSinkFactory`. Ignored if\n * `rawSinkFactory` is supplied.\n */\n workDir?: string\n /**\n * Extra `onRunComplete` hooks the campaign appends (after its own\n * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.\n */\n onRunComplete?: RunCompleteHook[]\n /**\n * Per-run integrity expectations. Defaults to:\n * `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.\n * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.\n */\n integrity?: RunIntegrityExpectations\n /** Behaviour when integrity fails. Default `'mark_failed'`. */\n onIntegrityFailure?: CampaignIntegrityPolicy\n /**\n * Per-run runner. Receives a fully-wired context; produces an outcome\n * the campaign converts into a `RunRecord`.\n */\n runner: CampaignRunner<V>\n /**\n * If set, the campaign computes `researchReport` at the end. `comparator`\n * is a `variantId`. Other fields are forwarded verbatim.\n */\n report?: { comparator?: string } & Omit<\n ResearchReportOptions,\n 'comparator' | 'preregistrationHash' | 'generatedAt'\n >\n /**\n * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).\n * Embedded in the campaign fingerprint and the research report.\n */\n preregistrationHash?: string\n /** Local concurrency. Default `1` (sequential). */\n concurrency?: number\n /**\n * Override the time source. Tests pass a mock to make wallMs deterministic.\n */\n now?: () => number\n /** Override the runId generator. Tests pin this. */\n runId?: (params: CampaignFactoryParams) => string\n /**\n * Agent profile cell for campaign runs. Static profiles can pass an object;\n * routers or variant-specific harnesses can pass a factory. The campaign\n * stamps the built cell onto every `RunRecord` and rejects profile/model or\n * profile/prompt contradictions.\n */\n agentProfile?:\n | AgentProfileCell\n | AgentProfileCellInput\n | ((\n params: CampaignFactoryParams & {\n variant: V\n scenarioTags: Record<string, string>\n },\n ) =>\n | AgentProfileCell\n | AgentProfileCellInput\n | Promise<AgentProfileCell | AgentProfileCellInput>)\n}\n\nexport interface CampaignFactoryParams {\n campaignId: string\n runId: string\n variantId: string\n scenarioId: string\n seed: number\n}\n\nexport interface FailedRun {\n runId: string\n variantId: string\n scenarioId: string\n seed: number\n reason: string\n error?: string\n}\n\nexport interface EvalCampaignResult {\n campaignId: string\n /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */\n campaignFingerprint: string\n preregistrationHash: string | null\n /** Successful runs only. Failed runs land in `failedRuns`. */\n runs: RunRecord[]\n /** Integrity reports for every successful run. */\n integrityReports: RunIntegrityReport[]\n failedRuns: FailedRun[]\n /** Computed when `report` is set on options. */\n report?: ResearchReport\n startedAt: string\n endedAt: string\n}\n\n// ── Implementation ───────────────────────────────────────────────────────\n\nconst DEFAULT_INTEGRITY: RunIntegrityExpectations = {\n llmSpansMin: 1,\n requireRawCoverageOfLlmSpans: true,\n requireOutcome: true,\n}\n\nconst DEFAULT_ROUTE: LlmRouteRequirements = {\n requireExplicitBaseUrl: true,\n requireAuth: true,\n}\n\nexport async function runEvalCampaign<V>(\n opts: EvalCampaignOptions<V>,\n): Promise<EvalCampaignResult> {\n // ── Preflight ──────────────────────────────────────────────────────\n assertLlmRoute(opts.llmOpts, opts.routeRequirements ?? DEFAULT_ROUTE)\n\n if (opts.variants.length === 0) {\n throw new Error('runEvalCampaign: variants must be non-empty.')\n }\n if (opts.scenarios.length === 0) {\n throw new Error('runEvalCampaign: scenarios must be non-empty.')\n }\n const variantIds = new Set<string>()\n for (const v of opts.variants) {\n if (variantIds.has(v.id)) {\n throw new Error(`runEvalCampaign: duplicate variant id \"${v.id}\".`)\n }\n variantIds.add(v.id)\n }\n const scenarioIds = new Set<string>()\n for (const s of opts.scenarios) {\n if (scenarioIds.has(s.scenarioId)) {\n throw new Error(`runEvalCampaign: duplicate scenarioId \"${s.scenarioId}\".`)\n }\n scenarioIds.add(s.scenarioId)\n }\n if (opts.report?.comparator && !variantIds.has(opts.report.comparator)) {\n throw new Error(\n `runEvalCampaign: report.comparator \"${opts.report.comparator}\" is not a configured variantId.`,\n )\n }\n if (!opts.commitSha) {\n throw new Error('runEvalCampaign: commitSha is required (every RunRecord needs it).')\n }\n\n const seeds = opts.seeds ?? [0, 1, 2]\n const splitTag: RunSplitTag = opts.splitTag ?? 'holdout'\n const concurrency = Math.max(1, opts.concurrency ?? 1)\n const integrity = { ...DEFAULT_INTEGRITY, ...(opts.integrity ?? {}) }\n const onIntegrityFailure: CampaignIntegrityPolicy = opts.onIntegrityFailure ?? 'mark_failed'\n const now = opts.now ?? (() => Date.now())\n const baseUrl = (opts.llmOpts.baseUrl ?? '').replace(/\\/+$/, '')\n const provider = opts.llmOpts.provider ?? null\n const preregistrationHash = opts.preregistrationHash ?? null\n\n const rawSinkFactory = opts.rawSinkFactory ?? defaultRawSinkFactory(opts.workDir)\n\n // ── Fingerprint ────────────────────────────────────────────────────\n const campaignFingerprint = await hashJson(\n canonicalize({\n campaignId: opts.campaignId,\n variants: opts.variants.map((v) => v.id).sort(),\n scenarios: opts.scenarios.map((s) => s.scenarioId).sort(),\n seeds: [...seeds].sort((a, b) => a - b),\n splitTag,\n comparator: opts.report?.comparator ?? null,\n baseUrl,\n provider,\n preregistrationHash,\n }),\n )\n\n // ── Plan the matrix ────────────────────────────────────────────────\n type Cell = { variant: CampaignVariant<V>; scenario: CampaignScenario; seed: number }\n const cells: Cell[] = []\n for (const variant of opts.variants) {\n for (const scenario of opts.scenarios) {\n for (const seed of seeds) {\n cells.push({ variant, scenario, seed })\n }\n }\n }\n\n const startedAt = new Date(now()).toISOString()\n const runs: RunRecord[] = []\n const integrityReports: RunIntegrityReport[] = []\n const failedRuns: FailedRun[] = []\n\n // ── Execute (bounded-concurrency worker pool) ──────────────────────\n // A genuine (non-CellExecutionError) error from any worker is a bug, not a\n // run-level failure. We must NOT let `Promise.all` reject mid-flight and\n // orphan the other workers' in-progress runs (emitters never finalized,\n // sinks/handles leak, partial work silently discarded). Instead: capture the\n // first genuine error, stop dispatching new cells so in-flight workers wind\n // down, finalize every still-open run, then re-throw the aggregated error.\n let cursor = 0\n let aborting = false\n const genuineErrors: unknown[] = []\n // Emitters for runs that are currently mid-flight, keyed by runId. A worker\n // registers its emitter before invoking the runner and removes it once the\n // run is finalized (via endRun/abortRun, success or failure). Anything left\n // here after the pool settles is an orphan we must abort.\n const openRuns = new Map<string, TraceEmitter>()\n\n async function worker(): Promise<void> {\n while (!aborting) {\n const i = cursor++\n if (i >= cells.length) return\n const cell = cells[i]!\n try {\n const result = await runOneCell(cell)\n runs.push(result.record)\n integrityReports.push(result.integrity)\n } catch (err) {\n if (err instanceof CellExecutionError) {\n failedRuns.push(err.failed)\n if (err.integrity) integrityReports.push(err.integrity)\n } else {\n // Genuine bug — not a runner failure, not an integrity failure.\n // Capture it and stop dispatching so peers wind down gracefully;\n // re-thrown after the pool settles. Do not surface here — that would\n // reject Promise.all and orphan the other workers' runs.\n genuineErrors.push(err)\n aborting = true\n return\n }\n }\n }\n }\n\n async function runOneCell(\n cell: Cell,\n ): Promise<{ record: RunRecord; integrity: RunIntegrityReport }> {\n const runId = (opts.runId ?? defaultRunId)({\n campaignId: opts.campaignId,\n runId: '', // unused by default generator\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n })\n const factoryParams: CampaignFactoryParams = {\n campaignId: opts.campaignId,\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n }\n const store = opts.storeFactory(factoryParams)\n const rawSink = rawSinkFactory(factoryParams)\n\n const emitter = new TraceEmitter(store, {\n runId,\n now: opts.now,\n onRunComplete: opts.onRunComplete,\n })\n // Track this run as open so a genuine error elsewhere in the pool can\n // finalize it instead of orphaning it. Removed in the finally below.\n openRuns.set(runId, emitter)\n\n const llmOpts: LlmClientOptions = {\n ...opts.llmOpts,\n rawSink,\n traceContext: { runId },\n }\n\n const ctx: CampaignRunContext<V> = {\n runId,\n experimentId: opts.campaignId,\n variant: cell.variant.payload,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n scenarioTags: cell.scenario.tags ?? {},\n seed: cell.seed,\n splitTag,\n emitter,\n store,\n rawSink,\n llmOpts,\n }\n\n try {\n const wallStart = now()\n let outcome: CampaignRunOutcome\n try {\n outcome = await opts.runner(ctx)\n } catch (err) {\n const message = err instanceof Error ? err.message : String(err)\n // The runner threw mid-execution. Abort the run so the emitter\n // finalizes. The only benign abortRun failure is \"the runner never\n // started the run\" (nothing to finalize). A store-write failure (disk\n // full, FS error) is a genuine diagnostic — surface it rather than\n // masking it as a plain runner failure.\n await finalizeAbort(emitter, runId, message)\n throw new CellExecutionError({\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n reason: 'runner_threw',\n error: message,\n })\n }\n const wallMs = now() - wallStart\n\n const integrityReport = await assertRunCaptured(store, runId, { ...integrity, rawSink })\n if (!integrityReport.ok) {\n switch (onIntegrityFailure) {\n case 'throw':\n throw new RunIntegrityError(integrityReport)\n case 'mark_failed':\n throw new CellExecutionError(\n {\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n reason: 'integrity_failed',\n error: integrityReport.issues.map((i) => i.code).join(', '),\n },\n integrityReport,\n )\n case 'log':\n // Caller wants the run admitted with a flagged report; fall through.\n break\n }\n }\n\n const recordOutcome: RunOutcome = {\n raw: outcome.raw ?? {},\n }\n if (splitTag === 'holdout') recordOutcome.holdoutScore = outcome.score\n else recordOutcome.searchScore = outcome.score\n if (outcome.judgeScores !== undefined) recordOutcome.judgeScores = outcome.judgeScores\n\n const record: RunRecord = {\n runId,\n experimentId: opts.campaignId,\n candidateId: cell.variant.id,\n seed: cell.seed,\n model: outcome.model,\n promptHash: outcome.promptHash,\n configHash: outcome.configHash,\n commitSha: opts.commitSha,\n wallMs,\n costUsd: outcome.costUsd,\n costProvenance: outcome.costProvenance,\n tokenUsage: outcome.tokenUsage,\n terminalOutcome: 'succeeded',\n judgeMetadata: outcome.judgeMetadata,\n outcome: recordOutcome,\n ...(outcome.failureClass ? { failureClass: outcome.failureClass } : {}),\n failureMode: outcome.failureMode,\n splitTag,\n scenarioId: cell.scenario.scenarioId,\n }\n const profileSource =\n outcome.agentProfile ??\n (typeof opts.agentProfile === 'function'\n ? await opts.agentProfile({\n campaignId: opts.campaignId,\n runId,\n variantId: cell.variant.id,\n scenarioId: cell.scenario.scenarioId,\n seed: cell.seed,\n variant: cell.variant.payload,\n scenarioTags: cell.scenario.tags ?? {},\n })\n : opts.agentProfile)\n if (profileSource !== undefined) {\n const agentProfile = await resolveAgentProfileCell(profileSource)\n assertAgentProfileMatchesRun(agentProfile, outcome.model, outcome.promptHash)\n record.agentProfile = agentProfile\n }\n return { record: validateRunRecord(record), integrity: integrityReport }\n } finally {\n // This run's worker has finished with it (success, run-level failure, or\n // genuine error). It is no longer the pool's job to finalize — drop it so\n // the post-settle sweep doesn't double-abort a finalized run.\n openRuns.delete(runId)\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, cells.length) }, () => worker())\n // allSettled (not all): a genuine error in one worker must not reject the\n // pool mid-flight and orphan the others. Each worker captures its own\n // genuine error into `genuineErrors` and returns; we re-throw below.\n await Promise.allSettled(workers)\n\n // Finalize any run still open after the pool wound down. With the\n // stop-dispatch flag these are the runs that were mid-flight in peer workers\n // when the first genuine error fired — abort them so their emitters finalize\n // (hooks fire, store records a terminal status) instead of leaking.\n for (const [runId, emitter] of openRuns) {\n await finalizeAbort(emitter, runId, 'campaign aborted: genuine error in a sibling run')\n }\n openRuns.clear()\n\n if (genuineErrors.length > 0) {\n throw genuineErrors.length === 1\n ? genuineErrors[0]\n : new AggregateError(\n genuineErrors,\n `runEvalCampaign: ${genuineErrors.length} runs failed with genuine (non-run-level) errors`,\n )\n }\n\n // ── Optional research report ───────────────────────────────────────\n let report: ResearchReport | undefined\n if (opts.report) {\n const reportOpts: ResearchReportOptions = {\n ...opts.report,\n comparator: opts.report.comparator,\n split: splitTag === 'dev' ? 'search' : splitTag,\n generatedAt: new Date(now()).toISOString(),\n preregistrationHash: preregistrationHash ?? undefined,\n }\n report = await researchReport(runs, reportOpts)\n }\n\n const endedAt = new Date(now()).toISOString()\n\n return {\n campaignId: opts.campaignId,\n campaignFingerprint,\n preregistrationHash,\n runs,\n integrityReports,\n failedRuns,\n report,\n startedAt,\n endedAt,\n }\n}\n\n// ── Internal ─────────────────────────────────────────────────────────────\n\nclass CellExecutionError extends Error {\n readonly failed: FailedRun\n readonly integrity?: RunIntegrityReport\n constructor(failed: FailedRun, integrity?: RunIntegrityReport) {\n super(`cell ${failed.variantId}/${failed.scenarioId}@${failed.seed} failed: ${failed.reason}`)\n this.failed = failed\n this.integrity = integrity\n }\n}\n\n/**\n * Abort a run whose owning work threw or was orphaned by a sibling's genuine\n * error, finalizing the emitter so hooks fire and the store records a terminal\n * status. Safe to call unconditionally: it only aborts runs that are still\n * `running`. Two no-op cases are intentional and benign:\n *\n * - the run was never started (absent from the store) — nothing to finalize\n * - the run is already terminal (`completed` / `failed` / `aborted`) —\n * finalizing again would overwrite the real outcome (e.g. flip a passed run\n * to `aborted` with `{ pass: false, notes: reason }`), so we leave it alone\n *\n * Any OTHER failure of the store read or the abort write (disk full, FS fault,\n * backend down) is a genuine diagnostic and propagates rather than being\n * swallowed.\n */\nexport async function finalizeAbort(\n emitter: TraceEmitter,\n runId: string,\n reason: string,\n): Promise<void> {\n const existing = await emitter.traceStore.getRun(runId)\n if (existing === undefined) return // run never started; nothing to abort\n if (existing.status !== 'running') return // already finalized; never overwrite a real outcome\n await emitter.abortRun(reason)\n}\n\nfunction defaultRawSinkFactory(workDir: string | undefined) {\n return (params: CampaignFactoryParams): RawProviderSink => {\n if (!workDir) {\n throw new Error(\n 'runEvalCampaign: rawSinkFactory not supplied and workDir not set. Pass either to enable raw provider capture, or pass `new NoopRawProviderSink()` via rawSinkFactory to opt out explicitly.',\n )\n }\n return new FileSystemRawProviderSink({\n dir: `${workDir}/raw-events/${params.runId}`,\n })\n }\n}\n\nasync function resolveAgentProfileCell(\n input: AgentProfileCell | AgentProfileCellInput,\n): Promise<AgentProfileCell> {\n if (isAgentProfileCell(input)) {\n if (!(await verifyAgentProfileCell(input))) {\n throw new Error(`runEvalCampaign: agentProfile.cellId does not match its content`)\n }\n return input\n }\n return buildAgentProfileCell(input)\n}\n\nfunction isAgentProfileCell(\n input: AgentProfileCell | AgentProfileCellInput,\n): input is AgentProfileCell {\n return 'schemaVersion' in input && 'cellId' in input\n}\n\nfunction assertAgentProfileMatchesRun(\n profile: AgentProfileCell,\n model: string,\n promptHash: string,\n): void {\n if (profile.model !== undefined && profile.model !== model) {\n throw new Error(\n `runEvalCampaign: agentProfile.model \"${profile.model}\" does not match outcome.model \"${model}\"`,\n )\n }\n if (profile.promptHash !== undefined && profile.promptHash !== promptHash) {\n throw new Error(\n `runEvalCampaign: agentProfile.promptHash \"${profile.promptHash}\" does not match outcome.promptHash \"${promptHash}\"`,\n )\n }\n}\n\nfunction defaultRunId(params: CampaignFactoryParams): string {\n // Stable across re-runs: fingerprint of (campaignId, variantId, scenarioId, seed).\n // Caller can override via opts.runId for non-deterministic IDs.\n const base = `${params.campaignId}::${params.variantId}::${params.scenarioId}::${params.seed}`\n // Lightweight hex: we don't need crypto-grade here, just stability + uniqueness.\n let h1 = 0x811c9dc5\n let h2 = 0x12345678\n for (let i = 0; i < base.length; i++) {\n const c = base.charCodeAt(i)\n h1 = Math.imul(h1 ^ c, 0x01000193) >>> 0\n h2 = Math.imul(h2 ^ c, 0x9e3779b1) >>> 0\n }\n return `run-${h1.toString(16).padStart(8, '0')}${h2.toString(16).padStart(8, '0')}`\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmSA,MAAM,oBAA8C;CAClD,aAAa;CACb,8BAA8B;CAC9B,gBAAgB;AAClB;AAEA,MAAM,gBAAsC;CAC1C,wBAAwB;CACxB,aAAa;AACf;AAEA,eAAsB,gBACpB,MAC6B;CAE7B,eAAe,KAAK,SAAS,KAAK,qBAAqB,aAAa;CAEpE,IAAI,KAAK,SAAS,WAAW,GAC3B,MAAM,IAAI,MAAM,8CAA8C;CAEhE,IAAI,KAAK,UAAU,WAAW,GAC5B,MAAM,IAAI,MAAM,+CAA+C;CAEjE,MAAM,6BAAa,IAAI,IAAY;CACnC,KAAK,MAAM,KAAK,KAAK,UAAU;EAC7B,IAAI,WAAW,IAAI,EAAE,EAAE,GACrB,MAAM,IAAI,MAAM,0CAA0C,EAAE,GAAG,GAAG;EAEpE,WAAW,IAAI,EAAE,EAAE;CACrB;CACA,MAAM,8BAAc,IAAI,IAAY;CACpC,KAAK,MAAM,KAAK,KAAK,WAAW;EAC9B,IAAI,YAAY,IAAI,EAAE,UAAU,GAC9B,MAAM,IAAI,MAAM,0CAA0C,EAAE,WAAW,GAAG;EAE5E,YAAY,IAAI,EAAE,UAAU;CAC9B;CACA,IAAI,KAAK,QAAQ,cAAc,CAAC,WAAW,IAAI,KAAK,OAAO,UAAU,GACnE,MAAM,IAAI,MACR,uCAAuC,KAAK,OAAO,WAAW,iCAChE;CAEF,IAAI,CAAC,KAAK,WACR,MAAM,IAAI,MAAM,oEAAoE;CAGtF,MAAM,QAAQ,KAAK,SAAS;EAAC;EAAG;EAAG;CAAC;CACpC,MAAM,WAAwB,KAAK,YAAY;CAC/C,MAAM,cAAc,KAAK,IAAI,GAAG,KAAK,eAAe,CAAC;CACrD,MAAM,YAAY;EAAE,GAAG;EAAmB,GAAI,KAAK,aAAa,CAAC;CAAG;CACpE,MAAM,qBAA8C,KAAK,sBAAsB;CAC/E,MAAM,MAAM,KAAK,cAAc,KAAK,IAAI;CACxC,MAAM,WAAW,KAAK,QAAQ,WAAW,GAAA,CAAI,QAAQ,QAAQ,EAAE;CAC/D,MAAM,WAAW,KAAK,QAAQ,YAAY;CAC1C,MAAM,sBAAsB,KAAK,uBAAuB;CAExD,MAAM,iBAAiB,KAAK,kBAAkB,sBAAsB,KAAK,OAAO;CAGhF,MAAM,sBAAsB,MAAM,SAChC,aAAa;EACX,YAAY,KAAK;EACjB,UAAU,KAAK,SAAS,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK;EAC9C,WAAW,KAAK,UAAU,KAAK,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK;EACxD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;EACtC;EACA,YAAY,KAAK,QAAQ,cAAc;EACvC;EACA;EACA;CACF,CAAC,CACH;CAIA,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,WAAW,KAAK,UACzB,KAAK,MAAM,YAAY,KAAK,WAC1B,KAAK,MAAM,QAAQ,OACjB,MAAM,KAAK;EAAE;EAAS;EAAU;CAAK,CAAC;CAK5C,MAAM,YAAY,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;CAC9C,MAAM,OAAoB,CAAC;CAC3B,MAAM,mBAAyC,CAAC;CAChD,MAAM,aAA0B,CAAC;CASjC,IAAI,SAAS;CACb,IAAI,WAAW;CACf,MAAM,gBAA2B,CAAC;CAKlC,MAAM,2BAAW,IAAI,IAA0B;CAE/C,eAAe,SAAwB;EACrC,OAAO,CAAC,UAAU;GAChB,MAAM,IAAI;GACV,IAAI,KAAK,MAAM,QAAQ;GACvB,MAAM,OAAO,MAAM;GACnB,IAAI;IACF,MAAM,SAAS,MAAM,WAAW,IAAI;IACpC,KAAK,KAAK,OAAO,MAAM;IACvB,iBAAiB,KAAK,OAAO,SAAS;GACxC,SAAS,KAAK;IACZ,IAAI,eAAe,oBAAoB;KACrC,WAAW,KAAK,IAAI,MAAM;KAC1B,IAAI,IAAI,WAAW,iBAAiB,KAAK,IAAI,SAAS;IACxD,OAAO;KAKL,cAAc,KAAK,GAAG;KACtB,WAAW;KACX;IACF;GACF;EACF;CACF;CAEA,eAAe,WACb,MAC+D;EAC/D,MAAM,SAAS,KAAK,SAAS,aAAA,CAAc;GACzC,YAAY,KAAK;GACjB,OAAO;GACP,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,MAAM,KAAK;EACb,CAAC;EACD,MAAM,gBAAuC;GAC3C,YAAY,KAAK;GACjB;GACA,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,MAAM,KAAK;EACb;EACA,MAAM,QAAQ,KAAK,aAAa,aAAa;EAC7C,MAAM,UAAU,eAAe,aAAa;EAE5C,MAAM,UAAU,IAAI,aAAa,OAAO;GACtC;GACA,KAAK,KAAK;GACV,eAAe,KAAK;EACtB,CAAC;EAGD,SAAS,IAAI,OAAO,OAAO;EAE3B,MAAM,UAA4B;GAChC,GAAG,KAAK;GACR;GACA,cAAc,EAAE,MAAM;EACxB;EAEA,MAAM,MAA6B;GACjC;GACA,cAAc,KAAK;GACnB,SAAS,KAAK,QAAQ;GACtB,WAAW,KAAK,QAAQ;GACxB,YAAY,KAAK,SAAS;GAC1B,cAAc,KAAK,SAAS,QAAQ,CAAC;GACrC,MAAM,KAAK;GACX;GACA;GACA;GACA;GACA;EACF;EAEA,IAAI;GACF,MAAM,YAAY,IAAI;GACtB,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,KAAK,OAAO,GAAG;GACjC,SAAS,KAAK;IACZ,MAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAM/D,MAAM,cAAc,SAAS,OAAO,OAAO;IAC3C,MAAM,IAAI,mBAAmB;KAC3B;KACA,WAAW,KAAK,QAAQ;KACxB,YAAY,KAAK,SAAS;KAC1B,MAAM,KAAK;KACX,QAAQ;KACR,OAAO;IACT,CAAC;GACH;GACA,MAAM,SAAS,IAAI,IAAI;GAEvB,MAAM,kBAAkB,MAAM,kBAAkB,OAAO,OAAO;IAAE,GAAG;IAAW;GAAQ,CAAC;GACvF,IAAI,CAAC,gBAAgB,IACnB,QAAQ,oBAAR;IACE,KAAK,SACH,MAAM,IAAI,kBAAkB,eAAe;IAC7C,KAAK,eACH,MAAM,IAAI,mBACR;KACE;KACA,WAAW,KAAK,QAAQ;KACxB,YAAY,KAAK,SAAS;KAC1B,MAAM,KAAK;KACX,QAAQ;KACR,OAAO,gBAAgB,OAAO,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI;IAC5D,GACA,eACF;IACF,KAAK,OAEH;GACJ;GAGF,MAAM,gBAA4B,EAChC,KAAK,QAAQ,OAAO,CAAC,EACvB;GACA,IAAI,aAAa,WAAW,cAAc,eAAe,QAAQ;QAC5D,cAAc,cAAc,QAAQ;GACzC,IAAI,QAAQ,gBAAgB,KAAA,GAAW,cAAc,cAAc,QAAQ;GAE3E,MAAM,SAAoB;IACxB;IACA,cAAc,KAAK;IACnB,aAAa,KAAK,QAAQ;IAC1B,MAAM,KAAK;IACX,OAAO,QAAQ;IACf,YAAY,QAAQ;IACpB,YAAY,QAAQ;IACpB,WAAW,KAAK;IAChB;IACA,SAAS,QAAQ;IACjB,gBAAgB,QAAQ;IACxB,YAAY,QAAQ;IACpB,iBAAiB;IACjB,eAAe,QAAQ;IACvB,SAAS;IACT,GAAI,QAAQ,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;IACrE,aAAa,QAAQ;IACrB;IACA,YAAY,KAAK,SAAS;GAC5B;GACA,MAAM,gBACJ,QAAQ,iBACP,OAAO,KAAK,iBAAiB,aAC1B,MAAM,KAAK,aAAa;IACtB,YAAY,KAAK;IACjB;IACA,WAAW,KAAK,QAAQ;IACxB,YAAY,KAAK,SAAS;IAC1B,MAAM,KAAK;IACX,SAAS,KAAK,QAAQ;IACtB,cAAc,KAAK,SAAS,QAAQ,CAAC;GACvC,CAAC,IACD,KAAK;GACX,IAAI,kBAAkB,KAAA,GAAW;IAC/B,MAAM,eAAe,MAAM,wBAAwB,aAAa;IAChE,6BAA6B,cAAc,QAAQ,OAAO,QAAQ,UAAU;IAC5E,OAAO,eAAe;GACxB;GACA,OAAO;IAAE,QAAQ,kBAAkB,MAAM;IAAG,WAAW;GAAgB;EACzE,UAAU;GAIR,SAAS,OAAO,KAAK;EACvB;CACF;CAEA,MAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,SAAS,OAAO,CAAC;CAI1F,MAAM,QAAQ,WAAW,OAAO;CAMhC,KAAK,MAAM,CAAC,OAAO,YAAY,UAC7B,MAAM,cAAc,SAAS,OAAO,kDAAkD;CAExF,SAAS,MAAM;CAEf,IAAI,cAAc,SAAS,GACzB,MAAM,cAAc,WAAW,IAC3B,cAAc,KACd,IAAI,eACF,eACA,oBAAoB,cAAc,OAAO,iDAC3C;CAIN,IAAI;CACJ,IAAI,KAAK,QAQP,SAAS,MAAM,eAAe,MAAM;EANlC,GAAG,KAAK;EACR,YAAY,KAAK,OAAO;EACxB,OAAO,aAAa,QAAQ,WAAW;EACvC,aAAa,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;EACzC,qBAAqB,uBAAuB,KAAA;CAED,CAAC;CAGhD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY;CAE5C,OAAO;EACL,YAAY,KAAK;EACjB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;AACF;AAIA,IAAM,qBAAN,cAAiC,MAAM;CACrC;CACA;CACA,YAAY,QAAmB,WAAgC;EAC7D,MAAM,QAAQ,OAAO,UAAU,GAAG,OAAO,WAAW,GAAG,OAAO,KAAK,WAAW,OAAO,QAAQ;EAC7F,KAAK,SAAS;EACd,KAAK,YAAY;CACnB;AACF;;;;;;;;;;;;;;;;AAiBA,eAAsB,cACpB,SACA,OACA,QACe;CACf,MAAM,WAAW,MAAM,QAAQ,WAAW,OAAO,KAAK;CACtD,IAAI,aAAa,KAAA,GAAW;CAC5B,IAAI,SAAS,WAAW,WAAW;CACnC,MAAM,QAAQ,SAAS,MAAM;AAC/B;AAEA,SAAS,sBAAsB,SAA6B;CAC1D,QAAQ,WAAmD;EACzD,IAAI,CAAC,SACH,MAAM,IAAI,MACR,6LACF;EAEF,OAAO,IAAI,0BAA0B,EACnC,KAAK,GAAG,QAAQ,cAAc,OAAO,QACvC,CAAC;CACH;AACF;AAEA,eAAe,wBACb,OAC2B;CAC3B,IAAI,mBAAmB,KAAK,GAAG;EAC7B,IAAI,CAAE,MAAM,uBAAuB,KAAK,GACtC,MAAM,IAAI,MAAM,iEAAiE;EAEnF,OAAO;CACT;CACA,OAAO,sBAAsB,KAAK;AACpC;AAEA,SAAS,mBACP,OAC2B;CAC3B,OAAO,mBAAmB,SAAS,YAAY;AACjD;AAEA,SAAS,6BACP,SACA,OACA,YACM;CACN,IAAI,QAAQ,UAAU,KAAA,KAAa,QAAQ,UAAU,OACnD,MAAM,IAAI,MACR,wCAAwC,QAAQ,MAAM,kCAAkC,MAAM,EAChG;CAEF,IAAI,QAAQ,eAAe,KAAA,KAAa,QAAQ,eAAe,YAC7D,MAAM,IAAI,MACR,6CAA6C,QAAQ,WAAW,uCAAuC,WAAW,EACpH;AAEJ;AAEA,SAAS,aAAa,QAAuC;CAG3D,MAAM,OAAO,GAAG,OAAO,WAAW,IAAI,OAAO,UAAU,IAAI,OAAO,WAAW,IAAI,OAAO;CAExF,IAAI,KAAK;CACT,IAAI,KAAK;CACT,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,IAAI,KAAK,WAAW,CAAC;EAC3B,KAAK,KAAK,KAAK,KAAK,GAAG,QAAU,MAAM;EACvC,KAAK,KAAK,KAAK,KAAK,GAAG,UAAU,MAAM;CACzC;CACA,OAAO,OAAO,GAAG,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG;AAClF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { l as AnalystRunResult, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-
|
|
1
|
+
import { l as AnalystRunResult, s as AnalystRunEvent, t as Analyst, u as AnalystRunSummary } from "./types-BvDKaULh.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
//#region src/analyst/exact-types.d.ts
|
|
4
4
|
/** Analyst metadata required before the exact registry path will execute it. */
|
|
@@ -231,4 +231,4 @@ declare const exactExecutionPlanSchema: z.ZodObject<{
|
|
|
231
231
|
}, z.core.$strict>;
|
|
232
232
|
//#endregion
|
|
233
233
|
export { ExactAnalystRunPolicySnapshot as a, ExactAnalystSnapshot as c, ExactExecutionComponentSnapshot as d, ExactAnalystRunEvent as i, ExactCapableAnalyst as l, ExactAnalystExecutionPlanSnapshot as n, ExactAnalystRunResult as o, ExactAnalystRunCompletion as r, ExactAnalystRunSummary as s, ExactAnalystBudgetSnapshot as t, ExactExecutionComponentIdentity as u };
|
|
234
|
-
//# sourceMappingURL=exact-types-
|
|
234
|
+
//# sourceMappingURL=exact-types-DSFFpLLI.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exact-types-
|
|
1
|
+
{"version":3,"file":"exact-types-DSFFpLLI.d.ts","names":[],"sources":["../src/analyst/exact-types.ts"],"mappings":";;;;UAMiB,oBAAoB,0BAA0B,QAAQ;;WAE5D,iBAAiB,SAAS;;KAGhC,aAAa,KAAK,0BAA0B,mBACpC,aAAa,UACtB,+BACc,aAAa,IAAI,aAAa,EAAE,WAC5C;KAEM,uBAAuB,aAAa,EAAE,aAAa;KACnD,6BAA6B,aAAa,EAAE,aAAa;;UAGpD;EACf;EACA;EACA,QAAQ,SAAS;;;UAIF;EACf;EACA;EACA;;KAyBU,gCAAgC,aAAa,EAAE,aAAa;KAC5D,oCAAoC,aAC9C,EAAE,aAAa;KAGL,yBAAyB;;EAEnC;;KAGU;EACN;;EAEA;EACA;IAAS;IAAe;;;UAGb,8BAA8B;EAC7C,aAAa;EACb,gBAAgB;EAChB,YAAY;;;KAIF,wBACP,QAAQ;EAAmB;;EAC1B,gBAAgB;MAEjB,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;KAEX,QAAQ;EAAmB;MAC1B,KAAK,QAAQ;EAAmB;;EAC/B,SAAS;MAEV,KAAK,QAAQ;EAAmB;;EAC/B,QAAQ;;cAwCR,uBAAqB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAOzB,EAAA,KAAA;cAII,sBAAoB,EAAA,uBAAA,EAAA;;;;;;;;;;;GAaxB,EAAA,KAAA;cAiBI,sBAAoB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAYxB,EAAA,KAAA;cAEI,0BAAwB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+E1B,EAAA,KAAA"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { l as ValidationError, r as CaptureIntegrityError } from "../errors-CKPfb2aH.js";
|
|
2
2
|
import { B as eProcess, J as mcnemarRequiredN, K as mcnemar, N as benjaminiHochberg, P as bonferroni, Q as pairedBootstrap, S as ProportionInterval, V as holm, Y as mulberry32, _t as wilson, d as EProcessState, dt as requiredPairedSampleSize, f as EProcessStep, ft as requiredSampleSize, it as pairedRiskDifferenceScore, l as EProcess, nt as pairedRiskDifference, q as mcnemarPower, rt as pairedRiskDifferenceExact, t as BOOTSTRAP_GATE_MIN_N, tt as pairedMde, u as EProcessOptions, v as PairedBootstrapOptions, y as PairedBootstrapResult } from "../statistics-D6Uebe_4.js";
|
|
3
|
-
import { A as PairArmsResult, C as hashJson, D as MatchedPair, I as comparePairedArms, L as pairArms, M as PairedArmRow, R as pairRunRecords, S as evaluateHypothesis, T as verifyManifest, _ as HypothesisManifest, c as pairHoldout, d as SequentialDecision, f as SequentialObservation, g as sequentialPairedGate, h as sequentialDecide, i as PairedHoldout, k as PairArmsOptions, m as SequentialPairedGateOptions, n as HeldoutSignificance, p as SequentialPairedGate, r as HeldoutSignificanceOptions, s as heldoutSignificance, v as HypothesisResult, w as signManifest, x as canonicalize, y as SignedManifest } from "../statistical-heldout-
|
|
4
|
-
import { S as powerPreflight, b as PowerPreflight, c as PromotionPolicy, d as paretoSignificanceGate, i as EvidenceVector, l as buildEvidenceVector, u as paretoPolicy, x as PowerPreflightOptions } from "../promotion-policy-
|
|
3
|
+
import { A as PairArmsResult, C as hashJson, D as MatchedPair, I as comparePairedArms, L as pairArms, M as PairedArmRow, R as pairRunRecords, S as evaluateHypothesis, T as verifyManifest, _ as HypothesisManifest, c as pairHoldout, d as SequentialDecision, f as SequentialObservation, g as sequentialPairedGate, h as sequentialDecide, i as PairedHoldout, k as PairArmsOptions, m as SequentialPairedGateOptions, n as HeldoutSignificance, p as SequentialPairedGate, r as HeldoutSignificanceOptions, s as heldoutSignificance, v as HypothesisResult, w as signManifest, x as canonicalize, y as SignedManifest } from "../statistical-heldout-TQ-4CYiN.js";
|
|
4
|
+
import { S as powerPreflight, b as PowerPreflight, c as PromotionPolicy, d as paretoSignificanceGate, i as EvidenceVector, l as buildEvidenceVector, u as paretoPolicy, x as PowerPreflightOptions } from "../promotion-policy-Ckjhzg_4.js";
|
|
5
5
|
import { C as ClusteredPairedBinaryOptions, E as clusteredPairedBinary, S as ClusteredMatchedPair, T as ClusteredPairedBinaryStatistics, _ as inMemoryExperimentStore, a as ExperimentStats, b as ClusterSignFlipResult, i as ExperimentRep, l as ExperimentVerdict, m as fileExperimentStore, n as Experiment, s as ExperimentTracker, v as ClusterBootstrapInterval, w as ClusteredPairedBinaryResult, x as ClusteredBinaryCluster } from "../experiment-tracker-IMntXr6J.js";
|
|
6
6
|
import { a as PairedEvalueStep, c as pairedEvalueSequence, i as PairedEvalueSequence, r as PairedEvalueOptions } from "../sequential-CYwq6Ff_.js";
|
|
7
7
|
//#region src/experiment/ast.d.ts
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { o as FailureClass } from "./schema-BtVldJ3T.js";
|
|
2
2
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
3
3
|
import { i as TraceEmitter } from "./emitter-DGQGoLyj.js";
|
|
4
|
-
import { i as AnalystFinding, l as AnalystRunResult, p as EvidenceRef } from "./types-
|
|
5
|
-
import "./exact-types-
|
|
4
|
+
import { i as AnalystFinding, l as AnalystRunResult, p as EvidenceRef } from "./types-BvDKaULh.js";
|
|
5
|
+
import "./exact-types-DSFFpLLI.js";
|
|
6
6
|
import { a as DatasetScenario, o as DatasetSplit } from "./dataset-C8xaLXdY.js";
|
|
7
7
|
//#region src/control-runtime.d.ts
|
|
8
8
|
type ControlSeverity = 'info' | 'warning' | 'error' | 'critical';
|
|
@@ -425,4 +425,4 @@ declare function controlRunToFeedbackTrajectory<TState, TAction, TActionResult>(
|
|
|
425
425
|
}): FeedbackTrajectory;
|
|
426
426
|
//#endregion
|
|
427
427
|
export { ControlRunResult as $, feedbackTrajectoryToOptimizerRow as A, AnalystReviewCounts as B, analystRunToReviewRequests as C, feedbackTrajectoriesToDatasetScenarios as D, createFeedbackTrajectory as E, serializeFeedbackTrajectoriesJsonl as F, analystFindingDigest as G, AnalystReviewQuality as H, summarizePreferenceMemory as I, ControlActionOutcome as J, analystRunDigest as K, withAssignedFeedbackSplit as L, renderPreferenceMemoryMarkdown as M, replayFeedbackTrajectories as N, feedbackTrajectoriesToOptimizerRows as O, replayFeedbackTrajectory as P, ControlEvalResult as Q, AnalystFindingDigest as R, analystRunToFeedbackTrajectory as S, controlRunToFeedbackTrajectory as T, AnalystReviewSource as U, AnalystReviewDecision as V, AnalystRunDigest as W, ControlContext as X, ControlBudget as Y, ControlDecision as Z, FeedbackTrajectoryStore as _, FeedbackLabel as a, StopDecision as at, PreferenceMemoryEntry as b, FeedbackOptimizerRow as c, runAgentControlLoop as ct, FeedbackReplayResult as d, subjectiveEval as dt, ControlRuntimeConfig as et, FeedbackSeverity as f, FeedbackTrajectoryFilter as g, FeedbackTrajectory as h, FeedbackAttempt as i, ControlStopPolicies as it, parseFeedbackTrajectoriesJsonl as j, feedbackTrajectoryToDatasetScenario as k, FeedbackOutcome as l, stopOnNoProgress as lt, FeedbackTask as m, AnalystReviewRequest as n, ControlSeverity as nt, FeedbackLabelKind as o, allCriticalPassed as ot, FeedbackSplitPolicy as p, ControlActionFailureMode as q, FeedbackArtifactType as r, ControlStep as rt, FeedbackLabelSource as s, objectiveEval as st, AnalystFeedbackTrajectoryOptions as t, ControlRuntimeError as tt, FeedbackReplayAdapter as u, stopOnRepeatedAction as ut, FileSystemFeedbackTrajectoryStore as v, assignFeedbackSplit as w, ProposedSideEffect as x, InMemoryFeedbackTrajectoryStore as y, AnalystMissedIssue as z };
|
|
428
|
-
//# sourceMappingURL=feedback-trajectory-
|
|
428
|
+
//# sourceMappingURL=feedback-trajectory-CacHpxsp.d.ts.map
|
package/dist/{feedback-trajectory-Rh280oXo.d.ts.map → feedback-trajectory-CacHpxsp.d.ts.map}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"feedback-trajectory-
|
|
1
|
+
{"version":3,"file":"feedback-trajectory-CacHpxsp.d.ts","names":[],"sources":["../src/control-runtime.ts","../src/feedback-trajectory-review.ts","../src/feedback-trajectory.ts"],"mappings":";;;;;;;KAiBY;KACA;UAEK;;EAEf;;EAEA;;EAEA;;EAEA,WAAW;;EAEX;;EAEA;;EAEA;;EAEA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe,oBAAoB,QAAQ;;;;;EAK3C;;;;;EAKA;;EAEA;;EAEA,oBAAoB,OAAO;;EAE3B,qBAAqB,QAAQ;;UAGd,eACf,QACA,SACA,eACA,cAAc,oBAAoB;EAElC;EACA,OAAO;EACP,OAAO;EACP,SAAS,YAAY,QAAQ,SAAS,eAAe;EACrD,QAAQ;EACR;EACA;EACA;EACA;EACA,aAAa;EACb,UAAU;;KAGA,gBAAgB;EAEtB;EACA,QAAQ;EACR;;EAGA;EACA;EACA;EACA;;EAEA,eAAe;;UAGJ;EACf;EACA;EACA;EACA;EACA,eAAe;;UAGA,qBAAqB;EACpC;EACA,SAAS;EACT;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe,YACf,QACA,SACA,eACA,cAAc,oBAAoB;EAElC;EACA,UAAU,gBAAgB;EAC1B,aAAa;EACb,YAAY;EACZ,aAAa;EACb,YAAY;EACZ,gBAAgB,qBAAqB;EACrC;EACA;;UAGe,iBACf,QACA,SACA,eACA,cAAc,oBAAoB;EAElC;EACA;EACA;EACA;EACA;EACA,OAAO,YAAY,QAAQ,SAAS,eAAe;EACnD,YAAY;EACZ,YAAY;EACZ;EACA;;EAEA;EACA,eAAe;EACf,eAAe;EACf;;UAGe,qBACf,QACA,SACA,eACA,cAAc,oBAAoB;EAElC;EACA,SAAS,QAAQ;EACjB,SAAS;;EAET,gBAAgB;;;;;EAKhB,oBAAoB;IAClB,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,OAAO;IACP,SAAS,YAAY,QAAQ,SAAS,eAAe;;;EAIvD,UAAU;IACR,SAAS,YAAY,QAAQ,SAAS,eAAe;IACrD,aAAa;QACT,QAAQ,UAAU;;EAGxB,WAAW;IACT;IACA,OAAO;IACP,SAAS,YAAY,QAAQ,SAAS,eAAe;IACrD,aAAa;QACT,QAAQ,WAAW;;EAGzB,SACE,KAAK,eAAe,QAAQ,SAAS,eAAe,WACjD,QAAQ,gBAAgB,YAAY,gBAAgB;;EAGzD,MACE,QAAQ,SACR,KAAK,eAAe,QAAQ,SAAS,eAAe,WACjD,QAAQ,iBAAiB;;EAG9B,cACE,KAAK,eAAe,QAAQ,SAAS,eAAe,WACjD,QAAQ,gBAAgB;;EAG7B,UAAU,MAAM,YAAY,QAAQ,SAAS,eAAe,WAAW;;EAGvE,eAAe,oBAAoB,QAAQ;;EAG3C,QAAQ;EACR;EACA;EACA;;iBAQoB,oBACpB,QACA,SACA,eACA,cAAc,oBAAoB,mBAElC,QAAQ,qBAAqB,QAAQ,SAAS,eAAe,SAC5D,QAAQ,iBAAiB,QAAQ,SAAS,eAAe;iBAmmB5C,iBAAiB,QAAQ,SACvC,4BACA,UAAS,KAAK,oBAAoB,QAAQ,kCACzC,oBAAoB,QAAQ;iBAIf,qBAAqB,QAAQ,SAC3C,4BACA,UAAS,KAAK,oBAAoB,QAAQ,kCACzC,oBAAoB,QAAQ;iBAIf,cAAc,OAAO,KAAK,kCAAkC;iBAI5D,eAAe,OAAO,KAAK,kCAAkC;iBAsD7D,kBAAkB,OAAO;;;KC74B7B,sBAAsB,QAAQ;KAC9B;KACA;UAEK;EACf;EACA;EACA,WAAW;;UAGH;EACR,WAAW;EACX,QAAQ;EACR;EACA;EACA;EACA;;KAGU,yBACP;EACC;EACA,eAAe;EACf;EACA;MAED;EACC;EACA,cAAc;EACd;EACA;;UAGW;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,QAAQ;;;iBAaM,qBAAqB,SAAS,iBAAiB;;iBAK/C,iBAAiB,KAAK,mBAAmB;;;KCjD7C;KAWA;KAEA;KAYA;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,MAAM;EACN;EACA;EACA,WAAW;EACX;EACA,WAAW;;UAGI;EACf;EACA;EACA,cAAc;EACd;EACA;EACA,iBAAiB;EACjB,QAAQ;EACR,WAAW;EACX;EACA,WAAW;;UAGI;EACf;EACA;EACA,UAAU;EACV;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA,MAAM;EACN,UAAU;EACV,QAAQ;EACR,UAAU;EACV,QAAQ;EACR,OAAO;EACP;EACA;EACA,WAAW;;UAGI;EACf,KAAK,YAAY,qBAAqB;EACtC,IAAI,aAAa,QAAQ;EACzB,KAAK,SAAS,2BAA2B,QAAQ;EACjD,cAAc,YAAY,SAAS,kBAAkB,QAAQ;EAC7D,YAAY,YAAY,OAAO,eAAe,qBAAqB,QAAQ;;UAG5D;EACf;EACA;EACA,QAAQ;EACR;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,YAAY;EACZ;EACA,WAAW;;UAGI;EACf;EACA;EACA;EACA,QAAQ;EACR,UAAU;EACV,WAAW;;UAGI;EACf,OACE,YAAY,qBAEV,QAAQ,KAAK,yCACb,KAAK;;UAGM;EACf,MAAM;EACN,kBAAkB;EAClB,0BAA0B;EAC1B,2BAA2B;EAC3B,UAAU;EACV;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP;EACA;IACE;IACA;IACA;;EAEF,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV;EACA;EACA,UAAU;EACV;EACA;;cAUW,2CAA2C;mBACrC;EAEX,KAAK,YAAY,qBAAqB;EAItC,IAAI,aAAa,QAAQ;EAKzB,KAAK,SAAQ,2BAAgC,QAAQ;EAMrD,cAAc,YAAY,SAAS,kBAAkB,QAAQ;EAa7D,YACJ,YACA,OAAO,eACP,qBACC,QAAQ;;cAsBA,6CAA6C;mBACvC;mBACA;UACT;EAER,YAAY;IAAW;;EAIjB,KAAK,YAAY,qBAAqB;EAMtC,IAAI,aAAa,QAAQ;EAKzB,KAAK,SAAQ,2BAAgC,QAAQ;EAKrD,cAAc,YAAY,SAAS,kBAAkB,QAAQ;EAO7D,YACJ,YACA,OAAO,eACP,qBACC,QAAQ;UAOG;UAWA;;iBA8BA,yBAAyB;EACvC;EACA;EACA;EACA,MAAM;EACN,WAAW;EACX,SAAS;EACT,UAAU;EACV,QAAQ;EACR,OAAO;EACP;EACA,WAAW;IACT;;iBAqBY,+BACd,KAAK,kBACL,SAAS,mCACR;;iBAsFa,2BACd,KAAK,kBACL;EAAW;IACV;iBA4Da,oBACd,YAAY,KAAK,iEACjB,SAAQ,sBACP;iBAca,0BACd,YAAY,oBACZ,SAAS,sBACR;iBAOa,oCACd,YAAY,qBACX;iBAca,uCACd,cAAc,uBACb;iBAIa,iCACd,YAAY,qBACX;iBA6Ca,oCACd,cAAc,uBACb;iBAImB,yBACpB,YAAY,oBACZ,SAAS,wBACR,QAAQ;iBAkCW,2BACpB,cAAc,sBACd,SAAS,wBACR,QAAQ;iBAQK,0BACd,cAAc,sBACd;EAAW;IACV;iBA2Ba,+BAA+B,SAAS;iBAWxC,mCAAmC,cAAc;iBAQjD,+BAA+B,gBAAgB;iBAS/C,+BAA+B,QAAQ,SAAS,eAC9D,KAAK,iBAAiB,QAAQ,SAAS,gBACvC;EACE;EACA;EACA,eAAe;EACf,oBAAoB,MAAM,YAAY,QAAQ,SAAS;EACvD,0BACE,MAAM,YAAY,QAAQ,SAAS,mBAChC;EACL;IAED"}
|
package/dist/hosted/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { j as MutableSurface } from "../types-
|
|
2
|
-
import { o as InsightReport } from "../insight-report-
|
|
3
|
-
import { _ as UnixNanoTimestamp, a as hostedTenantFromEnv, c as EvalRunGenerationSnapshot, d as HostedIngestHeaders, f as HostedWireVersion, g as TraceSpanEvent, h as IngestTracesRequest, i as hostedClientFromEnv, l as EvalRunStatus, m as IngestResponse, n as HostedTenant, o as EvalRunCellScore, p as IngestEvalRunsRequest, r as createHostedClient, s as EvalRunEvent, t as HostedClient, u as HOSTED_WIRE_VERSION } from "../client-
|
|
1
|
+
import { j as MutableSurface } from "../types-BnjdJ70P.js";
|
|
2
|
+
import { o as InsightReport } from "../insight-report-CgX_s0Ez.js";
|
|
3
|
+
import { _ as UnixNanoTimestamp, a as hostedTenantFromEnv, c as EvalRunGenerationSnapshot, d as HostedIngestHeaders, f as HostedWireVersion, g as TraceSpanEvent, h as IngestTracesRequest, i as hostedClientFromEnv, l as EvalRunStatus, m as IngestResponse, n as HostedTenant, o as EvalRunCellScore, p as IngestEvalRunsRequest, r as createHostedClient, s as EvalRunEvent, t as HostedClient, u as HOSTED_WIRE_VERSION } from "../client-C9gzZE59.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
//#region src/hosted/schemas.d.ts
|
|
6
6
|
declare const UnixNanoTimestampSchema: z.ZodType<UnixNanoTimestamp>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as RunRecord } from "./run-record-
|
|
1
|
+
import { a as RunRecord } from "./run-record-CF4Dwpxr.js";
|
|
2
2
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
3
3
|
import { a as GatedEvidence, b as RolloutSplit, g as RolloutOutcome, h as RolloutLine, n as ChatMessage, o as MintedRolloutLine, w as ToolDef, x as RolloutStep, y as RolloutRole } from "./schema-Cef2cFmb2.js";
|
|
4
4
|
import { DatabaseSync } from "node:sqlite";
|
|
@@ -936,4 +936,4 @@ declare function parseRolloutReleaseArgs(argv: string[]): RolloutReleaseCliArgs;
|
|
|
936
936
|
declare function runRolloutReleaseCli(argv: string[]): Promise<number>;
|
|
937
937
|
//#endregion
|
|
938
938
|
export { ScoreOrigin as $, toSftRows as $t, ReleaseRowRef as A, GATE_CHECKS as At, readOpencodeSessionMessages as B, gatedEvidenceOf as Bt, scrubRolloutLine as C, HarborSubagentTrajectoryRef as Ct, FormatGateCounts as D, relabelImportedSplit as Dt, FORMAT_GATE_DISPOSITION as E, fromHarborTrajectory as Et, DEFAULT_OPENCODE_DB as F, GateCheckId as Ft, claudeProjectSlug as G, SftRow as Gt, ClaudeTranscriptRef as H, RewardRow as Ht, OpencodeSessionRow as I, GateCheckedOutcome as It, MintRolloutOptions as J, realnessLabels as Jt, findClaudeTranscripts as K, VerifiersRolloutOutput as Kt, findOpencodeSessionById as L, GateEntryPoint as Lt, gatedRolloutIds as M, GATE_POLICIES as Mt, measureFormatGate as N, GateCheck as Nt, GateDisposition as O, toHarborTrajectories as Ot, releaseRowRefs as P, GateCheckDisposition as Pt, unmintableReasons as Q, toRftItems as Qt, findOpencodeSessionsByDirectory as R, GatePolicy as Rt, scrubLines as S, HarborStepSource as St, EmittedEvidence as T, HarborTrajectory as Tt, ClaudeUsageTotals as U, RftItem as Ut, ClaudeTranscript as V, RealnessLabels as Vt, DEFAULT_CLAUDE_PROJECTS_DIR as W, SftExportOptions as Wt, RolloutScrubber as X, toRewardRows as Xt, MintRolloutResult as Y, toJsonl as Yt, mintRolloutRows as Z, toRftItem as Zt, ScrubCounts as _, HarborImageSource as _t, ScrubReport as a, trainingReward as at, defaultRolloutScrubber as b, HarborObservationResult as bt, planPushCommand as c, readRolloutJournal as ct, DatasetCardInputs as d, ATIF_SCHEMA_VERSION as dt, toVerifiersRolloutOutput as en, ScorePreference as et, FORMAT_FILES as f, FromHarborOptions as ft, SCRUB_RULES as g, HarborFinalMetrics as gt, buildDatasetCard as h, HarborContentPart as ht, RolloutReleaseCliArgs as i, scoreOrigin as it, assertGateReport as j, GATE_CHECK_IDS as jt, GateReport as k, toHarborTrajectory as kt, pushDataset as l, readRolloutLedger as lt, ReleaseFormat as m, HarborAgent as mt, BuildSummary as n, observedScore as nt, buildHfDataset as o, trainingScore as ot, RELEASE_FORMATS as p, HARBOR_IMPORT_GAP as pt, readClaudeTranscript as q, VerifiersTokenUsage as qt, ROLLOUT_RELEASE_USAGE as r, observedSplitScore as rt, parseRolloutReleaseArgs as s, appendRolloutLines as st, BuildOptions as t, toVerifiersRolloutOutputs as tn, isRealnessGated as tt, runRolloutReleaseCli as u, writeRolloutLedger as ut, ScrubRule as v, HarborMetrics as vt, scrubText as w, HarborToolCall as wt, emptyScrubCounts as x, HarborStep as xt, addScrubCounts as y, HarborObservation as yt, openOpencodeDb as z, gateErrors as zt };
|
|
939
|
-
//# sourceMappingURL=index-
|
|
939
|
+
//# sourceMappingURL=index-BnEuDAK2.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-BnEuDAK2.d.ts","names":[],"sources":["../src/rollout/exporters.ts","../src/rollout/gate-checks.ts","../src/rollout/interchange/harbor.ts","../src/rollout/ledger.ts","../src/rollout/reward.ts","../src/rollout/mint.ts","../src/rollout/readers/claude-jsonl.ts","../src/rollout/readers/opencode-sqlite.ts","../src/rollout/release/gate-report.ts","../src/rollout/release/scrub.ts","../src/rollout/release/card.ts","../src/rollout/release/hf-dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;UAiDiB;;EAEf;;;;;;EAMA;;iBAGc,eAAe,MAAM,oBAAoB;UAcxC;;EAEf;;EAEA;;;;;;;;;;;;;;;;;;;;;;KAuBU;UAEK,kCAAkC;;EAEjD,aAAa;;KAGH,mBAAmB;UAEd;EACf,UAAU;EACV;IACE;IACA;IACA;IACA;IACA;MACE;;;;;;;;;;;;;;;;;iBAkBU,UAAU,OAAO,qBAAqB,UAAS,mBAAwB;UAyBtE;;EAEf;EACA,OAAO;EACP;EACA;IACE;IACA;IACA;IACA;IACA,OAAO;MAUL;;;;;iBAMU,aACd,OAAO,qBACP,UAAS,wBACR;UAgCc;EACf;EACA;EACA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,YAAY;EACZ;EACA,SAAS;EACT,WAAW;EACX,aAAa;EACb;IACE,MAAM;IACN,QAAQ;IACR;IACA;IACA;IACA;IACA;IACA;IACA,MAAM;MACJ;;iBASU,yBAAyB,MAAM,oBAAoB;iBA+BnD,0BACd,OAAO,qBACP,UAAS,4BACR;UAoBc;;EAEf,UAAU;;EAEV;IACE;IACA;IACA;IACA;IACA;IACA,OAAO;IACP;MACE;;iBAGU,UAAU,MAAM,oBAAoB;;iBAmBpC,WACd,OAAO,qBACP,UAAS,4BACR;iBAkBa,QAAQ,MAAM;;;;;;;;;;cCpUjB;KAOD,sBAAsB;;;;;;;;;KAUtB,qBAAqB,QAAQ,kBAAkB,SAAS;;;;;;;;;;;;;;;;;UAkBnD;EACf,SAAS;;EAET;;UAOe;EACf,IAAI;;EAEJ;;EAEA,SAAS,SAAS;;;;;;;;;;;;EAYlB,WAAW;;;;;;;;;;;;;iBA6LG,gBAAgB,SAAS,cAAc;;;;;;cA6I1C,yBAAyB,KAAK,cAAc;;;;;;;;;KAe7C;WACG;;;;WAEA;WAAyB;;;;WAEzB;WAAuB;;;KAU1B,yBAAyB,KAAK,cAAc;;;;;;;;;cAU3C;;;;;;;aAWT;eAnCiB;;aAoCjB,kBAAgB;aAMhB,2BAAyB;aAMzB,qBAAmB;;;;;;;;aAenB;;;aACA,kBAAgB;aAChB,2BAAyB;aACzB;;;;;;;;;;;aASA;;;aACA;;;aACA;;;aACA;;;;;;;;;;;aASA;;;aACA;;;aACA;;;aACA;;;;;;KAKQ,8BAA8B;;;;;;iBAO1B,WAAW,SAAS,aAAa,QAAQ;;;cCxd5C;;cAGA;KAwBD;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA,SAAS;;UAGM;EACf;EACA;;EAEA,WAAW;EACX,QAAQ;;UAGO;EACf;EACA;;EAEA;EACA,QAAQ;;UAGO;EACf;EACA,mBAAmB;EACnB,0BAA0B;EAC1B,QAAQ;;UAGO;EACf,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ;;UAGO;;EAEf;EACA;EACA,QAAQ;EACR;EACA;EACA,kBAAkB;EAClB;EACA,aAAa;EACb,cAAc;EACd,UAAU;EACV;EACA;EACA,QAAQ;;UAGO;EACf;EACA;EACA;;EAEA,mBAAmB;EACnB,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ;;UAGO;EACf;EACA;;EAEA;EACA,OAAO;EACP,OAAO;EACP;EACA,gBAAgB;EAChB;EACA,wBAAwB;EACxB,QAAQ;;;;;;;;;;;;;;;;;;iBAmXM,mBAAmB,OAAO,gBAAgB;;iBAa1C,qBAAqB,OAAO,gBAAgB;UA2O3C;;EAEf,YAAY;;;;;;;;;;;iBAwGE,qBACd,YAAY,kBACZ,UAAS,oBACR;;;;;;;;;;;;;;;iBAiCa,qBACd,gBAAgB,eAChB,OAAO,eACN;;;;iBC/6BmB,mBAAmB,cAAc,OAAO,gBAAgB;;iBAOxD,mBAAmB,cAAc,OAAO,gBAAgB;;;;;;;;;;;iBAiBxD,kBAAkB,eAAe,QAAQ;;;;;;;;;;;;;;;iBAkBzC,mBAAmB,eAAe,QAAQ;;;;;;;;KCzCpD;;KAGP,SAAS,KAAK;;iBAGH,gBAAgB,QAAQ;;;;;;;;;;;;;;iBAiBxB,mBAAmB,QAAQ,QAAQ,OAAO;;;;;;;;;;;;;;;;iBAmB1C,cACd,QAAQ,QACR,SAAQ;;KAOE;;;;;;;iBAQI,YAAY,QAAQ,QAAQ,SAAQ,kBAA8B;;;;;;;;;iBAelE,cACd,QAAQ,QACR,SAAQ;;;;;;;;;;;;iBAiBM,eAAe,QAAQ;EAAW;EAAuB;;;;;KCvE7D,mBAAmB;UAEd;EACf,QAAQ;;EAER;;EAEA,OAAO;;EAEP;;EAEA,YAAY;;UAGG;EACf,MAAM;;EAEN;;;;;;;;;;;;;;iBAkLc,kBAAkB,QAAQ;;;;;;;iBA+JpB,gBACpB,SAAS,aACT,OAAO,YACP,UAAS,qBACR,QAAQ;;;cCrYE;;iBAGG,kBAAkB;UAIjB;EACf;EACA;;;iBAIoB,sBACpB,aACA,uBACC,QAAQ;UASM;EACf;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV,OAAO;;EAEP;EACA;EACA;;UAyDe;;;;;;WAMN;;;iBAgBW,qBACpB,cACA,UAAS,8BACR,QAAQ;;;cCnHE;UAEI;EACf;EACA;EACA;EACA;;EAEA;IAAS;IAAa;;EACtB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;iBAYoB,eACpB,gBACC,QAAQ;;iBAiDK,gCACd,IAAI,cACJ,oBACC;iBAOa,wBACd,IAAI,cACJ,oBACC;;;;;;;iBA2Ba,4BAA4B,IAAI,cAAc,oBAAoB;;;;KCtFtE;cAEC,yBAAyB,OAAO,eAAe;;UAQ3C;EACf;EACA;;;;;;;;;;;;EAYA;;;;;;;;EAQA;;;;;;;;;;;;EAYA;;;UAIe;;EAEf;EACA;;UAGe;;EAEf;;EAEA;;;;;;EAMA;;EAEA;;;;;;;;;;;;;;EAcA,oBAAoB;;;;;;;;;;;;EAYpB;;EAEA;;;;;;;;;;EAUA,wBAAwB;;UAGT;;EAEf;EACA,UAAU,QAAQ,OAAO,eAAe;;;iBAI1B,gBAAgB,gBAAgB,sBAAsB;;;;;;;cAUzD;EAIX,MAAG,eAAkB,aAAW;EAShC,YAAS,eAAkB,6BAA2B;EAStD,MAAG,eAAkB,cAAY;EAOjC,MAAG,gBAAmB,wBAAsB;;;iBAgC9B,kBACd,OAAO,qBACP,eAAe,kBACd;;;;;;;;;;;;;;;;iBA+Fa,iBAAiB,QAAQ;;;UCrUxB;EACf;EACA,SAAS;;EAET,UAAU,eAAe;;cAGd,sBAAsB;;KAkEvB,cAAc;iBAEV,oBAAoB;iBAIpB,eAAe,MAAM,aAAa,MAAM,cAAc;iBAKtD,UAAU,cAAc,QAAQ;;;;;;;;;iBA8BhC,iBAAiB,MAAM,mBAAmB,QAAQ,cAAc;iBAIhE,WAAW,OAAO;EAChC,OAAO;EACP,QAAQ;;;;;;;;;iBAaM,uBAAuB;;;cCrI1B;KACD,wBAAwB;;cAGvB,cAAc,OAAO;UAejB;;EAEf,OAAO;EACP,SAAS;EACT;;EAEA;EACA,aAAa;EACb;IAAY;IAAmB;;EAC/B,cAAc,QAAQ,OAAO;;;;;;;EAO7B,MAAM;;iBA4GQ,iBAAiB,QAAQ;;;UC1HxB;EACf;EACA,SAAS;EACT;;UAGe;;EAEf,OAAO,eAAe;EACtB,QAAQ;EACR;IAAY;IAAmB;;;UAGhB;EACf;EACA;EACA;EACA,OAAO;EACP,cAAc,QAAQ,OAAO;;EAE7B,MAAM;EACN;;iBAGoB,eACpB,kBACA,SAAS,eACR,QAAQ;iBAuGK,gBAAgB,cAAc;iBAI9B,YAAY,cAAc;UAgBzB,8BAA8B;EAC7C;EACA;;cAGW;iBAGG,wBAAwB,iBAAiB;;iBA0CnC,qBAAqB,iBAAiB"}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { l as ValidationError, t as AgentEvalError } from "./errors-CKPfb2aH.js";
|
|
2
2
|
import { T as RunPaidCallInput, m as CostReceipt, p as CostProvenance } from "./cost-ledger-Bv_e8XHY.js";
|
|
3
|
-
import { a as RunRecord, s as RunSplitTag } from "./run-record-
|
|
4
|
-
import { f as AnalystUsageReceipt, i as AnalystFinding, l as AnalystRunResult, w as TraceAnalysisStore } from "./types-
|
|
5
|
-
import { n as CompletionVerdict, o as ProducedState, r as CorrectnessChecker, t as CompletionRequirement } from "./completion-verifier-
|
|
6
|
-
import { D as AnalystIssueExpectation, d as AnalystBenchmarkCase, h as AnalystBenchmarkLabelState, t as AgentProfile$1 } from "./agent-profile-
|
|
7
|
-
import { A as LabeledScenarioWrite, D as LabeledScenarioSampleArgs, E as LabeledScenarioRecord, O as LabeledScenarioSource, R as Scenario, S as JudgeConfig, T as LabelTrust, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, l as CodeSurface, p as Gate, u as ComponentSurface } from "./types-
|
|
3
|
+
import { a as RunRecord, s as RunSplitTag } from "./run-record-CF4Dwpxr.js";
|
|
4
|
+
import { f as AnalystUsageReceipt, i as AnalystFinding, l as AnalystRunResult, w as TraceAnalysisStore } from "./types-BvDKaULh.js";
|
|
5
|
+
import { n as CompletionVerdict, o as ProducedState, r as CorrectnessChecker, t as CompletionRequirement } from "./completion-verifier-DJA5BhPb.js";
|
|
6
|
+
import { D as AnalystIssueExpectation, d as AnalystBenchmarkCase, h as AnalystBenchmarkLabelState, t as AgentProfile$1 } from "./agent-profile-DPi7IZg7.js";
|
|
7
|
+
import { A as LabeledScenarioWrite, D as LabeledScenarioSampleArgs, E as LabeledScenarioRecord, O as LabeledScenarioSource, R as Scenario, S as JudgeConfig, T as LabelTrust, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, l as CodeSurface, p as Gate, u as ComponentSurface } from "./types-BnjdJ70P.js";
|
|
8
8
|
import { y as PairedBootstrapResult } from "./statistics-D6Uebe_4.js";
|
|
9
|
-
import { Ht as CampaignStorage, It as PlanCampaignRunOptions, Pt as CampaignRunPlan } from "./skillopt-optimization-method-
|
|
10
|
-
import { N as PairedArmsComparison } from "./statistical-heldout-
|
|
11
|
-
import "./promotion-policy-
|
|
9
|
+
import { Ht as CampaignStorage, It as PlanCampaignRunOptions, Pt as CampaignRunPlan } from "./skillopt-optimization-method-D8wysvU1.js";
|
|
10
|
+
import { N as PairedArmsComparison } from "./statistical-heldout-TQ-4CYiN.js";
|
|
11
|
+
import "./promotion-policy-Ckjhzg_4.js";
|
|
12
12
|
import { D as LedgerHash, m as LedgerTrustedHeadRemoval, p as LedgerTrustedHead } from "./index-CvSN3IG1.js";
|
|
13
13
|
import { AgentProfile } from "@tangle-network/agent-interface";
|
|
14
14
|
//#region src/campaign/analyst-surface.d.ts
|
|
@@ -1512,4 +1512,4 @@ declare function verifyCodeSurface(surface: CodeSurface, worktreeDir?: string):
|
|
|
1512
1512
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
1513
1513
|
//#endregion
|
|
1514
1514
|
export { SearchPlannedEvent as $, CrossSurfacePairIncompatibilityReason as $n, ToolCallEventLike as $t, SearchAccountingAudit as A, CrossSurfaceAttemptCompleteness as An, UserStory as At, SearchCostAccounting as B, CrossSurfaceCompositionStep as Bn, RunProfileMatrixOptions as Bt, surfaceHash as C, loadEvalFixture as Cn, tangleTracesRoot as Ct, FileSearchLedger as D, AnalyzeCrossSurfaceInteractionsInput as Dn, ScoreboardRenderOptions as Dt, acquireSingleRunLock as E, analyzeCrossSurfaceInteractions as En, PlaybackStep as Et, SearchCandidateRegisteredEvent as F, CrossSurfaceCandidateEvidence as Fn, scoreboardSummary as Ft, SearchLedgerEvent as G, CrossSurfaceInteractionAwareSelection as Gn, BackendIntegrityReport as Gt, SearchLedger as H, CrossSurfaceEligibility as Hn, ScenarioRollup as Ht, SearchCandidateSlot as I, CrossSurfaceCandidateOutcome as In, userStoryScoreboard as It, SearchLedgerTrustedHeadMode as J, CrossSurfaceInteractionReport as Jn, summarizeAgentReceiptIntegrity as Jt, SearchLedgerHash as K, CrossSurfaceInteractionEffect as Kn, assertRealAgentReceipts as Kt, SearchCandidateSlotClosedEvent as L, CrossSurfaceCandidateSummary as Ln, ProfileDispatchFn as Lt, SearchAttemptAccounting as M, CrossSurfaceBootstrapPolicy as Mn, makePlaybackDispatch as Mt, SearchCandidateDecidedEvent as N, CrossSurfaceCandidate as Nn, renderScoreboardMarkdown as Nt, OpenSearchLedgerOptions as O, CrossSurfaceAdditionDecision as On, ScoreboardRow as Ot, SearchCandidateLineage as P, CrossSurfaceCandidateComparison as Pn, scoreUserStory as Pt, SearchPlan as Q, CrossSurfacePairEvidence as Qn, RuntimeEventLike as Qt, SearchCandidateSurface as R, CrossSurfaceComponent as Rn, ProfileMatrixError as Rt, surfaceContentHash as S, discoverEvalFixtures as Sn, resolveRunDir as St, SingleRunLockOptions as T, planEvalFixtureRun as Tn, PlaybackDriver as Tt, SearchLedgerAppendResult as U, CrossSurfaceEvidenceBreakdown as Un, runProfileMatrix as Ut, SearchFailureReason as V, CrossSurfaceDistribution as Vn, RunProfileMatrixResult as Vt, SearchLedgerEntry as W, CrossSurfaceIneligibilityReason as Wn, BackendIntegrityError as Wt, SearchOperationKind as X, CrossSurfaceNaiveStackSelection as Xn, ArtifactEventLike as Xt, SearchModelIdentity as Y, CrossSurfaceInteractionTask as Yn, summarizeBackendIntegrity as Yt, SearchOperationRecordedEvent as Z, CrossSurfacePairCompatibility as Zn, ProposalEventLike as Zt, assertCodeSurfaceIdentity as _, EvalFixtureRunPlan as _n, compareRankKeys as _t, WorktreeAdapterError as a, RolloutArgumentDiff as an, CrossSurfaceTaskRow as ar, SearchSurfaceKind as at, componentSurfaceIdentityMaterial as b, LoadEvalFixtureScenariosOptions as bn, scoreDiscrimination as bt, verifyCodeSurface as c, ScoredRollout as cn, TraceAnalystScenario as cr, SearchTokenAccounting as ct, PhoenixEvaluationResultLike as d, rolloutArgumentDiff as dn, SearchLedgerConflictError as dt, extractProducedState as en, CrossSurfacePairwiseEntry as er, SearchPlannedOperation as et, PhoenixEvaluatorLike as f, NeutralizationGateOptions as fn, SearchLedgerError as ft, isTransientTransportFailure as g, EvalFixtureLoadOptions as gn, campaignMeanComposite as gt, TransientFailureOptions as h, EvalFixtureFile as hn, campaignBreakdown as ht, WorktreeAdapter as i, LabeledScenarioStoreError as in, CrossSurfaceSelections as ir, SearchSurfaceEvidence as it, SearchArtifactRef as j, CrossSurfaceBestSingleSelection as jn, UserStoryVerdict as jt, SEARCH_LEDGER_SCHEMA as k, CrossSurfaceAdditionRejectionReason as kn, ScoreboardSummary as kt, AutoevalsScoreLike as l, UngroundedLiteralReport as ln, buildTraceAnalystSurfaceDispatch as lr, openSearchLedger as lt, phoenixEvaluatorJudge as m, EvalFixture as mn, CampaignBreakdown as mt, GitWorktreeAdapterOptions as n, FsLabeledScenarioStore as nn, CrossSurfaceRelativeCost as nr, SearchSourceRef as nt, gitWorktreeAdapter as o, RolloutArgumentDiffOptions as on, BuildTraceAnalystSurfaceDispatchOptions as or, SearchTaskAttemptedEvent as ot, autoevalsScorerJudge as p, neutralizationGate as pn, SearchLedgerIntegrityError as pt, SearchLedgerReplay as q, CrossSurfaceInteractionPath as qn, assertRealBackend as qt, Worktree as r, FsLabeledScenarioStoreOptions as rn, CrossSurfaceSelectionPolicy as rr, SearchSurfaceEffect as rt, resolveWorktreePath as s, RolloutCall as sn, TraceAnalystArtifact as sr, SearchTaskOutcome as st, CodeSurfaceVerification as t, neutralizeText as tn, CrossSurfaceRankedSingle as tr, SearchPlannedTask as tt, AutoevalsScorerLike as u, classifyUngroundedLiterals as un, traceAnalystQualityJudge as ur, validateSearchLedgerEvent as ut, assertComponentSurface as v, EvalFixtureScenario as vn, DiscriminationScore as vt, SingleRunLock as w, loadEvalFixtureScenarios as wn, PlaybackContext as wt, renderSurfaceDiff as x, PlanEvalFixtureRunOptions as xn, selectDiscriminative as xt, codeSurfaceIdentityMaterial as y, EvalFixtureValidationMode as yn, ScenarioSignal as yt, SearchCompletedEvent as z, CrossSurfaceComponentEvidence as zn, ProfileSummary as zt };
|
|
1515
|
-
//# sourceMappingURL=index-
|
|
1515
|
+
//# sourceMappingURL=index-CBEMKO8d.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-CBEMKO8d.d.ts","names":[],"sources":["../src/campaign/analyst-surface.ts","../src/campaign/cross-surface-types.ts","../src/campaign/cross-surface-interaction.ts","../src/campaign/fixtures.ts","../src/campaign/gates/neutralization-gate.ts","../src/campaign/grounded-reflection.ts","../src/campaign/labeled-store/fs-adapter.ts","../src/campaign/neutralize.ts","../src/produced-state.ts","../src/integrity/backend-integrity.ts","../src/campaign/presets/run-profile-matrix.ts","../src/campaign/presets/playback.ts","../src/campaign/run-dir.ts","../src/campaign/scenario-selection.ts","../src/campaign/score-utils.ts","../src/campaign/search-ledger-errors.ts","../src/campaign/search-ledger.ts","../src/campaign/single-run-lock.ts","../src/campaign/surface-identity.ts","../src/campaign/transient-failure.ts","../src/campaign/upstream-evaluators.ts","../src/campaign/worktree/index.ts"],"mappings":";;;;;;;;;;;;;;UAWiB,6BAA6B;EAC5C;EACA,YAAY;EACZ,YAAY;EACZ,yBAAyB;EACzB,kBAAkB;;UAGH;EACf,mBAAmB;EACnB,QAAQ;EACR,MAAM;;UAGS;EACf,QAAQ;IACN;IACA,YAAY;IACZ;IACA,QAAQ;MACN,QAAQ;;iBAGE,iCACd,SAAS,2CAET,SAAS,gBACT,UAAU,sBACV,SAAS,oBACN,QAAQ;iBAgBG,4BAA4B,YAC1C,sBACA;;;;KCtDU;;UAGK;EACf;EACA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;;;UAIe;EACf;;EAEA;;EAEA;;;;;;;UAQe;EACf;EACA;;EAEA;EACA,cAAc;EACd;EACA;;;;;;EAMA,MAAM;EACN,mBAAmB;;EAEnB;;UAGe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;;EAEA,kCAAkC;;EAElC;;UAGe,qCACf,aAAa,sBAAsB;EAEnC,qBAAqB;EACrB,qBAAqB;EACrB,eAAe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf,aAAa;EACb;EACA;EACA;EACA;;KAGU;UAWK;EACf;EACA,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX,SAAS;EACT,OAAO;EACP,OAAO,eAAe;EACtB,QAAQ;EACR,QAAQ;;EAER,sBAAsB;;EAEtB,aAAa;;UAGE;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ;EACR,cAAc,eAAe;;UAGd;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;EACA;;UAGe;EACf,SAAS;EACT;EACA;EACA;EACA;EACA,eAAe;EACf,gBAAgB;;KAGN;UAYK;EACf;EACA,SAAS;EACT;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,4BAA4B,iCAAiC;EAC7D,wBAAwB,eAAe;EACvC,QAAQ;EACR,QAAQ;EACR,aAAa;EACb,eAAe;;UAGA;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,SAAS;;UAGM;;EAEf;EACA;;KAGU;UAaK;EACf;EACA;EACA;EACA;EACA;EACA,uBAAuB;EACvB;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,YAAY;EACZ;;;UAIe;EACf;EACA;EACA;EACA;EACA,OAAO;;UAGQ;;EAEf;;EAEA;EACA;;EAEA;EACA;;EAEA,gBAAgB;;EAEhB,OAAO;;UAGQ;EACf,YAAY;EACZ,YAAY;EACZ,kBAAkB;;UAGH,8BACf,aAAa,sBAAsB;EAEnC;EACA;EACA;EACA;;EAEA,MAAM;EACN,iBAAiB;EACjB,iBAAiB;EACjB,YAAY;EACZ,UAAU;EACV,YAAY;;;;;;;;;iBCnRE,gCAAgC,aAAa,qBAC3D,OAAO,qCAAqC,QAC3C,8BAA8B;;;KC7CrB;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP;;UAGe,4BAA4B;EAC3C;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf,aAAa;;EAEb;;UAGe,wCAAwC;EACvD;;UAGe,0BAA0B,6BACjC,KACN,uBAAuB,qBAAqB;EAG9C;EACA,aAAa;EACb;EACA;EACA,UAAU;;KAGA,qBAAqB;EAC/B,UAAU,MAAM,KAAK;;;iBAIP,qBAAqB;;;;;iBA4BrB,gBACd,kBACA,cACA,UAAS,yBACR;;iBA4Ca,yBACd,kBACA,UAAS,kCACR;;;;;iBAuBa,mBAAmB,qBACjC,SAAS,0BAA0B,aAClC;;;UChJc,0BAA0B,kBAAkB,WAAW;EACtE,WAAW;;;;;EAKX;;;;;;iBAuBc,mBAAmB,WAAW,kBAAkB,UAC9D,SAAS,0BAA0B,aAClC,KAAK,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UC9BF;WACN;WACA,MAAM,SAAS;;;UAIT;;WAEN;;WAEA;WACA,gBAAgB;;UAGV;;WAEN;;WAEA;;UAGM;;WAEN;;WAEA,eAAe;;WAEf,eAAe;;;;;;;;iBASV,oBACd,mBAAmB,iBACnB,OAAM,6BACL;UAsCc;;WAEN;;WAEA;;;;;;;;;iBAUK,2BACd,cACA,MAAM,KAAK,0DACV;;;UCjFc;;EAEf;;;EAGA;;EAEA;;;cAIW,kCAAkC;WAE3B;EADlB,YACkB,cAChB;;;;;;cAiBS,kCAAkC;mBAIhB;mBAHZ;mBACA;EAEjB,YAA6B,SAAS;EAKhC,QAAQ,OAAO,uBAAuB;EAYtC,OAAO,MAAM,4BAA4B,QAAQ;EA8CjD,QAAQ;IACZ;IACA;IACA,UAAU;IACV,SAAS,OAAO;;UAkCV;UAiCA;UAmBA;UAkBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBCtNM,eAAe;;;;UCZd;EACf;EACA;;;;;;;UAQe;EACf;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EAIA;;;;;;;;KASU,mBACR,oBACA,oBACA;EACE;;;;;;;;;;;;iBAmBU,qBAAqB,iBAAiB,qBAAqB;;;UCpD1D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;cAQW,8BAA8B;WAGvB,QAAQ;EAF1B,YACE,iBACgB,QAAQ;;;;;;;iBAWZ,0BACd,SAAS,cAAc,aACtB;;iBAWa,+BACd,UAAU,cAAc,eACvB;;;;;;;;iBA0Ga,kBACd,SAAS,cAAc,YACvB;EAAQ;IACP;;iBAMa,wBACd,UAAU,cAAc,cACxB;EAAQ;IACP;;;;;;cC7HU,2BAA2B;EACtC,YAAY;;;;;KAQF,kBAAkB,kBAAkB,UAAU,cACxD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;UAEI,wBAAwB,kBAAkB,UAAU;;EAEnE,UAAU;;EAEV,WAAW;;EAEX,UAAU,kBAAkB,WAAW;;EAEvC,SAAS,YAAY,WAAW;;;EAGhC;;;EAGA;;;EAGA;;;;EAIA;;EAEA,WAAW;;EAEX;;EAEA;;;;;;;;EAQA;;;EAGA;;EAEA;;;EAGA;;EAEA;;EAEA,eAAe;EACf,gBAAgB;;;EAGhB,UAAU;;EAEV,YAAY;;;EAGZ,aAAa,UAAU;;;;EAIvB;;;;;;;EAOA,cACE,UAAU,WACV,UAAU;IACL;IAAgB;;;UAGR;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA,gBAAgB;;;EAGhB,WAAW;;UAGI;EACf;EACA;;UAGe,uBAAuB,WAAW,kBAAkB;EACnE;EACA;;;;EAIA,SAAS;EACT,WAAW,eAAe;EAC1B,YAAY,eAAe;;EAE3B,YAAY,eAAe;;EAE3B,WAAW;;EAEX,WAAW,eAAe,eAAe,WAAW;;;;;iBA2IhC,iBAAiB,kBAAkB,UAAU,WACjE,MAAM,wBAAwB,WAAW,aACxC,QAAQ,uBAAuB,WAAW;;;;;UCzT5B;;EAEf;;EAEA,UAAU;;;;;;;UAQK,kBAAkB;;EAEjC;;EAEA,OAAO;;EAEP,cAAc;;;UAIC,wBAAwB;EACvC,SAAS;;;;;;;;;;;UAYM,eAAe,eAAe,YAAY;EACzD,IAAI,OAAO,QAAQ,KAAK,kBAAkB,iBAAiB;;;;;;;iBAQ7C,qBAAqB,eAAe,WAClD,QAAQ,eAAe,UACtB,kBAAkB,QAAQ;;UAQZ,yBAAyB;EACxC;;;;;;;;iBASoB,eACpB,OAAO,WACP,OAAO,eACP,kBAAkB,qBACjB,QAAQ;;UAUM;EACf;EACA;EACA;EACA;EACA;EACA;;;;;;;iBAQc,oBAAoB,mBAAmB,qBAAqB;;UAkB3D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;iBAIc,kBAAkB,eAAe,kBAAkB;UAwBlD;;EAEf;;EAEA,OAAO;;EAEP;;;;;;;;;iBAkBc,yBACd,eAAe,iBACf,OAAM;;;;;;;;iBCnMQ;;;;;iBAQA,cAAc,gBAAgB;;;;;;;;;;;;;;;;;UCD7B;EACf;;EAEA;;UAGe;EACf;;EAEA;;EAEA;EACA;;EAEA;;;;;;;;;;iBA4Cc,oBACd,SAAS,kBACT;EAAS;IACR;;;;;;;;;;;iBA0Ba,qBACd,SAAS,kBACT,WACA;EAAS;;;;;;;iBC3FK,sBAAsB,WAAW,kBAAkB,UACjE,UAAU,eAAe,WAAW;;;;iBA0CtB,gBAAgB,sBAAsB;UAWrC;;EAEf,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;;iBAK5D,kBAAkB,WAAW,kBAAkB,UAC7D,UAAU,eAAe,WAAW,aACnC;;;;cC/EU,0BAA0B;;cAG1B,mCAAmC;;cAGnC,kCAAkC;;;cCgClC;KAED,mBAAmB;KAEnB;;;UAYK;EACf;EACA;EACA,QAAQ;EACR;;;;UAKe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,MAAM;EACN,UAAU;;UAGK;;EAEf;EACA;EACA;EACA;EACA,gBAAgB;;KAGN;UAOK;EACf;EACA,QAAQ;EACR,WAAW;;;EAGX;;UAGe;EACf;EACA,MAAM;;UAGS;EACf;;;EAGA;;UAGe;;EAEf,gBAAgB;;EAEhB,OAAO;;EAEP,YAAY;;KAGF;EAEN;EACA;EACA;EACA;;EAGA;EACA;;KAGM;EAEN;EACA;EACA;;EAGA;;EAEA;EACA;;UAGW;EACf,QAAQ;EACR,MAAM;;UAGS;EACf;EACA;;KAGU;EAEN;EACA;EACA,SAAS;;EAGT;EACA;EACA,SAAS;EACT,SAAS;;EAGT;EACA,SAAS;EACT,OAAO;IAAwB;;;KAGzB;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;;;;UAKW;EACf;EACA;EACA;EACA,QAAQ;EACR,UAAU;;UAGF;EACR;EACA;EACA,WAAW;;UAGI,2BAA2B;EAC1C;EACA,MAAM;;UAGS,uCAAuC;EACtD;EACA;EACA;EACA;EACA,SAAS;EACT,UAAU;;UAGK,uCAAuC;EACtD;EACA;EACA;EACA,QAAQ;;UAGO,iCAAiC;EAChD;EACA;EACA;EACA;EACA;IACE;IACA,QAAQ;;EAEV;IACE,OAAO;IACP,OAAO;IACP,WAAW;;EAEb,SAAS;EACT,YAAY;EACZ,iBAAiB;;UAGF,qCAAqC;EACpD;EACA;EACA,eAAe;EACf;IAEM;IACA,OAAO;IACP,QAAQ;;IAGR;IACA,QAAQ;;EAEd;IACM;;IACA;IAAmB,SAAS;;IAC5B;IAAkB,SAAS;;EACjC,YAAY;;UAGG,oCAAoC;EACnD;EACA;EACA;IACM;;IAEA;IACA,QAAQ;;;UAIC,6BAA6B;EAC5C;EACA;IAEM;IACA;;IAGA;IACA,QAAQ;;;KAIJ,oBACR,qBACA,iCACA,iCACA,2BACA,+BACA,8BACA;UAEa;EACf,eAAe;EACf;EACA;EACA,cAAc;EACd,OAAO;EACP,WAAW;;KAGD;EAEN;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;EACA;EACA;EACA;IAAY;IAAgB;IAAgB;;EAC5C;IAAqB;IAAmB;IAAiB;;EACzD;IAAa;IAAkB;IAAkB;;EACjD;IACE;IACA;IACA;IACA;IACA;IACA;;EAEF;EACA;EACA,YAAY;EACZ,UAAU;;UAGK;EACf,SAAS;EACT,MAAM;EACN,YAAY;EACZ,sBAAsB;EACtB,UAAU;EACV,YAAY;EACZ,WAAW;EACX,YAAY;EACZ,OAAO;;UAGQ;EACf,OAAO;;EAEP;EACA,QAAQ;;;;iBA+ZM,0BAA0B,iBAAiB;;;;;;;;;;;;;;;KAsB/C;UAEK;EACf;EACA;EACA,cAAc;;UAGC;WACN;WACA;;WAEA;EACT,OAAO,OAAO,oBAAoB,QAAQ;EAC1C,UAAU,QAAQ;;EAElB,eAAe,QAAQ;;;EAGvB,kBAAkB,QAAQ;;;;;;;EAO1B,oBAAoB,QAAQ;;;;iBAKd,iBAAiB,SAAS,0BAA0B;;cA8BvD,4BAA4B;WAC9B;WACA;WACA;mBACQ;mBACA;EAMjB,YAAY,cAAc,oBAAoB,cAAa;EAerD,UAAU,QAAQ;EAIlB,OAAO,OAAO,oBAAoB,QAAQ;EAU1C,eAAe,QAAQ;EAIvB,kBAAkB,QAAQ;EAI1B,oBAAoB,QAAQ;;;;;;;;;;;;;;;;;;;UC/3BnB;;WAEN;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf;;;;;;;iBAwBc,qBAAqB,MAAM,uBAAuB;;;;iBCpDlD,0BAA0B,2BAA2B,WAAW;;iBAmChE,uBAAuB,2BAA2B,WAAW;;iBA8B7D,iCAAiC,SAAS;;;;iBAa1C,4BAA4B,SAAS;;iBAgBrC,mBAAmB,SAAS;;iBAW5B,YAAY,SAAS;;iBAKrB,kBACd,eAAe,gBACf,iBAAiB;;;;;;;;;;;;;;;;;;;;;UCpGF;;;;;;;WAON;;WAEA,yBAAyB;;;;;;iBAYpB,4BACd,oCACA,OAAM;;;UCnCS;EACf;EACA;EACA;;UAGe,qBAAqB,gBAAgB;EACpD;EACA;EACA;EACA,SACE,QAAQ,SACR,SAAS,4BACR,QAAQ;;UAGI;EACf;EACA;EACA,WAAW;;KAGD,oBAAoB,eAAe,4BAC7C,OAAO,QACP,SAAS,8BACN,qBAAqB,QAAQ;UAEjB;WACN,QAAQ;WACR;;KAGN,sBAAsB,WAAW,KACpC,iBAAiB;EAGjB;;UAGQ,qBAAqB,kBAAkB;EAC/C;EACA;EACA;EACA,aAAa,UAAU;;EAEvB,eAAe;;iBAGD,sBACd,gBAAgB,yBAChB,WACA,kBAAkB,WAAW,UAE7B,WAAW,qBAAqB,UAChC,SAAS,qBAAqB;EAC5B,SAAS;IAAS,UAAU;IAAW,UAAU;MAAc;EAC/D,WAAW,sBAAsB;IAElC,YAAY,WAAW;iBA8CV,qBACd,eAAe,yBACf,WACA,kBAAkB,WAAW,UAE7B,QAAQ,oBAAoB,SAC5B,SAAS,qBAAqB;EAC5B;EACA,SAAS;IAAS,UAAU;IAAW,UAAU;MAAc;;EAEzD;EAAc;;EACd;EAAa,UAAU,sBAAsB;KAEpD,YAAY,WAAW;;;KCxFrB,qBAAqB;KACrB,iBAAiB,SAAS;KAC1B,aAAa,gBAAgB,aAAa,MAAM,mBAAmB;UAcvD;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf,OAAO;IAAQ;IAAiB;MAAkB,QAAQ;;;EAG1D,SAAS,UAAU,UAAU,kBAAkB,QAAQ;;EAEvD,QAAQ,UAAU,WAAW;;;cAIlB,6BAA6B;WAG7B;EAFX,YACE,iBACS;;UAOI;;EAEf;;EAEA;;EAEA;;;;EAIA,MAAM;;UAobS;;EAEf;;EAEA;;EAEA;;;EAGA,YAAY;;;;;iBAoJE,mBAAmB,MAAM,4BAA4B;;;;iBA0FrD,kBACd,SAAS,aACT,uBACC;;;iBAOa,oBAAoB,SAAS,aAAa"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { s as RunSplitTag } from "./run-record-
|
|
2
|
-
import { R as Scenario, a as CampaignResult, d as DispatchContext } from "./types-
|
|
3
|
-
import { Ht as CampaignStorage } from "./skillopt-optimization-method-
|
|
4
|
-
import "./index-
|
|
1
|
+
import { s as RunSplitTag } from "./run-record-CF4Dwpxr.js";
|
|
2
|
+
import { R as Scenario, a as CampaignResult, d as DispatchContext } from "./types-BnjdJ70P.js";
|
|
3
|
+
import { Ht as CampaignStorage } from "./skillopt-optimization-method-D8wysvU1.js";
|
|
4
|
+
import "./index-CBEMKO8d.js";
|
|
5
5
|
//#region src/benchmarks/types.d.ts
|
|
6
6
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
7
7
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -332,4 +332,4 @@ declare namespace index_d_exports {
|
|
|
332
332
|
}
|
|
333
333
|
//#endregion
|
|
334
334
|
export { BenchmarkMetricCalibrationOptions as A, BenchmarkSource as B, BenchmarkRunOptions as C, runBenchmarkAdapter as D, renderBenchmarkReportMarkdown as E, BenchmarkDatasetItem as F, deterministicSplit as H, BenchmarkEvaluation as I, BenchmarkFamily as L, calibrateBenchmarkMetric as M, BENCHMARK_SPLIT_SEED as N, summarizeBenchmarkCampaign as O, BenchmarkAdapter as P, BenchmarkResponder as R, BenchmarkReport as S, BenchmarkSliceSummary as T, BenchmarkTaskKind as V, parseJsonlRows as _, StandardRetrievalDocument as a, retrievalMetricsAtCutoff as b, StandardRetrievalQrel as c, buildStandardRetrievalItems as d, createRetrievalIdBenchmarkAdapter as f, parseBeirQueriesJsonl as g, parseBeirCorpusJsonl as h, StandardRetrievalArtifact as i, BenchmarkMetricCalibrationResult as j, index_d_exports$1 as k, StandardRetrievalQuery as l, normalizeRetrievedDocumentIds as m, BuildStandardRetrievalItemsOptions as n, StandardRetrievalEvaluationOptions as o, evaluateStandardRetrieval as p, RetrievalIdAdapterOptions as r, StandardRetrievalPayload as s, index_d_exports as t, StandardRetrievalResult as u, parseQrels as v, BenchmarkRunResult as w, BenchmarkDistribution as x, parseTsvRows as y, BenchmarkScenario as z };
|
|
335
|
-
//# sourceMappingURL=index-
|
|
335
|
+
//# sourceMappingURL=index-CLKMJTJF2.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-CLKMJTJF2.d.ts","names":[],"sources":["../src/benchmarks/types.ts","../src/benchmarks/calibration.ts","../src/benchmarks/routing/dataset.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/runner.ts","../src/benchmarks/standard-formats.ts","../src/benchmarks/index.ts"],"mappings":";;;;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC;;;UCjJc,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;iBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;UCFM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;;;UClCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;iBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;iBAmGxB,2BAA2B,UAAU,WAAW;EAC9D,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;IACpD;iBA6CY,8BAA8B,QAAQ;;;UCpOrC;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;iBAGc,eAAe,aAAa,eAAe;iBAc3C,aAAa;iBASb,WAAW,eAAe;iBAgB1B,qBAAqB,eAAe;iBAgBpC,sBAAsB,eAAe;iBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;iBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;iBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS;;;;;;;;;;iBAsCK,8BACd,UAAU,2BACV,oBAAmB;iBA0BL,yBAAyB;EACvC;EACA,gBAAgB;EAChB;IACE"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { u as RunTokenUsage } from "./run-record-
|
|
1
|
+
import { u as RunTokenUsage } from "./run-record-CF4Dwpxr.js";
|
|
2
2
|
import { Ft as RawProviderSink, Mt as ProviderRedactor } from "./types-D216SgwM.js";
|
|
3
3
|
//#region src/trace/extract-usage.d.ts
|
|
4
4
|
type ExtractedUsage = RunTokenUsage;
|
|
@@ -65,4 +65,4 @@ interface CaptureFetchOptions {
|
|
|
65
65
|
declare function captureFetchToRawSink(fetch: typeof globalThis.fetch, sink: RawProviderSink, ctx: CaptureFetchContext, opts?: CaptureFetchOptions): typeof globalThis.fetch;
|
|
66
66
|
//#endregion
|
|
67
67
|
export { ExtractedUsage as a, extractUsageFromResponse as c, ExtractUsageFromSseOptions as i, extractUsageFromSse as l, CaptureFetchOptions as n, SseUsageMode as o, captureFetchToRawSink as r, extractUsage as s, CaptureFetchContext as t };
|
|
68
|
-
//# sourceMappingURL=index-
|
|
68
|
+
//# sourceMappingURL=index-CQTZ-4XN.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-CQTZ-4XN.d.ts","names":[],"sources":["../src/trace/extract-usage.ts","../src/trace/capture-fetch.ts"],"mappings":";;;KAgBY,iBAAiB;KACjB;UAEK;;EAEf,OAAO;;;;;;;iBAuDO,aAAa,gBAAgB;;;;;;iBA6C7B,oBACd,cACA,UAAS,6BACR;;;;;;;iBA4DmB,yBACpB,UAAU,UACV,aAAa,6BACZ,QAAQ;;;UCrKM;;;EAGf;;EAEA;;;EAGA;;EAEA;;EAEA;;UAGe;;EAEf,WAAW;;;EAGX;;;EAGA;;;;;;;;EAQA,WAAW,OAAO,gBAAgB,KAAK;;EAEvC,eAAe;;iBAkDD,sBACd,cAAc,WAAW,OACzB,MAAM,iBACN,KAAK,qBACL,OAAM,6BACE,WAAW"}
|