@tangle-network/agent-eval 0.173.3 → 0.175.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +1 -1
- package/dist/{proposal-findings-bko3GGy-.js → abort-signal-CtzAM_sJ.js} +11 -11
- package/dist/abort-signal-CtzAM_sJ.js.map +1 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/agent-profile-_xPxqVJt.d.ts +488 -0
- package/dist/agent-profile-_xPxqVJt.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +7 -9
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +8 -8
- package/dist/{benchmark-C4wk_Sjr.js → benchmark-DQKzykkO.js} +2 -2
- package/dist/{benchmark-C4wk_Sjr.js.map → benchmark-DQKzykkO.js.map} +1 -1
- package/dist/{benchmark-command-BY9oscke.js → benchmark-command-D_5xG9LG.js} +13 -13
- package/dist/{benchmark-command-BY9oscke.js.map → benchmark-command-D_5xG9LG.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -4
- package/dist/benchmarks/index.d.ts.map +1 -1
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -9
- package/dist/campaign/index.js +7 -7
- package/dist/{campaign-B3kPMU8S.js → campaign-BzMSCejE.js} +8 -8
- package/dist/{campaign-B3kPMU8S.js.map → campaign-BzMSCejE.js.map} +1 -1
- package/dist/{opencode-sqlite-eK6HW6dr.js → claude-jsonl-CxZZrDJ3.js} +9 -149
- package/dist/claude-jsonl-CxZZrDJ3.js.map +1 -0
- package/dist/cli.js +9 -2
- package/dist/cli.js.map +1 -1
- package/dist/{client-DlqdbM7n.d.ts → client-vyYQg3bm.d.ts} +2 -2
- package/dist/{client-DlqdbM7n.d.ts.map → client-vyYQg3bm.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -10
- package/dist/contract/index.js +8 -8
- package/dist/{default-registry-B0bKikCb.js → default-registry-DBqVI4pq.js} +5 -5
- package/dist/{default-registry-B0bKikCb.js.map → default-registry-DBqVI4pq.js.map} +1 -1
- package/dist/{default-registry-BKwc8bN5.d.ts → default-registry-FfNzaUHV.d.ts} +3 -3
- package/dist/{default-registry-BKwc8bN5.d.ts.map → default-registry-FfNzaUHV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CY6qdlGV.d.ts → define-agent-eval-V1jQyCDR.d.ts} +102 -11
- package/dist/define-agent-eval-V1jQyCDR.d.ts.map +1 -0
- package/dist/{define-agent-eval-8h3lXXee.js → define-agent-eval-ox5McL6e.js} +331 -144
- package/dist/define-agent-eval-ox5McL6e.js.map +1 -0
- package/dist/{dspy-rlm-engine-CF0t2ITD.js → dspy-rlm-engine-Caz2pl4L.js} +3 -3
- package/dist/{dspy-rlm-engine-CF0t2ITD.js.map → dspy-rlm-engine-Caz2pl4L.js.map} +1 -1
- package/dist/{engine-DhFir3Ys.d.ts → engine-CvW_I72-.d.ts} +2 -2
- package/dist/{engine-DhFir3Ys.d.ts.map → engine-CvW_I72-.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -4
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/{external-optimizer-process-BwITA9Jp.js → external-optimizer-process-CxnFL1hd.js} +2 -2
- package/dist/{external-optimizer-process-BwITA9Jp.js.map → external-optimizer-process-CxnFL1hd.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-wBWeoG6A.js → external-optimizer-subprocess-CQi27uEI.js} +2 -2
- package/dist/{external-optimizer-subprocess-wBWeoG6A.js.map → external-optimizer-subprocess-CQi27uEI.js.map} +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/fuzz.js.map +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-D0Db5X-4.d.ts → index-BAAiSF3_.d.ts} +5 -5
- package/dist/{index-D0Db5X-4.d.ts.map → index-BAAiSF3_.d.ts.map} +1 -1
- package/dist/{index-BQqOjerE.d.ts → index-BTrx5s8m.d.ts} +8 -9
- package/dist/index-BTrx5s8m.d.ts.map +1 -0
- package/dist/index-DKXuBPXf.d.ts +3840 -0
- package/dist/index-DKXuBPXf.d.ts.map +1 -0
- package/dist/index.d.ts +11 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -11
- package/dist/{integrity-BWywb34E.js → integrity-DsHWCebQ.js} +11 -435
- package/dist/integrity-DsHWCebQ.js.map +1 -0
- package/dist/{kind-factory-gP6lDySe.js → kind-factory-BLvL-E44.js} +2 -2
- package/dist/{kind-factory-gP6lDySe.js.map → kind-factory-BLvL-E44.js.map} +1 -1
- package/dist/{llm-judge-BfqMFo4h.js → llm-judge-DmNaBrXB.js} +2541 -2435
- package/dist/llm-judge-DmNaBrXB.js.map +1 -0
- package/dist/{matrix-DGu8KhSs.d.ts → matrix-CJtXz1ky.d.ts} +2 -2
- package/dist/{matrix-DGu8KhSs.d.ts.map → matrix-CJtXz1ky.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/opencode-sqlite-CNw3vubS.js +145 -0
- package/dist/opencode-sqlite-CNw3vubS.js.map +1 -0
- package/dist/{produced-state-D91uDvQw.js → produced-state-B8mw6zj9.js} +2 -2
- package/dist/{produced-state-D91uDvQw.js.map → produced-state-B8mw6zj9.js.map} +1 -1
- package/dist/report-command-DKlXfU5r.js +1528 -0
- package/dist/report-command-DKlXfU5r.js.map +1 -0
- package/dist/rl.d.ts +1 -1
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.js +3 -2
- package/dist/{rollout-C-znbbYg.js → rollout-CGlDq1GI.js} +3 -2
- package/dist/{rollout-C-znbbYg.js.map → rollout-CGlDq1GI.js.map} +1 -1
- package/dist/{semantic-concept-judge-Dok7_35a.js → semantic-concept-judge-E3s_fEjB.js} +3 -3
- package/dist/{semantic-concept-judge-Dok7_35a.js.map → semantic-concept-judge-E3s_fEjB.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DDw3v3gA.js → skillopt-optimization-method-f7399oGb.js} +5 -5
- package/dist/{skillopt-optimization-method-DDw3v3gA.js.map → skillopt-optimization-method-f7399oGb.js.map} +1 -1
- package/dist/statistical-heldout-Cqb73yE9.d.ts +1127 -0
- package/dist/statistical-heldout-Cqb73yE9.d.ts.map +1 -0
- package/dist/{store-otlp-Dow0pk_5.js → store-otlp-DV_H2HDu.js} +2 -2
- package/dist/{store-otlp-Dow0pk_5.js.map → store-otlp-DV_H2HDu.js.map} +1 -1
- package/dist/{store-tool-spans-CCZNsihA.d.ts → store-tool-spans-4o55ABER.d.ts} +3 -3
- package/dist/{store-tool-spans-CCZNsihA.d.ts.map → store-tool-spans-4o55ABER.d.ts.map} +1 -1
- package/dist/{store-tool-spans-CeNj_m2L.js → store-tool-spans-B9tjys_h.js} +3 -3
- package/dist/{store-tool-spans-CeNj_m2L.js.map → store-tool-spans-B9tjys_h.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +71 -6
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +6 -1357
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{task-failure-attributes-CZjZeBsY.js → task-failure-attributes-CUy9mkIY.js} +2 -2
- package/dist/{task-failure-attributes-CZjZeBsY.js.map → task-failure-attributes-CUy9mkIY.js.map} +1 -1
- package/dist/terminal-record-Ce9_UjRz.js +539 -0
- package/dist/terminal-record-Ce9_UjRz.js.map +1 -0
- package/dist/{tool-groups-Cp4Xdzrp.d.ts → tool-groups-DAe1t6zb.d.ts} +2 -2
- package/dist/tool-groups-DAe1t6zb.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/traces.js +4 -4
- package/dist/{types-Ba5UQyVD.d.ts → types-BJz2CPTM.d.ts} +2 -2
- package/dist/{types-Ba5UQyVD.d.ts.map → types-BJz2CPTM.d.ts.map} +1 -1
- package/dist/{types-CiWITkGo.js → types-DQ0e2E7y.js} +2 -2
- package/dist/types-DQ0e2E7y.js.map +1 -0
- package/dist/{types-BDV4PiMR.d.ts → types-Dd1ejaeI.d.ts} +2 -2
- package/dist/{types-BDV4PiMR.d.ts.map → types-Dd1ejaeI.d.ts.map} +1 -1
- package/dist/{types-CoPUTiXb.d.ts → types-vUdAx2Cj.d.ts} +65 -3
- package/dist/types-vUdAx2Cj.d.ts.map +1 -0
- package/docs/campaign-proposers.md +42 -0
- package/package.json +1 -1
- package/dist/agent-profile-B9_GGsG8.d.ts +0 -84
- package/dist/agent-profile-B9_GGsG8.d.ts.map +0 -1
- package/dist/backend-integrity-CeuTgqsd.d.ts +0 -280
- package/dist/backend-integrity-CeuTgqsd.d.ts.map +0 -1
- package/dist/benchmark-BjLGkfnN.d.ts +0 -236
- package/dist/benchmark-BjLGkfnN.d.ts.map +0 -1
- package/dist/define-agent-eval-8h3lXXee.js.map +0 -1
- package/dist/define-agent-eval-CY6qdlGV.d.ts.map +0 -1
- package/dist/external-optimizer-contracts-CQCpyrIL.d.ts +0 -172
- package/dist/external-optimizer-contracts-CQCpyrIL.d.ts.map +0 -1
- package/dist/heldout-gate-Df5hsqmm.d.ts +0 -453
- package/dist/heldout-gate-Df5hsqmm.d.ts.map +0 -1
- package/dist/index-BQqOjerE.d.ts.map +0 -1
- package/dist/index-CFDffsKz.d.ts +0 -1135
- package/dist/index-CFDffsKz.d.ts.map +0 -1
- package/dist/integrity-BWywb34E.js.map +0 -1
- package/dist/llm-judge-BfqMFo4h.js.map +0 -1
- package/dist/opencode-sqlite-eK6HW6dr.js.map +0 -1
- package/dist/power-preflight-Ptse_Kq7.d.ts +0 -117
- package/dist/power-preflight-Ptse_Kq7.d.ts.map +0 -1
- package/dist/pre-registration-BoI4ucR3.d.ts +0 -592
- package/dist/pre-registration-BoI4ucR3.d.ts.map +0 -1
- package/dist/promotion-policy-CvMda3kU.d.ts +0 -134
- package/dist/promotion-policy-CvMda3kU.d.ts.map +0 -1
- package/dist/proposal-findings-bko3GGy-.js.map +0 -1
- package/dist/provenance-CRY67X50.d.ts +0 -1995
- package/dist/provenance-CRY67X50.d.ts.map +0 -1
- package/dist/statistical-heldout-DTyB_6-1.d.ts +0 -295
- package/dist/statistical-heldout-DTyB_6-1.d.ts.map +0 -1
- package/dist/tool-groups-Cp4Xdzrp.d.ts.map +0 -1
- package/dist/types-CiWITkGo.js.map +0 -1
- package/dist/types-CoPUTiXb.d.ts.map +0 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-DN2WdT5S.js";
|
|
2
2
|
import { l as ExactCapableAnalyst } from "./exact-types-BKOEILRP.js";
|
|
3
3
|
import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-7pOUBrtX.js";
|
|
4
|
-
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-
|
|
5
|
-
import { t as TraceToolGroupName } from "./tool-groups-
|
|
4
|
+
import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-CvW_I72-.js";
|
|
5
|
+
import { t as TraceToolGroupName } from "./tool-groups-DAe1t6zb.js";
|
|
6
6
|
//#region src/trace-analyst/behavioral-metrics.d.ts
|
|
7
7
|
type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
|
|
8
8
|
interface SuboptimalSignal {
|
|
@@ -110,4 +110,4 @@ interface DefaultAnalystRegistryOptions {
|
|
|
110
110
|
declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
|
|
111
111
|
//#endregion
|
|
112
112
|
export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
|
|
113
|
-
//# sourceMappingURL=default-registry-
|
|
113
|
+
//# sourceMappingURL=default-registry-FfNzaUHV.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"default-registry-
|
|
1
|
+
{"version":3,"file":"default-registry-FfNzaUHV.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
|
|
@@ -1,10 +1,85 @@
|
|
|
1
1
|
import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { _ as ProposalFinding } from "./types-DN2WdT5S.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { B as PowerPreflight } from "./statistical-heldout-Cqb73yE9.js";
|
|
4
|
+
import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate, v as GateResult } from "./types-BJz2CPTM.js";
|
|
5
|
+
import { $r as OptimizationMethodProvenance, Ni as PremeasuredOptimizationBaseline, Pi as RunOptimizationOptions, Yr as OptimizationMethod, Zn as LoopProvenanceRecord, ao as CampaignCellRetryPolicy, ei as OptimizationMethodResult, ji as RunImprovementLoopResult, no as RunEvalOptions, oo as RunCampaignOptions, si as ComparisonCost, ta as SearchHistoryReceipt, to as openAutoPr, vo as CampaignStorage } from "./index-DKXuBPXf.js";
|
|
6
6
|
import { o as InsightReport } from "./insight-report-DETqPc_A.js";
|
|
7
|
-
import { n as HostedTenant } from "./client-
|
|
7
|
+
import { n as HostedTenant } from "./client-vyYQg3bm.js";
|
|
8
|
+
//#region src/campaign/presets/run-final-comparison.d.ts
|
|
9
|
+
interface FinalComparisonOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch'> {
|
|
10
|
+
baselineSurface: MutableSurface;
|
|
11
|
+
winnerSurface: MutableSurface;
|
|
12
|
+
dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
|
|
13
|
+
gate: Gate<TArtifact, TScenario>;
|
|
14
|
+
holdout?: 'measured' | 'deferred';
|
|
15
|
+
label?: string;
|
|
16
|
+
neutralize?: (winner: MutableSurface, baseline: MutableSurface) => MutableSurface;
|
|
17
|
+
}
|
|
18
|
+
/** Compare a search-selected surface without changing the search's selection. */
|
|
19
|
+
declare function runFinalComparison<TScenario extends Scenario, TArtifact>(opts: FinalComparisonOptions<TScenario, TArtifact>): Promise<{
|
|
20
|
+
baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
|
|
21
|
+
winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
|
|
22
|
+
neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario> | undefined;
|
|
23
|
+
neutralizedSurface?: MutableSurface | undefined;
|
|
24
|
+
holdout?: "deferred" | undefined;
|
|
25
|
+
gateResult: GateResult;
|
|
26
|
+
promotedDiff: string;
|
|
27
|
+
}>;
|
|
28
|
+
//#endregion
|
|
29
|
+
//#region src/contract/self-improve-method.d.ts
|
|
30
|
+
interface SelfImproveMethodProvenance {
|
|
31
|
+
schema: 'tangle.method-improvement';
|
|
32
|
+
recordDigest: `sha256:${string}`;
|
|
33
|
+
runId: string;
|
|
34
|
+
runDir: string;
|
|
35
|
+
timestamp: string;
|
|
36
|
+
baselineContentHash: string;
|
|
37
|
+
winnerContentHash: string;
|
|
38
|
+
diff: string;
|
|
39
|
+
optimizationMethod: NonNullable<SelfImproveProposerResult<Scenario, unknown>['optimization']>;
|
|
40
|
+
evidence: {
|
|
41
|
+
trainSplitDigest: `sha256:${string}`;
|
|
42
|
+
selectionSplitDigest: `sha256:${string}`;
|
|
43
|
+
holdoutSplitDigest: `sha256:${string}`;
|
|
44
|
+
baselineCampaignDigest: `sha256:${string}`;
|
|
45
|
+
winnerCampaignDigest: `sha256:${string}`;
|
|
46
|
+
costReceiptsDigest: `sha256:${string}`;
|
|
47
|
+
neutralizedCampaignDigest?: `sha256:${string}`;
|
|
48
|
+
};
|
|
49
|
+
gate: Awaited<ReturnType<typeof runFinalComparison>>['gateResult'];
|
|
50
|
+
holdout?: 'deferred';
|
|
51
|
+
baselineHoldoutComposite?: number;
|
|
52
|
+
winnerHoldoutComposite?: number;
|
|
53
|
+
heldOutLift?: number;
|
|
54
|
+
cost: ComparisonCost;
|
|
55
|
+
totalCostUsd: number;
|
|
56
|
+
totalDurationMs: number;
|
|
57
|
+
}
|
|
58
|
+
interface SelfImproveMethodResult<TScenario extends Scenario, TArtifact> extends Omit<SelfImproveProposerResult<TScenario, TArtifact>, 'mode' | 'baseline' | 'winner' | 'provenance' | 'generationsExplored' | 'cost' | 'raw' | 'optimization' | 'insight'> {
|
|
59
|
+
mode: 'method';
|
|
60
|
+
/** No final measurement exists when holdout is deferred. */
|
|
61
|
+
baseline: SelfImproveProposerResult<TScenario, TArtifact>['baseline'] | null;
|
|
62
|
+
winner: Omit<SelfImproveProposerResult<TScenario, TArtifact>['winner'], 'compositeMean'> & {
|
|
63
|
+
compositeMean: number | null;
|
|
64
|
+
};
|
|
65
|
+
provenance: SelfImproveMethodProvenance;
|
|
66
|
+
optimization: NonNullable<SelfImproveProposerResult<TScenario, TArtifact>['optimization']>;
|
|
67
|
+
/** Includes the method's reported cost and final measurements without double counting receipts. */
|
|
68
|
+
cost: ComparisonCost;
|
|
69
|
+
/** Only calls actually recorded by the shared ledger have token and channel breakdowns. */
|
|
70
|
+
ledgerCost: CostLedgerSummary;
|
|
71
|
+
insight?: SelfImproveProposerResult<TScenario, TArtifact>['insight'];
|
|
72
|
+
raw: Awaited<ReturnType<typeof runFinalComparison<TScenario, TArtifact>>> & {
|
|
73
|
+
kind: 'method';
|
|
74
|
+
baselineSurface: SelfImproveProposerResult<TScenario, TArtifact>['raw']['baselineSurface'];
|
|
75
|
+
winnerSurface: SelfImproveProposerResult<TScenario, TArtifact>['raw']['winnerSurface'];
|
|
76
|
+
winnerSurfaceHash: string;
|
|
77
|
+
method: OptimizationMethodResult;
|
|
78
|
+
cost: ComparisonCost;
|
|
79
|
+
prResult?: ReturnType<typeof openAutoPr>;
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
//#endregion
|
|
8
83
|
//#region src/contract/self-improve.d.ts
|
|
9
84
|
interface SelfImproveBudget {
|
|
10
85
|
/** Hard spend cap across the full run. Each paid call reserves its enforced
|
|
@@ -96,6 +171,8 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
96
171
|
* Omit this when every cell reports its concrete model in a receipt.
|
|
97
172
|
*/
|
|
98
173
|
model?: string;
|
|
174
|
+
/** Version of execution behavior outside the candidate surface, used by measurement caches. */
|
|
175
|
+
dispatchRef?: string;
|
|
99
176
|
/** Scenarios to evaluate against. Train/holdout split is computed from
|
|
100
177
|
* these unless `budget.holdoutScenarios` is set explicitly. */
|
|
101
178
|
scenarios: TScenario[];
|
|
@@ -109,13 +186,14 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
109
186
|
/**
|
|
110
187
|
* Complete prior measurement of `baselineSurface` over the TRAIN split.
|
|
111
188
|
* Forwarded to the loop body, which validates its surface hash, scenario
|
|
112
|
-
* split, seed (42), reps, and coverage, then skips the baseline search
|
|
189
|
+
* split, seed (42), reps, evaluator manifest, and coverage, then skips the baseline search
|
|
113
190
|
* campaign entirely — no baseline dispatch, no resumability lookup. The
|
|
114
191
|
* train split is `scenarios` minus the holdout split, so premeasure with
|
|
115
192
|
* exactly that scenario set (explicit `budget.holdoutScenarios`, or
|
|
116
193
|
* `budget.holdout: 'deferred'` with no reserved set, makes the train split
|
|
117
194
|
* deterministic). Prior spend stays in the imported campaign aggregates and
|
|
118
|
-
* is not re-added to this run's cost ledger.
|
|
195
|
+
* is not re-added to this run's cost ledger. Premeasure with
|
|
196
|
+
* `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` and the same judge revision.
|
|
119
197
|
*/
|
|
120
198
|
premeasuredBaseline?: PremeasuredOptimizationBaseline<TArtifact, TScenario>;
|
|
121
199
|
/**
|
|
@@ -151,9 +229,9 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
151
229
|
* `mem://selfImprove-<timestamp>`. External method mode defaults to
|
|
152
230
|
* `.agent-eval/runs/self-improve-<timestamp>`. */
|
|
153
231
|
runDir?: string;
|
|
154
|
-
/** Fires once the durable provenance record
|
|
232
|
+
/** Fires once the durable provenance record is written.
|
|
155
233
|
* Receives the structured record for inline assertions / custom routing. */
|
|
156
|
-
onProvenance?: (record: LoopProvenanceRecord) => void;
|
|
234
|
+
onProvenance?: (record: LoopProvenanceRecord | SelfImproveMethodProvenance) => void;
|
|
157
235
|
/** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`.
|
|
158
236
|
* Returns an opaque placement key the substrate forwards to your agent
|
|
159
237
|
* as `ctx.placement`. Combined with `httpDispatch` from
|
|
@@ -243,7 +321,8 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
|
243
321
|
* `RunOptimizationOptions.searchLedger`. */
|
|
244
322
|
searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
|
|
245
323
|
}
|
|
246
|
-
interface
|
|
324
|
+
interface SelfImproveProposerResult<TScenario extends Scenario, TArtifact> {
|
|
325
|
+
mode: 'proposer';
|
|
247
326
|
/** Composite mean across all scenarios, baseline run. When
|
|
248
327
|
* `budget.holdout === 'deferred'` this is measured on the improvement
|
|
249
328
|
* (search) split — no holdout campaign ran. */
|
|
@@ -321,6 +400,16 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
|
|
|
321
400
|
*/
|
|
322
401
|
raw: RunImprovementLoopResult<TArtifact, TScenario>;
|
|
323
402
|
}
|
|
403
|
+
type SelfImproveResult<TScenario extends Scenario, TArtifact> = SelfImproveProposerResult<TScenario, TArtifact> | SelfImproveMethodResult<TScenario, TArtifact>;
|
|
404
|
+
type SelfImproveMethodOptions<TScenario extends Scenario, TArtifact> = Omit<SelfImproveOptions<TScenario, TArtifact>, 'proposer' | 'onProvenance'> & {
|
|
405
|
+
method: OptimizationMethod<TScenario, TArtifact>;
|
|
406
|
+
proposer?: never;
|
|
407
|
+
onProvenance?: (record: SelfImproveMethodProvenance) => void;
|
|
408
|
+
};
|
|
409
|
+
type SelfImproveProposerOptions<TScenario extends Scenario, TArtifact> = Omit<SelfImproveOptions<TScenario, TArtifact>, 'method' | 'onProvenance'> & {
|
|
410
|
+
method?: never;
|
|
411
|
+
onProvenance?: (record: LoopProvenanceRecord) => void;
|
|
412
|
+
};
|
|
324
413
|
/** Failed self-improvement run with an immutable receipt snapshot. */
|
|
325
414
|
declare class SelfImproveRunError extends Error {
|
|
326
415
|
readonly cost: CostLedgerSummary;
|
|
@@ -353,6 +442,8 @@ declare class SelfImproveRunError extends Error {
|
|
|
353
442
|
* budget: { maxConcurrency: 12 },
|
|
354
443
|
* })
|
|
355
444
|
*/
|
|
445
|
+
declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveMethodOptions<TScenario, TArtifact>): Promise<SelfImproveMethodResult<TScenario, TArtifact>>;
|
|
446
|
+
declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveProposerOptions<TScenario, TArtifact>): Promise<SelfImproveProposerResult<TScenario, TArtifact>>;
|
|
356
447
|
declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
|
|
357
448
|
//#endregion
|
|
358
449
|
//#region src/contract/define-agent-eval.d.ts
|
|
@@ -403,5 +494,5 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
|
|
|
403
494
|
*/
|
|
404
495
|
declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
|
|
405
496
|
//#endregion
|
|
406
|
-
export { DefinedAgentEval as a,
|
|
407
|
-
//# sourceMappingURL=define-agent-eval-
|
|
497
|
+
export { SelfImproveMethodResult as _, DefinedAgentEval as a, SelfImproveMethodOptions as c, SelfImproveProposerOptions as d, SelfImproveProposerResult as f, SelfImproveMethodProvenance as g, selfImprove as h, DefineAgentEvalOptions as i, SelfImproveOptions as l, SelfImproveRunError as m, AgentEvalEvaluateOptions as n, defineAgentEval as o, SelfImproveResult as p, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveProgressEvent as u };
|
|
498
|
+
//# sourceMappingURL=define-agent-eval-V1jQyCDR.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"define-agent-eval-V1jQyCDR.d.ts","names":[],"sources":["../src/campaign/presets/run-final-comparison.ts","../src/contract/self-improve-method.ts","../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAMiB,uBAAuB,kBAAkB,UAAU,mBAC1D,KAAK,mBAAmB,WAAW;EAC3C,iBAAiB;EACjB,eAAe;EACf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;EACb,MAAM,KAAK,WAAW;EACtB;EACA;EACA,cAAc,QAAQ,gBAAgB,UAAU,mBAAmB;;;iBAI/C,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,uBAAuB,WAAW,aAAU;;;;;;;;;;;UCSnC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,oBAAoB,YAAY,0BAA0B;EAC1D;IACE;IACA;IACA;IACA;IACA;IACA;IACA;;EAEF,MAAM,QAAQ,kBAAkB;EAChC;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;;UAGe,wBAAwB,kBAAkB,UAAU,mBAC3D,KACN,0BAA0B,WAAW;EAWvC;;EAEA,UAAU,0BAA0B,WAAW;EAC/C,QAAQ,KAAK,0BAA0B,WAAW;IAChD;;EAEF,YAAY;EACZ,cAAc,YAAY,0BAA0B,WAAW;;EAE/D,MAAM;;EAEN,YAAY;EACZ,UAAU,0BAA0B,WAAW;EAC/C,KAAK,QAAQ,kBAAkB,mBAAmB,WAAW;IAC3D;IACA,iBAAiB,0BAA0B,WAAW;IACtD,eAAe,0BAA0B,WAAW;IACpD;IACA,QAAQ;IACR,MAAM;IACN,WAAW,kBAAkB;;;;;UCzBhB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU;;;;;;;;;;;;;;EAc9D,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;EAEA;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;;EAcT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ,uBAAuB;;;;;EAM/C,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;UAGlC,0BAA0B,kBAAkB,UAAU;EACrE;;;;EAIA;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;KAG/B,kBAAkB,kBAAkB,UAAU,aACtD,0BAA0B,WAAW,aACrC,wBAAwB,WAAW;KAE3B,yBAAyB,kBAAkB,UAAU,aAAa,KAC5E,mBAAmB,WAAW;EAG9B,QAAQ,mBAAmB,WAAW;EACtC;EACA,gBAAgB,QAAQ;;KAGd,2BAA2B,kBAAkB,UAAU,aAAa,KAC9E,mBAAmB,WAAW;EAG9B;EACA,gBAAgB,QAAQ;;;cAIb,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAwLtB,YAAY,kBAAkB,UAAU,WACtD,MAAM,yBAAyB,WAAW,aACzC,QAAQ,wBAAwB,WAAW;iBAC9B,YAAY,kBAAkB,UAAU,WACtD,MAAM,2BAA2B,WAAW,aAC3C,QAAQ,0BAA0B,WAAW;iBAChC,YAAY,kBAAkB,UAAU,WACtD,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KC1lB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}
|