@tangle-network/agent-eval 0.125.0 → 0.126.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -35
- package/README.md +270 -189
- package/dist/analyst/index.d.ts +15 -145
- package/dist/analyst/index.js +33 -47
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +45 -162
- package/dist/benchmarks/index.js +8 -9
- package/dist/campaign/index.d.ts +3674 -5393
- package/dist/campaign/index.js +21 -95
- package/dist/{chunk-R226UZOI.js → chunk-474LBSOX.js} +2 -2
- package/dist/{chunk-HM6V7F3M.js → chunk-FO7HEH76.js} +3 -3
- package/dist/chunk-IILEIWGW.js +635 -0
- package/dist/chunk-IILEIWGW.js.map +1 -0
- package/dist/{chunk-EQUK3RFS.js → chunk-J5SQWP6Y.js} +8 -5
- package/dist/chunk-J5SQWP6Y.js.map +1 -0
- package/dist/{chunk-W5B3ZGP3.js → chunk-KE2VWPZX.js} +8 -6
- package/dist/{chunk-W5B3ZGP3.js.map → chunk-KE2VWPZX.js.map} +1 -1
- package/dist/{chunk-DT7OXY3C.js → chunk-LUNF2SEL.js} +538 -851
- package/dist/chunk-LUNF2SEL.js.map +1 -0
- package/dist/chunk-NGUYT5CI.js +4637 -0
- package/dist/chunk-NGUYT5CI.js.map +1 -0
- package/dist/{chunk-QFQZ3U3X.js → chunk-OCFJACJU.js} +2 -2
- package/dist/{chunk-GID26AN4.js → chunk-P22LJ3Y2.js} +4 -6
- package/dist/{chunk-GID26AN4.js.map → chunk-P22LJ3Y2.js.map} +1 -1
- package/dist/{chunk-SJT4OBVL.js → chunk-SDPM6554.js} +3 -3
- package/dist/{chunk-D5JZ7UDZ.js → chunk-UCLVDLCH.js} +136 -50
- package/dist/chunk-UCLVDLCH.js.map +1 -0
- package/dist/chunk-VMUENW6F.js +7274 -0
- package/dist/chunk-VMUENW6F.js.map +1 -0
- package/dist/{chunk-JKDNAOF5.js → chunk-W4L6C2XT.js} +2 -2
- package/dist/chunk-WGXIEX7P.js +116 -0
- package/dist/chunk-WGXIEX7P.js.map +1 -0
- package/dist/{chunk-GRCDRKII.js → chunk-WS3NZZQQ.js} +58 -20
- package/dist/chunk-WS3NZZQQ.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/contract/index.d.ts +3220 -3094
- package/dist/contract/index.js +173 -42
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +2 -3
- package/dist/fuzz.d.ts +14 -1
- package/dist/fuzz.js +1 -1
- package/dist/hosted/index.d.ts +8 -1
- package/dist/index.d.ts +71 -687
- package/dist/index.js +178 -497
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +5 -100
- package/dist/rl.js +4 -5
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-I3JXKVAK.js → run-campaign-LVFKZCEU.js} +3 -3
- package/dist/traces.js +2 -3
- package/dist/wire/index.d.ts +14 -1
- package/dist/wire/index.js +3 -3
- package/docs/campaign-proposers.md +363 -168
- package/docs/design/loop-taxonomy.md +142 -190
- package/docs/design.md +1 -1
- package/docs/distributed-driver.md +8 -11
- package/docs/feature-guide.md +20 -19
- package/docs/knowledge-readiness.md +2 -5
- package/docs/multi-shot-optimization.md +35 -27
- package/docs/rollout.md +5 -5
- package/package.json +4 -4
- package/dist/chunk-A62YMFWA.js +0 -9269
- package/dist/chunk-A62YMFWA.js.map +0 -1
- package/dist/chunk-A6GT67HT.js +0 -550
- package/dist/chunk-A6GT67HT.js.map +0 -1
- package/dist/chunk-D5JZ7UDZ.js.map +0 -1
- package/dist/chunk-DT7OXY3C.js.map +0 -1
- package/dist/chunk-EQUK3RFS.js.map +0 -1
- package/dist/chunk-GC4ATIKK.js +0 -317
- package/dist/chunk-GC4ATIKK.js.map +0 -1
- package/dist/chunk-GRCDRKII.js.map +0 -1
- package/dist/chunk-LOW3U7JZ.js +0 -328
- package/dist/chunk-LOW3U7JZ.js.map +0 -1
- package/dist/chunk-PMITBABE.js +0 -3841
- package/dist/chunk-PMITBABE.js.map +0 -1
- /package/dist/{chunk-R226UZOI.js.map → chunk-474LBSOX.js.map} +0 -0
- /package/dist/{chunk-HM6V7F3M.js.map → chunk-FO7HEH76.js.map} +0 -0
- /package/dist/{chunk-QFQZ3U3X.js.map → chunk-OCFJACJU.js.map} +0 -0
- /package/dist/{chunk-SJT4OBVL.js.map → chunk-SDPM6554.js.map} +0 -0
- /package/dist/{chunk-JKDNAOF5.js.map → chunk-W4L6C2XT.js.map} +0 -0
- /package/dist/{run-campaign-I3JXKVAK.js.map → run-campaign-LVFKZCEU.js.map} +0 -0
package/dist/campaign/index.js
CHANGED
|
@@ -1,79 +1,42 @@
|
|
|
1
1
|
import {
|
|
2
|
-
DEFAULT_POLICY_EDIT_HISTORY_LIMITS,
|
|
3
2
|
FileSearchLedger,
|
|
4
3
|
FsLabeledScenarioStore,
|
|
5
4
|
LabeledScenarioStoreError,
|
|
6
|
-
Lineage,
|
|
7
|
-
LineageStoreConflictError,
|
|
8
5
|
ProfileMatrixError,
|
|
9
6
|
SEARCH_LEDGER_SCHEMA,
|
|
10
|
-
SkillPatchParseError,
|
|
11
7
|
WorktreeAdapterError,
|
|
12
|
-
aceProposer,
|
|
13
|
-
acquireSingleRunLock,
|
|
14
8
|
analyzeCrossSurfaceInteractions,
|
|
15
|
-
analyzeOtlpTraceFile,
|
|
16
|
-
applySkillPatch,
|
|
17
|
-
assertPolicyEditAuthorContextBudget,
|
|
18
9
|
buildAnalystSurfaceDispatch,
|
|
19
|
-
callbackGovernor,
|
|
20
|
-
campaignLineageStore,
|
|
21
10
|
classifyUngroundedLiterals,
|
|
22
|
-
compareOptimizationMethods,
|
|
23
|
-
compositeProposer,
|
|
24
|
-
costFromLedgerSummary,
|
|
25
11
|
discoverEvalFixtures,
|
|
26
|
-
extractFapoAttributionSignals,
|
|
27
12
|
failureModeRecallJudge,
|
|
28
|
-
fapoEscalationMethod,
|
|
29
|
-
fapoProposer,
|
|
30
|
-
fsLineageStore,
|
|
31
|
-
gepaOptimizationMethod,
|
|
32
|
-
gepaParetoMethod,
|
|
33
|
-
gepaReflectionMethod,
|
|
34
13
|
gitWorktreeAdapter,
|
|
35
|
-
haloProposer,
|
|
36
|
-
heuristicGovernor,
|
|
37
14
|
isTransientTransportFailure,
|
|
38
|
-
lineageNodeId,
|
|
39
|
-
llmPolicyEditProposer,
|
|
40
15
|
loadEvalFixture,
|
|
41
16
|
loadEvalFixtureScenarios,
|
|
42
17
|
makePlaybackDispatch,
|
|
43
|
-
memLineageStore,
|
|
44
|
-
memoryCurationProposer,
|
|
45
18
|
neutralizationGate,
|
|
46
19
|
neutralizeText,
|
|
47
20
|
openSearchLedger,
|
|
48
|
-
parameterSweepProposer,
|
|
49
|
-
parseSkillPatchResponse,
|
|
50
|
-
patchEditCount,
|
|
51
21
|
planEvalFixtureRun,
|
|
52
|
-
policyEditProposer,
|
|
53
|
-
projectPolicyEditHistory,
|
|
54
22
|
renderScoreboardMarkdown,
|
|
55
23
|
resolveWorktreePath,
|
|
56
24
|
rolloutArgumentDiff,
|
|
57
|
-
runLineage,
|
|
58
|
-
runLineageLoop,
|
|
59
25
|
runProfileMatrix,
|
|
60
|
-
runSkillOpt,
|
|
61
26
|
scoreDiscrimination,
|
|
62
27
|
scoreUserStory,
|
|
63
28
|
scoreboardSummary,
|
|
64
29
|
selectDiscriminative,
|
|
65
|
-
selectPolicyEditAuthorRows,
|
|
66
30
|
sequentialDecide,
|
|
67
31
|
sequentialPairedGate,
|
|
68
|
-
skillOptMethod,
|
|
69
|
-
skillOptProposer,
|
|
70
|
-
traceAnalystProposer,
|
|
71
32
|
userStoryScoreboard,
|
|
72
33
|
validateSearchLedgerEvent,
|
|
73
34
|
verifyCodeSurface
|
|
74
|
-
} from "../chunk-
|
|
35
|
+
} from "../chunk-NGUYT5CI.js";
|
|
75
36
|
import {
|
|
37
|
+
acquireSingleRunLock,
|
|
76
38
|
assertCodeSurfaceIdentity,
|
|
39
|
+
assertComponentSurface,
|
|
77
40
|
buildEvidenceVector,
|
|
78
41
|
buildLoopProvenanceRecord,
|
|
79
42
|
campaignBreakdown,
|
|
@@ -81,16 +44,17 @@ import {
|
|
|
81
44
|
campaignMeasurementDigest,
|
|
82
45
|
canonicalDigest,
|
|
83
46
|
codeSurfaceIdentityMaterial,
|
|
47
|
+
compareOptimizationMethods,
|
|
48
|
+
componentSurfaceIdentityMaterial,
|
|
84
49
|
composeGate,
|
|
85
|
-
|
|
50
|
+
costFromLedgerSummary,
|
|
86
51
|
createReferenceEquivalenceJudge,
|
|
87
52
|
defaultProductionGate,
|
|
88
53
|
detectScale,
|
|
89
54
|
dimensionRegressions,
|
|
90
55
|
emitLoopProvenance,
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
gepaProposer,
|
|
56
|
+
externalTextOptimizationMethod,
|
|
57
|
+
gepaOptimizationMethod,
|
|
94
58
|
heldOutGate,
|
|
95
59
|
heldoutSignificance,
|
|
96
60
|
isProposedCandidate,
|
|
@@ -99,6 +63,7 @@ import {
|
|
|
99
63
|
loopProvenanceArgsFromResult,
|
|
100
64
|
loopProvenanceSpans,
|
|
101
65
|
openAutoPr,
|
|
66
|
+
optimizationTokenUsageFromSummary,
|
|
102
67
|
pairHoldout,
|
|
103
68
|
paretoPolicy,
|
|
104
69
|
paretoSignificanceGate,
|
|
@@ -109,10 +74,11 @@ import {
|
|
|
109
74
|
runEval,
|
|
110
75
|
runImprovementLoop,
|
|
111
76
|
runOptimization,
|
|
77
|
+
skillOptOptimizationMethod,
|
|
112
78
|
surfaceContentHash,
|
|
113
79
|
surfaceHash,
|
|
114
80
|
verifyLoopProvenanceRecord
|
|
115
|
-
} from "../chunk-
|
|
81
|
+
} from "../chunk-VMUENW6F.js";
|
|
116
82
|
import {
|
|
117
83
|
SearchLedgerConflictError,
|
|
118
84
|
SearchLedgerError,
|
|
@@ -129,57 +95,43 @@ import {
|
|
|
129
95
|
resolveRunDir,
|
|
130
96
|
runCampaign,
|
|
131
97
|
tangleTracesRoot
|
|
132
|
-
} from "../chunk-
|
|
133
|
-
import
|
|
134
|
-
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
135
|
-
validatePolicyEditCandidateRecord
|
|
136
|
-
} from "../chunk-DT7OXY3C.js";
|
|
98
|
+
} from "../chunk-UCLVDLCH.js";
|
|
99
|
+
import "../chunk-WGXIEX7P.js";
|
|
137
100
|
import "../chunk-ARU2PZFM.js";
|
|
138
|
-
import "../chunk-
|
|
101
|
+
import "../chunk-J5SQWP6Y.js";
|
|
139
102
|
import "../chunk-PJQFMIOX.js";
|
|
140
|
-
import "../chunk-
|
|
103
|
+
import "../chunk-WS3NZZQQ.js";
|
|
141
104
|
import "../chunk-VI2UW6B6.js";
|
|
142
105
|
import "../chunk-NUKSVU3W.js";
|
|
143
106
|
import "../chunk-GGE4NNQT.js";
|
|
144
107
|
import "../chunk-IR3KBHOY.js";
|
|
145
108
|
import "../chunk-PC4UYEBM.js";
|
|
146
|
-
import "../chunk-
|
|
109
|
+
import "../chunk-IILEIWGW.js";
|
|
147
110
|
import "../chunk-MA6HLL3S.js";
|
|
148
|
-
import "../chunk-GC4ATIKK.js";
|
|
149
111
|
import "../chunk-VSMTAMNK.js";
|
|
150
112
|
import "../chunk-ONWEPEDO.js";
|
|
151
113
|
import "../chunk-K4DBDHLK.js";
|
|
152
114
|
import "../chunk-PZ5AY32C.js";
|
|
153
115
|
export {
|
|
154
|
-
DEFAULT_POLICY_EDIT_HISTORY_LIMITS,
|
|
155
116
|
FileSearchLedger,
|
|
156
117
|
FsLabeledScenarioStore,
|
|
157
118
|
LabeledScenarioStoreError,
|
|
158
|
-
Lineage,
|
|
159
|
-
LineageStoreConflictError,
|
|
160
|
-
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
161
119
|
ProfileMatrixError,
|
|
162
120
|
SEARCH_LEDGER_SCHEMA,
|
|
163
121
|
SearchLedgerConflictError,
|
|
164
122
|
SearchLedgerError,
|
|
165
123
|
SearchLedgerIntegrityError,
|
|
166
|
-
SkillPatchParseError,
|
|
167
124
|
WorktreeAdapterError,
|
|
168
|
-
aceProposer,
|
|
169
125
|
acquireSingleRunLock,
|
|
170
126
|
analyzeCrossSurfaceInteractions,
|
|
171
|
-
analyzeOtlpTraceFile,
|
|
172
|
-
applySkillPatch,
|
|
173
127
|
assertCampaignDesign,
|
|
174
128
|
assertCampaignSplitIdentity,
|
|
175
129
|
assertCodeSurfaceIdentity,
|
|
176
|
-
|
|
130
|
+
assertComponentSurface,
|
|
177
131
|
buildAnalystSurfaceDispatch,
|
|
178
132
|
buildEvidenceVector,
|
|
179
133
|
buildLoopProvenanceRecord,
|
|
180
|
-
callbackGovernor,
|
|
181
134
|
campaignBreakdown,
|
|
182
|
-
campaignLineageStore,
|
|
183
135
|
campaignMeanComposite,
|
|
184
136
|
campaignMeasurementDigest,
|
|
185
137
|
campaignScenarioIdentity,
|
|
@@ -189,10 +141,9 @@ export {
|
|
|
189
141
|
classifyUngroundedLiterals,
|
|
190
142
|
codeSurfaceIdentityMaterial,
|
|
191
143
|
compareOptimizationMethods,
|
|
144
|
+
componentSurfaceIdentityMaterial,
|
|
192
145
|
composeGate,
|
|
193
|
-
compositeProposer,
|
|
194
146
|
costFromLedgerSummary,
|
|
195
|
-
countSentenceEdits,
|
|
196
147
|
createReferenceEquivalenceJudge,
|
|
197
148
|
createRunCostLedger,
|
|
198
149
|
defaultProductionGate,
|
|
@@ -200,52 +151,34 @@ export {
|
|
|
200
151
|
dimensionRegressions,
|
|
201
152
|
discoverEvalFixtures,
|
|
202
153
|
emitLoopProvenance,
|
|
203
|
-
|
|
204
|
-
extractFapoAttributionSignals,
|
|
205
|
-
extractH2Sections,
|
|
154
|
+
externalTextOptimizationMethod,
|
|
206
155
|
failureModeRecallJudge,
|
|
207
|
-
fapoEscalationMethod,
|
|
208
|
-
fapoProposer,
|
|
209
156
|
fsCampaignStorage,
|
|
210
|
-
fsLineageStore,
|
|
211
157
|
gepaOptimizationMethod,
|
|
212
|
-
gepaParetoMethod,
|
|
213
|
-
gepaProposer,
|
|
214
|
-
gepaReflectionMethod,
|
|
215
158
|
gitWorktreeAdapter,
|
|
216
|
-
haloProposer,
|
|
217
159
|
heldOutGate,
|
|
218
160
|
heldoutSignificance,
|
|
219
|
-
heuristicGovernor,
|
|
220
161
|
inMemoryCampaignStorage,
|
|
221
162
|
isProposedCandidate,
|
|
222
163
|
isTransientTransportFailure,
|
|
223
164
|
labelTrustRank,
|
|
224
|
-
lineageNodeId,
|
|
225
165
|
llmJudge,
|
|
226
|
-
llmPolicyEditProposer,
|
|
227
166
|
loadEvalFixture,
|
|
228
167
|
loadEvalFixtureScenarios,
|
|
229
168
|
loopProvenanceArgsFromResult,
|
|
230
169
|
loopProvenanceSpans,
|
|
231
170
|
makePlaybackDispatch,
|
|
232
|
-
memLineageStore,
|
|
233
|
-
memoryCurationProposer,
|
|
234
171
|
neutralizationGate,
|
|
235
172
|
neutralizeText,
|
|
236
173
|
openAutoPr,
|
|
237
174
|
openSearchLedger,
|
|
175
|
+
optimizationTokenUsageFromSummary,
|
|
238
176
|
pairHoldout,
|
|
239
|
-
parameterSweepProposer,
|
|
240
177
|
paretoPolicy,
|
|
241
178
|
paretoSignificanceGate,
|
|
242
|
-
parseSkillPatchResponse,
|
|
243
|
-
patchEditCount,
|
|
244
179
|
planCampaignRun,
|
|
245
180
|
planEvalFixtureRun,
|
|
246
|
-
policyEditProposer,
|
|
247
181
|
powerPreflight,
|
|
248
|
-
projectPolicyEditHistory,
|
|
249
182
|
provenanceRecordPath,
|
|
250
183
|
provenanceSpansPath,
|
|
251
184
|
renderScoreboardMarkdown,
|
|
@@ -256,26 +189,19 @@ export {
|
|
|
256
189
|
runCampaign,
|
|
257
190
|
runEval,
|
|
258
191
|
runImprovementLoop,
|
|
259
|
-
runLineage,
|
|
260
|
-
runLineageLoop,
|
|
261
192
|
runOptimization,
|
|
262
193
|
runProfileMatrix,
|
|
263
|
-
runSkillOpt,
|
|
264
194
|
scoreDiscrimination,
|
|
265
195
|
scoreUserStory,
|
|
266
196
|
scoreboardSummary,
|
|
267
197
|
selectDiscriminative,
|
|
268
|
-
selectPolicyEditAuthorRows,
|
|
269
198
|
sequentialDecide,
|
|
270
199
|
sequentialPairedGate,
|
|
271
|
-
|
|
272
|
-
skillOptProposer,
|
|
200
|
+
skillOptOptimizationMethod,
|
|
273
201
|
surfaceContentHash,
|
|
274
202
|
surfaceHash,
|
|
275
203
|
tangleTracesRoot,
|
|
276
|
-
traceAnalystProposer,
|
|
277
204
|
userStoryScoreboard,
|
|
278
|
-
validatePolicyEditCandidateRecord,
|
|
279
205
|
validateSearchLedgerEvent,
|
|
280
206
|
verifyCodeSurface,
|
|
281
207
|
verifyLoopProvenanceRecord
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
} from "./chunk-VQMK5FMP.js";
|
|
4
4
|
import {
|
|
5
5
|
validateRunRecord
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-IILEIWGW.js";
|
|
7
7
|
|
|
8
8
|
// src/action-policy.ts
|
|
9
9
|
function evaluateActionPolicy(action, policy = {}, options = {}) {
|
|
@@ -1523,4 +1523,4 @@ export {
|
|
|
1523
1523
|
runProposeReviewAsControlLoop,
|
|
1524
1524
|
controlFailureClassFromVerification
|
|
1525
1525
|
};
|
|
1526
|
-
//# sourceMappingURL=chunk-
|
|
1526
|
+
//# sourceMappingURL=chunk-474LBSOX.js.map
|
|
@@ -5,10 +5,10 @@ import {
|
|
|
5
5
|
costReceiptFromLlm,
|
|
6
6
|
costReceiptFromLlmError,
|
|
7
7
|
maximumChargeForLlmRequest
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-J5SQWP6Y.js";
|
|
9
9
|
import {
|
|
10
10
|
CostLedger
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-WS3NZZQQ.js";
|
|
12
12
|
|
|
13
13
|
// src/wire/schemas.ts
|
|
14
14
|
import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";
|
|
@@ -1053,4 +1053,4 @@ export {
|
|
|
1053
1053
|
startServer,
|
|
1054
1054
|
startServerAsync
|
|
1055
1055
|
};
|
|
1056
|
-
//# sourceMappingURL=chunk-
|
|
1056
|
+
//# sourceMappingURL=chunk-FO7HEH76.js.map
|