@tangle-network/agent-eval 0.126.7 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +2 -2
- package/dist/analyst/index.d.ts +51 -39
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +34 -36
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +432 -146
- package/dist/campaign/index.js +11 -9
- package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
- package/dist/chunk-424XHVKV.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
- package/dist/chunk-65QETEKG.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
- package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
- package/dist/chunk-CLELFRQG.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
- package/dist/chunk-FA3MFYNA.js.map +1 -0
- package/dist/chunk-IG35YDN3.js +614 -0
- package/dist/chunk-IG35YDN3.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
- package/dist/chunk-IMGJZBFG.js.map +1 -0
- package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
- package/dist/chunk-IN455XKS.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
- package/dist/chunk-NL33OXEP.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
- package/dist/chunk-OD5NQ2CS.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
- package/dist/chunk-PRB53E3J.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
- package/dist/chunk-UTA2TGC4.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
- package/dist/chunk-WED5KLZQ.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
- package/dist/chunk-X6VOPMLD.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
- package/dist/chunk-Y5CYOL42.js.map +1 -0
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/chunk-ZPUEEM5U.js +179 -0
- package/dist/chunk-ZPUEEM5U.js.map +1 -0
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +421 -124
- package/dist/contract/index.js +242 -115
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +34 -32
- package/dist/control.js +2 -2
- package/dist/hosted/index.d.ts +142 -41
- package/dist/hosted/index.js +23 -1
- package/dist/index.d.ts +364 -255
- package/dist/index.js +315 -226
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +32 -32
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +97 -74
- package/dist/reporting.js +5 -4
- package/dist/rl.d.ts +177 -131
- package/dist/rl.js +232 -126
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +53 -63
- package/dist/rollout/index.js +7 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +66 -41
- package/dist/traces.js +9 -5
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +57 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
// src/hosted/types.ts
|
|
2
|
+
var HOSTED_WIRE_VERSION = "2026-07-24.v1";
|
|
3
|
+
|
|
4
|
+
// src/hosted/schemas.ts
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
var finiteNumber = z.number().finite();
|
|
7
|
+
var nonNegativeNumber = finiteNumber.nonnegative();
|
|
8
|
+
var nonNegativeInteger = z.number().int().nonnegative();
|
|
9
|
+
var nonEmptyString = z.string().min(1).refine((value) => value.trim().length > 0, "must not be blank");
|
|
10
|
+
var attributeValue = z.union([z.string(), finiteNumber, z.boolean()]);
|
|
11
|
+
var attributes = z.record(z.string(), attributeValue);
|
|
12
|
+
var UINT64_MAX = 18446744073709551615n;
|
|
13
|
+
var UnixNanoTimestampSchema = z.string().regex(/^(0|[1-9][0-9]*)$/, "expected an unsigned base-10 integer string").refine((value) => BigInt(value) <= UINT64_MAX, "must fit in an unsigned 64-bit integer");
|
|
14
|
+
var GainDistributionBinSchema = z.object({
|
|
15
|
+
lo: finiteNumber,
|
|
16
|
+
hi: finiteNumber,
|
|
17
|
+
count: nonNegativeInteger
|
|
18
|
+
}).strict();
|
|
19
|
+
var ScalarDistributionSchema = z.object({
|
|
20
|
+
n: nonNegativeInteger,
|
|
21
|
+
mean: finiteNumber.nullable(),
|
|
22
|
+
p50: finiteNumber.nullable(),
|
|
23
|
+
p95: finiteNumber.nullable(),
|
|
24
|
+
stddev: nonNegativeNumber.nullable(),
|
|
25
|
+
min: finiteNumber.nullable(),
|
|
26
|
+
max: finiteNumber.nullable(),
|
|
27
|
+
histogram: z.array(GainDistributionBinSchema),
|
|
28
|
+
tailRuns: z.array(
|
|
29
|
+
z.object({
|
|
30
|
+
runId: nonEmptyString,
|
|
31
|
+
score: finiteNumber
|
|
32
|
+
}).strict()
|
|
33
|
+
).optional()
|
|
34
|
+
}).strict().superRefine((distribution, ctx) => {
|
|
35
|
+
const values = [
|
|
36
|
+
distribution.mean,
|
|
37
|
+
distribution.p50,
|
|
38
|
+
distribution.p95,
|
|
39
|
+
distribution.stddev,
|
|
40
|
+
distribution.min,
|
|
41
|
+
distribution.max
|
|
42
|
+
];
|
|
43
|
+
if (distribution.n === 0 && values.some((value) => value !== null)) {
|
|
44
|
+
ctx.addIssue({
|
|
45
|
+
code: "custom",
|
|
46
|
+
message: "distribution values must be null when n is 0"
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (distribution.n > 0 && values.some((value) => value === null)) {
|
|
50
|
+
ctx.addIssue({
|
|
51
|
+
code: "custom",
|
|
52
|
+
message: "distribution values must be numbers when n is greater than 0"
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
if (distribution.min !== null && distribution.max !== null && distribution.min > distribution.max) {
|
|
56
|
+
ctx.addIssue({ code: "custom", path: ["min"], message: "min must not exceed max" });
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
var TokenUsageInsightSchema = z.object({
|
|
60
|
+
input: ScalarDistributionSchema,
|
|
61
|
+
output: ScalarDistributionSchema,
|
|
62
|
+
reasoning: ScalarDistributionSchema,
|
|
63
|
+
cached: ScalarDistributionSchema,
|
|
64
|
+
cacheWrite: ScalarDistributionSchema,
|
|
65
|
+
totals: z.object({
|
|
66
|
+
input: nonNegativeNumber,
|
|
67
|
+
output: nonNegativeNumber,
|
|
68
|
+
reasoning: nonNegativeNumber,
|
|
69
|
+
cached: nonNegativeNumber,
|
|
70
|
+
cacheWrite: nonNegativeNumber
|
|
71
|
+
}).strict()
|
|
72
|
+
}).strict();
|
|
73
|
+
var ExecutionErrorOutcomeCellSchema = z.object({
|
|
74
|
+
withErrors: nonNegativeInteger,
|
|
75
|
+
withoutErrors: nonNegativeInteger,
|
|
76
|
+
unreported: nonNegativeInteger
|
|
77
|
+
}).strict();
|
|
78
|
+
var ExecutionInsightSchema = z.object({
|
|
79
|
+
durationMs: ScalarDistributionSchema,
|
|
80
|
+
queueMs: ScalarDistributionSchema,
|
|
81
|
+
tokenUsage: TokenUsageInsightSchema,
|
|
82
|
+
aggregateUsage: z.object({
|
|
83
|
+
runs: nonNegativeInteger,
|
|
84
|
+
tokenUsage: TokenUsageInsightSchema,
|
|
85
|
+
costUsd: ScalarDistributionSchema,
|
|
86
|
+
totalCostUsd: nonNegativeNumber
|
|
87
|
+
}).strict(),
|
|
88
|
+
models: z.array(
|
|
89
|
+
z.object({
|
|
90
|
+
model: nonEmptyString,
|
|
91
|
+
runs: nonNegativeInteger
|
|
92
|
+
}).strict()
|
|
93
|
+
),
|
|
94
|
+
modelCalls: z.object({
|
|
95
|
+
runs: nonNegativeInteger,
|
|
96
|
+
events: nonNegativeInteger,
|
|
97
|
+
reportingRuns: nonNegativeInteger
|
|
98
|
+
}).strict(),
|
|
99
|
+
executionErrors: z.object({
|
|
100
|
+
runs: nonNegativeInteger,
|
|
101
|
+
fraction: finiteNumber.min(0).max(1).nullable(),
|
|
102
|
+
events: nonNegativeInteger,
|
|
103
|
+
reportingRuns: nonNegativeInteger,
|
|
104
|
+
errorSpanEvents: nonNegativeInteger,
|
|
105
|
+
errorSpanReportingRuns: nonNegativeInteger,
|
|
106
|
+
byTerminalOutcome: z.object({
|
|
107
|
+
succeeded: ExecutionErrorOutcomeCellSchema,
|
|
108
|
+
failed: ExecutionErrorOutcomeCellSchema,
|
|
109
|
+
cancelled: ExecutionErrorOutcomeCellSchema,
|
|
110
|
+
incomplete: ExecutionErrorOutcomeCellSchema,
|
|
111
|
+
unknown: ExecutionErrorOutcomeCellSchema
|
|
112
|
+
}).strict()
|
|
113
|
+
}).strict(),
|
|
114
|
+
terminalOutcomes: z.object({
|
|
115
|
+
succeeded: nonNegativeInteger,
|
|
116
|
+
failed: nonNegativeInteger,
|
|
117
|
+
cancelled: nonNegativeInteger,
|
|
118
|
+
incomplete: nonNegativeInteger,
|
|
119
|
+
unknown: nonNegativeInteger
|
|
120
|
+
}).strict()
|
|
121
|
+
}).strict();
|
|
122
|
+
var CostProvenanceSummarySchema = z.object({
|
|
123
|
+
observed: z.object({
|
|
124
|
+
n: nonNegativeInteger,
|
|
125
|
+
totalUsd: nonNegativeNumber
|
|
126
|
+
}).strict(),
|
|
127
|
+
estimated: z.object({
|
|
128
|
+
n: nonNegativeInteger,
|
|
129
|
+
totalUsd: nonNegativeNumber
|
|
130
|
+
}).strict(),
|
|
131
|
+
uncaptured: z.object({ n: nonNegativeInteger }).strict(),
|
|
132
|
+
knownFraction: finiteNumber.min(0).max(1)
|
|
133
|
+
}).strict();
|
|
134
|
+
var ParetoFigureSpecSchema = z.object({
|
|
135
|
+
kind: z.literal("pareto-cost-quality"),
|
|
136
|
+
split: z.enum(["search", "holdout"]),
|
|
137
|
+
points: z.array(
|
|
138
|
+
z.object({
|
|
139
|
+
candidateId: nonEmptyString,
|
|
140
|
+
cost: nonNegativeNumber,
|
|
141
|
+
quality: finiteNumber,
|
|
142
|
+
n: nonNegativeInteger,
|
|
143
|
+
onFrontier: z.boolean(),
|
|
144
|
+
gate: z.enum(["promote", "reject"]).optional()
|
|
145
|
+
}).strict()
|
|
146
|
+
),
|
|
147
|
+
axes: z.object({ x: z.literal("costUsd"), y: z.literal("score") }).strict()
|
|
148
|
+
}).strict();
|
|
149
|
+
var ContinuousAgreementSchema = z.object({
|
|
150
|
+
weightedKappa: finiteNumber,
|
|
151
|
+
icc: finiteNumber,
|
|
152
|
+
pearson: finiteNumber,
|
|
153
|
+
spearman: finiteNumber,
|
|
154
|
+
ci: z.object({
|
|
155
|
+
icc: z.tuple([finiteNumber, finiteNumber]),
|
|
156
|
+
weightedKappa: z.tuple([finiteNumber, finiteNumber])
|
|
157
|
+
}).strict(),
|
|
158
|
+
n: nonNegativeInteger,
|
|
159
|
+
raters: nonNegativeInteger
|
|
160
|
+
}).strict();
|
|
161
|
+
var JudgeInsightSchema = z.object({
|
|
162
|
+
n: nonNegativeInteger,
|
|
163
|
+
meanScore: finiteNumber,
|
|
164
|
+
calibration: ContinuousAgreementSchema.optional(),
|
|
165
|
+
positionalBias: finiteNumber.optional(),
|
|
166
|
+
selfPreference: finiteNumber.optional(),
|
|
167
|
+
verbosityBias: finiteNumber.optional()
|
|
168
|
+
}).strict();
|
|
169
|
+
var InterRaterInsightSchema = z.object({
|
|
170
|
+
raters: nonNegativeInteger,
|
|
171
|
+
jointlyRated: nonNegativeInteger,
|
|
172
|
+
kappa: finiteNumber,
|
|
173
|
+
icc: finiteNumber,
|
|
174
|
+
pearson: finiteNumber,
|
|
175
|
+
spearman: finiteNumber,
|
|
176
|
+
perPair: z.record(z.string(), finiteNumber),
|
|
177
|
+
disagreementCases: z.array(
|
|
178
|
+
z.object({
|
|
179
|
+
runId: nonEmptyString,
|
|
180
|
+
ratings: z.array(
|
|
181
|
+
z.object({
|
|
182
|
+
rater: nonEmptyString,
|
|
183
|
+
score: finiteNumber
|
|
184
|
+
}).strict()
|
|
185
|
+
),
|
|
186
|
+
range: nonNegativeNumber
|
|
187
|
+
}).strict()
|
|
188
|
+
)
|
|
189
|
+
}).strict();
|
|
190
|
+
var LiftInsightSchema = z.object({
|
|
191
|
+
baselineMean: finiteNumber,
|
|
192
|
+
candidateMean: finiteNumber,
|
|
193
|
+
delta: finiteNumber,
|
|
194
|
+
ci95: z.tuple([finiteNumber, finiteNumber]),
|
|
195
|
+
pValue: finiteNumber.min(0).max(1),
|
|
196
|
+
n: nonNegativeInteger,
|
|
197
|
+
unpairedBaseline: nonNegativeInteger,
|
|
198
|
+
unpairedCandidate: nonNegativeInteger,
|
|
199
|
+
cohensD: finiteNumber.nullable(),
|
|
200
|
+
mde: nonNegativeNumber,
|
|
201
|
+
requiredN: nonNegativeInteger.nullable()
|
|
202
|
+
}).strict();
|
|
203
|
+
var FailureClusterInsightSchema = z.object({
|
|
204
|
+
clusters: z.array(
|
|
205
|
+
z.object({
|
|
206
|
+
id: nonEmptyString,
|
|
207
|
+
name: nonEmptyString,
|
|
208
|
+
share: finiteNumber.min(0).max(1),
|
|
209
|
+
exemplars: z.array(nonEmptyString).max(5),
|
|
210
|
+
suggestedFix: nonEmptyString.optional()
|
|
211
|
+
}).strict()
|
|
212
|
+
),
|
|
213
|
+
totalFailures: nonNegativeInteger
|
|
214
|
+
}).strict();
|
|
215
|
+
var ContaminationInsightSchema = z.object({
|
|
216
|
+
leaks: nonNegativeInteger,
|
|
217
|
+
holdoutAuditPassed: z.boolean(),
|
|
218
|
+
details: z.array(
|
|
219
|
+
z.object({
|
|
220
|
+
runId: nonEmptyString,
|
|
221
|
+
canary: nonEmptyString,
|
|
222
|
+
matched: nonEmptyString
|
|
223
|
+
}).strict()
|
|
224
|
+
).optional()
|
|
225
|
+
}).strict();
|
|
226
|
+
var OutcomeCorrelationInsightSchema = z.object({
|
|
227
|
+
metric: nonEmptyString,
|
|
228
|
+
n: nonNegativeInteger,
|
|
229
|
+
pearson: finiteNumber,
|
|
230
|
+
spearman: finiteNumber,
|
|
231
|
+
rewardModel: z.object({
|
|
232
|
+
intercept: finiteNumber,
|
|
233
|
+
slope: finiteNumber,
|
|
234
|
+
r2: finiteNumber
|
|
235
|
+
}).strict().optional()
|
|
236
|
+
}).strict();
|
|
237
|
+
var ReleaseSummarySchema = z.object({
|
|
238
|
+
status: z.enum(["pass", "warn", "fail"]),
|
|
239
|
+
axes: z.array(
|
|
240
|
+
z.object({
|
|
241
|
+
name: z.enum(["quality-lift", "contamination", "composite-distribution"]),
|
|
242
|
+
status: z.enum(["pass", "warn", "fail", "not_evaluated"]),
|
|
243
|
+
detail: nonEmptyString
|
|
244
|
+
}).strict()
|
|
245
|
+
),
|
|
246
|
+
issues: z.array(z.string())
|
|
247
|
+
}).strict();
|
|
248
|
+
var MetricDeltaSchema = z.object({
|
|
249
|
+
current: finiteNumber,
|
|
250
|
+
baseline: finiteNumber,
|
|
251
|
+
delta: finiteNumber,
|
|
252
|
+
ci95: z.tuple([finiteNumber, finiteNumber]),
|
|
253
|
+
pValue: finiteNumber.min(0).max(1),
|
|
254
|
+
cohensD: finiteNumber,
|
|
255
|
+
baselineN: nonNegativeInteger,
|
|
256
|
+
currentN: nonNegativeInteger,
|
|
257
|
+
significant: z.boolean()
|
|
258
|
+
}).strict();
|
|
259
|
+
var PriorPeriodComparisonSchema = z.object({
|
|
260
|
+
baselineN: nonNegativeInteger,
|
|
261
|
+
currentN: nonNegativeInteger,
|
|
262
|
+
windowLabel: nonEmptyString.optional(),
|
|
263
|
+
metrics: z.record(z.string(), MetricDeltaSchema),
|
|
264
|
+
regressedMetrics: z.array(nonEmptyString),
|
|
265
|
+
improvedMetrics: z.array(nonEmptyString)
|
|
266
|
+
}).strict();
|
|
267
|
+
var RecommendationSchema = z.object({
|
|
268
|
+
priority: z.enum(["critical", "high", "medium", "low"]),
|
|
269
|
+
kind: z.enum(["ship", "hold", "investigate", "fix", "recalibrate", "expand-corpus"]),
|
|
270
|
+
title: nonEmptyString,
|
|
271
|
+
detail: nonEmptyString,
|
|
272
|
+
evidencePath: nonEmptyString.optional()
|
|
273
|
+
}).strict();
|
|
274
|
+
var InsightReportSchema = z.object({
|
|
275
|
+
n: nonNegativeInteger,
|
|
276
|
+
execution: ExecutionInsightSchema,
|
|
277
|
+
composite: ScalarDistributionSchema,
|
|
278
|
+
perDimension: z.record(z.string(), ScalarDistributionSchema),
|
|
279
|
+
costQuality: z.object({
|
|
280
|
+
cost: ScalarDistributionSchema,
|
|
281
|
+
pareto: ParetoFigureSpecSchema,
|
|
282
|
+
provenance: CostProvenanceSummarySchema.optional(),
|
|
283
|
+
degraded: z.object({
|
|
284
|
+
cost: nonEmptyString.optional(),
|
|
285
|
+
pareto: nonEmptyString.optional()
|
|
286
|
+
}).strict().optional()
|
|
287
|
+
}).strict(),
|
|
288
|
+
judges: z.record(z.string(), JudgeInsightSchema),
|
|
289
|
+
interRater: InterRaterInsightSchema.optional(),
|
|
290
|
+
lift: LiftInsightSchema.optional(),
|
|
291
|
+
failureClusters: FailureClusterInsightSchema.optional(),
|
|
292
|
+
contamination: ContaminationInsightSchema.optional(),
|
|
293
|
+
outcomeCorrelation: OutcomeCorrelationInsightSchema.optional(),
|
|
294
|
+
release: ReleaseSummarySchema,
|
|
295
|
+
priorPeriodComparison: PriorPeriodComparisonSchema.optional(),
|
|
296
|
+
failureModes: z.array(
|
|
297
|
+
z.object({
|
|
298
|
+
mode: nonEmptyString,
|
|
299
|
+
count: nonNegativeInteger,
|
|
300
|
+
share: finiteNumber.min(0).max(1)
|
|
301
|
+
}).strict()
|
|
302
|
+
).optional(),
|
|
303
|
+
recommendations: z.array(RecommendationSchema)
|
|
304
|
+
}).strict();
|
|
305
|
+
var sha256Digest = z.custom(
|
|
306
|
+
(value) => typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value),
|
|
307
|
+
"expected sha256:<64 lowercase hex characters>"
|
|
308
|
+
);
|
|
309
|
+
var MutableSurfaceSchema = z.union([
|
|
310
|
+
z.string(),
|
|
311
|
+
z.object({
|
|
312
|
+
kind: z.literal("components"),
|
|
313
|
+
components: z.record(z.string(), z.string())
|
|
314
|
+
}).strict(),
|
|
315
|
+
z.object({
|
|
316
|
+
kind: z.literal("code"),
|
|
317
|
+
worktreeRef: nonEmptyString,
|
|
318
|
+
baseRef: nonEmptyString,
|
|
319
|
+
baseCommit: nonEmptyString,
|
|
320
|
+
baseTree: nonEmptyString,
|
|
321
|
+
candidateCommit: nonEmptyString,
|
|
322
|
+
candidateTree: nonEmptyString,
|
|
323
|
+
patch: z.object({
|
|
324
|
+
format: z.literal("git-diff-binary"),
|
|
325
|
+
sha256: sha256Digest,
|
|
326
|
+
byteLength: nonNegativeInteger
|
|
327
|
+
}).strict(),
|
|
328
|
+
summary: z.string().optional()
|
|
329
|
+
}).strict()
|
|
330
|
+
]);
|
|
331
|
+
var RunTerminalOutcomeSchema = z.enum([
|
|
332
|
+
"succeeded",
|
|
333
|
+
"failed",
|
|
334
|
+
"cancelled",
|
|
335
|
+
"incomplete",
|
|
336
|
+
"unknown"
|
|
337
|
+
]);
|
|
338
|
+
var EvalRunCellScoreSchema = z.object({
|
|
339
|
+
scenarioId: nonEmptyString,
|
|
340
|
+
rep: nonNegativeInteger,
|
|
341
|
+
compositeMean: finiteNumber.nullable(),
|
|
342
|
+
dimensions: z.record(z.string(), z.record(z.string(), finiteNumber)),
|
|
343
|
+
terminalOutcome: RunTerminalOutcomeSchema,
|
|
344
|
+
executionErrorCount: nonNegativeInteger.nullable(),
|
|
345
|
+
errorMessage: z.string().optional()
|
|
346
|
+
}).strict();
|
|
347
|
+
var EvalRunGenerationSnapshotSchema = z.object({
|
|
348
|
+
index: nonNegativeInteger,
|
|
349
|
+
surfaceHash: nonEmptyString,
|
|
350
|
+
surface: MutableSurfaceSchema.optional(),
|
|
351
|
+
cells: z.array(EvalRunCellScoreSchema),
|
|
352
|
+
compositeMean: finiteNumber.nullable(),
|
|
353
|
+
costUsd: nonNegativeNumber,
|
|
354
|
+
durationMs: nonNegativeNumber
|
|
355
|
+
}).strict();
|
|
356
|
+
var EvalRunStatusSchema = z.enum([
|
|
357
|
+
"started",
|
|
358
|
+
"baseline-complete",
|
|
359
|
+
"generation-complete",
|
|
360
|
+
"gate-decided",
|
|
361
|
+
"finished",
|
|
362
|
+
"errored"
|
|
363
|
+
]);
|
|
364
|
+
var GateDecisionSchema = z.enum([
|
|
365
|
+
"ship",
|
|
366
|
+
"hold",
|
|
367
|
+
"need_more_work",
|
|
368
|
+
"model_ceiling",
|
|
369
|
+
"arch_ceiling"
|
|
370
|
+
]);
|
|
371
|
+
var EvalRunEventSchema = z.object({
|
|
372
|
+
runId: nonEmptyString,
|
|
373
|
+
runDir: nonEmptyString,
|
|
374
|
+
timestamp: z.string().datetime({ offset: true }),
|
|
375
|
+
status: EvalRunStatusSchema,
|
|
376
|
+
labels: z.record(z.string(), z.string()),
|
|
377
|
+
baseline: EvalRunGenerationSnapshotSchema.optional(),
|
|
378
|
+
generations: z.array(EvalRunGenerationSnapshotSchema),
|
|
379
|
+
gateDecision: GateDecisionSchema.optional(),
|
|
380
|
+
holdoutLift: finiteNumber.optional(),
|
|
381
|
+
totalCostUsd: nonNegativeNumber,
|
|
382
|
+
totalDurationMs: nonNegativeNumber,
|
|
383
|
+
errorMessage: z.string().optional(),
|
|
384
|
+
insightReport: InsightReportSchema.optional()
|
|
385
|
+
}).strict().superRefine((event, ctx) => {
|
|
386
|
+
if (event.baseline && event.baseline.index !== 0) {
|
|
387
|
+
ctx.addIssue({
|
|
388
|
+
code: "custom",
|
|
389
|
+
path: ["baseline", "index"],
|
|
390
|
+
message: "baseline index must be 0"
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
const seen = /* @__PURE__ */ new Set();
|
|
394
|
+
for (let i = 0; i < event.generations.length; i++) {
|
|
395
|
+
const index = event.generations[i].index;
|
|
396
|
+
if (seen.has(index)) {
|
|
397
|
+
ctx.addIssue({
|
|
398
|
+
code: "custom",
|
|
399
|
+
path: ["generations", i, "index"],
|
|
400
|
+
message: `duplicate generation index ${index}`
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
seen.add(index);
|
|
404
|
+
}
|
|
405
|
+
if (event.status === "errored" && !event.errorMessage?.trim()) {
|
|
406
|
+
ctx.addIssue({
|
|
407
|
+
code: "custom",
|
|
408
|
+
path: ["errorMessage"],
|
|
409
|
+
message: "errorMessage is required when status is errored"
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
});
|
|
413
|
+
var TraceSpanEventEntrySchema = z.object({
|
|
414
|
+
timeUnixNano: UnixNanoTimestampSchema,
|
|
415
|
+
name: nonEmptyString,
|
|
416
|
+
attributes: attributes.optional()
|
|
417
|
+
}).strict();
|
|
418
|
+
var TraceSpanEventSchema = z.object({
|
|
419
|
+
traceId: nonEmptyString,
|
|
420
|
+
spanId: nonEmptyString,
|
|
421
|
+
parentSpanId: nonEmptyString.optional(),
|
|
422
|
+
name: nonEmptyString,
|
|
423
|
+
startTimeUnixNano: UnixNanoTimestampSchema,
|
|
424
|
+
endTimeUnixNano: UnixNanoTimestampSchema,
|
|
425
|
+
attributes,
|
|
426
|
+
events: z.array(TraceSpanEventEntrySchema).optional(),
|
|
427
|
+
status: z.object({
|
|
428
|
+
code: z.enum(["OK", "ERROR", "UNSET"]),
|
|
429
|
+
message: z.string().optional()
|
|
430
|
+
}).strict().optional(),
|
|
431
|
+
"tangle.runId": nonEmptyString.optional(),
|
|
432
|
+
"tangle.generation": nonNegativeInteger.optional(),
|
|
433
|
+
"tangle.cellId": nonEmptyString.optional(),
|
|
434
|
+
"tangle.scenarioId": nonEmptyString.optional()
|
|
435
|
+
}).strict().refine((span) => BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano), {
|
|
436
|
+
path: ["endTimeUnixNano"],
|
|
437
|
+
message: "endTimeUnixNano must be greater than or equal to startTimeUnixNano"
|
|
438
|
+
});
|
|
439
|
+
var IngestEvalRunsEnvelopeSchema = z.object({
|
|
440
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
441
|
+
events: z.array(z.unknown())
|
|
442
|
+
}).strict();
|
|
443
|
+
var IngestTracesEnvelopeSchema = z.object({
|
|
444
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
445
|
+
spans: z.array(z.unknown())
|
|
446
|
+
}).strict();
|
|
447
|
+
var IngestEvalRunsRequestSchema = z.object({
|
|
448
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
449
|
+
events: z.array(EvalRunEventSchema)
|
|
450
|
+
}).strict();
|
|
451
|
+
var IngestTracesRequestSchema = z.object({
|
|
452
|
+
wireVersion: z.literal(HOSTED_WIRE_VERSION),
|
|
453
|
+
spans: z.array(TraceSpanEventSchema)
|
|
454
|
+
}).strict();
|
|
455
|
+
var IngestResponseSchema = z.object({
|
|
456
|
+
accepted: nonNegativeInteger,
|
|
457
|
+
rejected: z.array(
|
|
458
|
+
z.object({
|
|
459
|
+
index: nonNegativeInteger,
|
|
460
|
+
reason: nonEmptyString
|
|
461
|
+
}).strict()
|
|
462
|
+
)
|
|
463
|
+
}).strict();
|
|
464
|
+
|
|
465
|
+
// src/hosted/client.ts
|
|
466
|
+
var MAX_IDEMPOTENCY_KEY_LENGTH = 256;
|
|
467
|
+
function sleep(ms) {
|
|
468
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
469
|
+
}
|
|
470
|
+
function normalizeHostedBase(endpoint) {
|
|
471
|
+
return endpoint.trim().replace(/\/+$/, "").replace(/\/v1$/, "");
|
|
472
|
+
}
|
|
473
|
+
function resolveIdempotencyKey(key) {
|
|
474
|
+
const resolved = key ?? globalThis.crypto.randomUUID();
|
|
475
|
+
if (resolved.trim().length === 0) throw new Error("idempotency key must not be blank");
|
|
476
|
+
if (resolved.length > MAX_IDEMPOTENCY_KEY_LENGTH) {
|
|
477
|
+
throw new Error(`idempotency key must be at most ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`);
|
|
478
|
+
}
|
|
479
|
+
return resolved;
|
|
480
|
+
}
|
|
481
|
+
function responseValidationReason(error) {
|
|
482
|
+
return error.issues.map(
|
|
483
|
+
(issue) => `${issue.path.length > 0 ? issue.path.map(String).join(".") : "value"}: ${issue.message}`
|
|
484
|
+
).join("; ");
|
|
485
|
+
}
|
|
486
|
+
async function post(tenant, path, body, opts = {}) {
|
|
487
|
+
const timeoutMs = tenant.timeoutMs ?? 3e4;
|
|
488
|
+
const maxRetries = tenant.retries ?? 2;
|
|
489
|
+
const f = tenant.fetchImpl ?? ((...args) => fetch(...args));
|
|
490
|
+
const base = normalizeHostedBase(tenant.endpoint);
|
|
491
|
+
const url = `${base}${path}`;
|
|
492
|
+
const idempotencyKey = resolveIdempotencyKey(opts.idempotencyKey);
|
|
493
|
+
const headers = {
|
|
494
|
+
"content-type": "application/json",
|
|
495
|
+
authorization: `Bearer ${tenant.apiKey}`,
|
|
496
|
+
"x-tangle-tenant-id": tenant.tenantId,
|
|
497
|
+
"x-tangle-wire-version": HOSTED_WIRE_VERSION,
|
|
498
|
+
"idempotency-key": idempotencyKey
|
|
499
|
+
};
|
|
500
|
+
let lastError;
|
|
501
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
502
|
+
const ourTimeout = AbortSignal.timeout(timeoutMs);
|
|
503
|
+
const combinedSignal = opts.signal ? AbortSignal.any([opts.signal, ourTimeout]) : ourTimeout;
|
|
504
|
+
let res;
|
|
505
|
+
try {
|
|
506
|
+
res = await f(url, {
|
|
507
|
+
method: "POST",
|
|
508
|
+
headers,
|
|
509
|
+
body: JSON.stringify(body),
|
|
510
|
+
signal: combinedSignal
|
|
511
|
+
});
|
|
512
|
+
} catch (err) {
|
|
513
|
+
if (opts.signal?.aborted) throw err;
|
|
514
|
+
lastError = err;
|
|
515
|
+
if (attempt === maxRetries) throw err;
|
|
516
|
+
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (!res.ok) {
|
|
520
|
+
const text = await res.text().catch(() => "");
|
|
521
|
+
const error = new Error(`hosted ingest ${url} failed (${res.status}): ${text.slice(0, 500)}`);
|
|
522
|
+
const retryable = res.status >= 500 || res.status === 408 || res.status === 429;
|
|
523
|
+
if (!retryable || attempt === maxRetries) throw error;
|
|
524
|
+
lastError = error;
|
|
525
|
+
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
let rawResponse;
|
|
529
|
+
try {
|
|
530
|
+
rawResponse = await res.json();
|
|
531
|
+
} catch (error) {
|
|
532
|
+
throw new Error(`hosted ingest ${url} returned invalid JSON`, { cause: error });
|
|
533
|
+
}
|
|
534
|
+
const parsed = IngestResponseSchema.safeParse(rawResponse);
|
|
535
|
+
if (!parsed.success) {
|
|
536
|
+
throw new Error(
|
|
537
|
+
`hosted ingest ${url} returned an invalid response: ${responseValidationReason(parsed.error)}`
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
return parsed.data;
|
|
541
|
+
}
|
|
542
|
+
throw lastError ?? new Error("hosted ingest exhausted retries");
|
|
543
|
+
}
|
|
544
|
+
function createHostedClient(tenant) {
|
|
545
|
+
if (normalizeHostedBase(tenant.endpoint).length === 0) throw new Error("endpoint is required");
|
|
546
|
+
if (tenant.apiKey.trim().length === 0) throw new Error("apiKey is required");
|
|
547
|
+
if (tenant.tenantId.trim().length === 0) throw new Error("tenantId is required");
|
|
548
|
+
if (tenant.timeoutMs !== void 0 && (!Number.isFinite(tenant.timeoutMs) || tenant.timeoutMs <= 0)) {
|
|
549
|
+
throw new Error("timeoutMs must be greater than 0");
|
|
550
|
+
}
|
|
551
|
+
if (tenant.retries !== void 0 && (!Number.isInteger(tenant.retries) || tenant.retries < 0)) {
|
|
552
|
+
throw new Error("retries must be a non-negative integer");
|
|
553
|
+
}
|
|
554
|
+
return {
|
|
555
|
+
tenant,
|
|
556
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
557
|
+
async ingestEvalRun(event, idempotencyKey) {
|
|
558
|
+
return this.ingestEvalRuns([event], idempotencyKey);
|
|
559
|
+
},
|
|
560
|
+
async ingestEvalRuns(events, idempotencyKey) {
|
|
561
|
+
const body = IngestEvalRunsRequestSchema.parse({
|
|
562
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
563
|
+
events
|
|
564
|
+
});
|
|
565
|
+
return post(tenant, "/v1/ingest/eval-runs", body, {
|
|
566
|
+
idempotencyKey
|
|
567
|
+
});
|
|
568
|
+
},
|
|
569
|
+
async ingestTraces(spans, idempotencyKey) {
|
|
570
|
+
const body = IngestTracesRequestSchema.parse({
|
|
571
|
+
wireVersion: HOSTED_WIRE_VERSION,
|
|
572
|
+
spans
|
|
573
|
+
});
|
|
574
|
+
return post(tenant, "/v1/ingest/traces", body, {
|
|
575
|
+
idempotencyKey
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
function hostedTenantFromEnv(overrides = {}) {
|
|
581
|
+
const env = overrides.env ?? process.env;
|
|
582
|
+
const endpoint = (overrides.endpoint ?? env.TANGLE_INGEST_URL ?? env.TANGLE_ORCHESTRATOR_URL)?.trim();
|
|
583
|
+
const apiKey = (overrides.apiKey ?? env.TANGLE_INGEST_API_KEY ?? env.TANGLE_API_KEY)?.trim();
|
|
584
|
+
const tenantId = (overrides.tenantId ?? env.TANGLE_TENANT_ID)?.trim();
|
|
585
|
+
if (!endpoint || !apiKey || !tenantId) return void 0;
|
|
586
|
+
const tenant = { endpoint: endpoint.replace(/\/+$/, ""), apiKey, tenantId };
|
|
587
|
+
if (overrides.fetchImpl) tenant.fetchImpl = overrides.fetchImpl;
|
|
588
|
+
if (overrides.timeoutMs !== void 0) tenant.timeoutMs = overrides.timeoutMs;
|
|
589
|
+
if (overrides.retries !== void 0) tenant.retries = overrides.retries;
|
|
590
|
+
return tenant;
|
|
591
|
+
}
|
|
592
|
+
function hostedClientFromEnv(overrides = {}) {
|
|
593
|
+
const tenant = hostedTenantFromEnv(overrides);
|
|
594
|
+
return tenant ? createHostedClient(tenant) : void 0;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
export {
|
|
598
|
+
HOSTED_WIRE_VERSION,
|
|
599
|
+
UnixNanoTimestampSchema,
|
|
600
|
+
InsightReportSchema,
|
|
601
|
+
MutableSurfaceSchema,
|
|
602
|
+
RunTerminalOutcomeSchema,
|
|
603
|
+
EvalRunCellScoreSchema,
|
|
604
|
+
EvalRunGenerationSnapshotSchema,
|
|
605
|
+
EvalRunEventSchema,
|
|
606
|
+
TraceSpanEventSchema,
|
|
607
|
+
IngestEvalRunsRequestSchema,
|
|
608
|
+
IngestTracesRequestSchema,
|
|
609
|
+
IngestResponseSchema,
|
|
610
|
+
createHostedClient,
|
|
611
|
+
hostedTenantFromEnv,
|
|
612
|
+
hostedClientFromEnv
|
|
613
|
+
};
|
|
614
|
+
//# sourceMappingURL=chunk-IG35YDN3.js.map
|