@fevex/core 0.1.0-alpha.1 → 0.1.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +424 -6
- package/dist/agents/index.d.mts +1 -1
- package/dist/agents/index.mjs +1 -1
- package/dist/channels/index.d.mts +2 -0
- package/dist/channels/index.mjs +150 -0
- package/dist/evals/index.d.mts +19 -11
- package/dist/evals/index.mjs +125 -122
- package/dist/fevex-Ddyi6z6L.d.mts +60 -0
- package/dist/http/index.d.mts +90 -0
- package/dist/http/index.mjs +470 -0
- package/dist/{index-ojCY3lwm.d.mts → index-B4PIkqOU.d.mts} +5 -4
- package/dist/index-Bg9rr8Qe.d.mts +50 -0
- package/dist/index-Bgcxgy66.d.mts +82 -0
- package/dist/index-CTIwrVf9.d.mts +51 -0
- package/dist/index-CqOjvtQ9.d.mts +132 -0
- package/dist/index-CrOId2px.d.mts +66 -0
- package/dist/index-DeXUtyd9.d.mts +70 -0
- package/dist/index-DviHtsLo.d.mts +58 -0
- package/dist/index-Dx_LotWZ.d.mts +425 -0
- package/dist/{index-QTkd6a7C.d.mts → index-dkFyTNh4.d.mts} +4 -3
- package/dist/index.d.mts +22 -10
- package/dist/index.mjs +2784 -720
- package/dist/knowledge/index.d.mts +2 -0
- package/dist/knowledge/index.mjs +89 -0
- package/dist/models/index.d.mts +2 -46
- package/dist/observability/index.d.mts +1 -1
- package/dist/policies/index.d.mts +1 -1
- package/dist/policies/index.mjs +1 -1
- package/dist/run-error-Cr17wPGH.mjs +23 -0
- package/dist/runtime/index.d.mts +2 -2
- package/dist/runtime/index.mjs +58 -9
- package/dist/sandbox/index.d.mts +2 -0
- package/dist/sandbox/index.mjs +122 -0
- package/dist/teams/index.d.mts +2 -0
- package/dist/teams/index.mjs +2 -0
- package/dist/teams-Ctz676G4.mjs +82 -0
- package/dist/testing/index.d.mts +33 -9
- package/dist/testing/index.mjs +415 -27
- package/dist/tools/index.d.mts +3 -3
- package/dist/tools/index.mjs +222 -2
- package/dist/types-CuoRx12Q.d.mts +333 -0
- package/dist/workflows/index.d.mts +2 -0
- package/dist/workflows/index.mjs +6 -0
- package/package.json +27 -3
- package/dist/fevex-Dz6GBz9o.d.mts +0 -32
- package/dist/index-BzxHfyYV.d.mts +0 -23
- package/dist/index-CZGyhNIQ.d.mts +0 -47
- package/dist/index-C_Zpr13f.d.mts +0 -124
- package/dist/index-DGWOGXzc.d.mts +0 -181
package/dist/evals/index.d.mts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../
|
|
2
|
-
import { ModelMetadata, ModelUsage } from "../
|
|
3
|
-
import { l as TraceCost, t as CostCalculator } from "../index-
|
|
4
|
-
import { t as Fevex } from "../fevex-
|
|
5
|
-
//#region src/evals/
|
|
1
|
+
import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CuoRx12Q.mjs";
|
|
2
|
+
import { r as ModelMetadata, s as ModelUsage } from "../index-CTIwrVf9.mjs";
|
|
3
|
+
import { l as TraceCost, t as CostCalculator } from "../index-B4PIkqOU.mjs";
|
|
4
|
+
import { t as Fevex } from "../fevex-Ddyi6z6L.mjs";
|
|
5
|
+
//#region src/evals/canonical-json.d.ts
|
|
6
|
+
/** Serializes JSON-like data with stable object-key ordering. */
|
|
7
|
+
declare function serializeCanonical(value: unknown): string;
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/evals/types.d.ts
|
|
6
10
|
interface EvalExpectation {
|
|
7
11
|
output?: JsonValue;
|
|
8
12
|
tools?: string[];
|
|
@@ -121,7 +125,17 @@ interface RegressionOptions {
|
|
|
121
125
|
maxLatencyIncreaseRatio?: number;
|
|
122
126
|
maxCostIncreaseRatio?: number;
|
|
123
127
|
}
|
|
128
|
+
//#endregion
|
|
129
|
+
//#region src/evals/evaluation.d.ts
|
|
130
|
+
/** Runs every case and scorer, producing a deterministic aggregate report. */
|
|
124
131
|
declare function runEvaluation<TInput = unknown>(options: RunEvaluationOptions<TInput>): Promise<EvalReport>;
|
|
132
|
+
declare function serializeEvalReport(report: EvalReport): string;
|
|
133
|
+
declare function parseEvalReport(json: string): EvalReport;
|
|
134
|
+
declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
135
|
+
declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
136
|
+
declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
|
|
137
|
+
//#endregion
|
|
138
|
+
//#region src/evals/scorers.d.ts
|
|
125
139
|
declare function exactOutputScorer(id?: string): EvalScorer;
|
|
126
140
|
declare function toolSelectionScorer(options?: {
|
|
127
141
|
id?: string;
|
|
@@ -135,11 +149,5 @@ declare function maxTokensScorer(limits: {
|
|
|
135
149
|
total?: number;
|
|
136
150
|
}, id?: string): EvalScorer;
|
|
137
151
|
declare function maxCostScorer(maxAmount: number, currency: string, id?: string): EvalScorer;
|
|
138
|
-
declare function serializeCanonical(value: unknown): string;
|
|
139
|
-
declare function serializeEvalReport(report: EvalReport): string;
|
|
140
|
-
declare function parseEvalReport(json: string): EvalReport;
|
|
141
|
-
declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
142
|
-
declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
143
|
-
declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
|
|
144
152
|
//#endregion
|
|
145
153
|
export { EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalExpectation, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, Regression, RegressionOptions, RegressionReport, RunEvaluationOptions, compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
|
package/dist/evals/index.mjs
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
//#region src/evals/
|
|
1
|
+
//#region src/evals/canonical-json.ts
|
|
2
|
+
function canonicalize(value) {
|
|
3
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
4
|
+
if (value === null || typeof value !== "object") return value;
|
|
5
|
+
return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
|
|
6
|
+
}
|
|
7
|
+
/** Serializes JSON-like data with stable object-key ordering. */
|
|
8
|
+
function serializeCanonical(value) {
|
|
9
|
+
return JSON.stringify(canonicalize(value), null, 2);
|
|
10
|
+
}
|
|
11
|
+
//#endregion
|
|
12
|
+
//#region src/evals/evaluation.ts
|
|
2
13
|
function assertNonEmpty(value, name) {
|
|
3
14
|
if (typeof value !== "string" || !value.trim()) throw new TypeError(`${name} must be non-empty`);
|
|
4
15
|
}
|
|
5
16
|
function errorMessage(error) {
|
|
6
17
|
return error instanceof Error ? error.message : String(error);
|
|
7
18
|
}
|
|
8
|
-
function equalJson(left, right) {
|
|
9
|
-
return serializeCanonical(left) === serializeCanonical(right);
|
|
10
|
-
}
|
|
11
19
|
function addUsage(target, usage) {
|
|
12
20
|
if (!usage) return;
|
|
13
21
|
for (const key of [
|
|
@@ -157,6 +165,7 @@ function summarize(cases) {
|
|
|
157
165
|
latencyMs: cases.reduce((sum, testCase) => sum + testCase.latencyMs, 0)
|
|
158
166
|
};
|
|
159
167
|
}
|
|
168
|
+
/** Runs every case and scorer, producing a deterministic aggregate report. */
|
|
160
169
|
async function runEvaluation(options) {
|
|
161
170
|
validateOptions(options);
|
|
162
171
|
const cases = [];
|
|
@@ -177,6 +186,116 @@ async function runEvaluation(options) {
|
|
|
177
186
|
for (const reporter of options.reporters ?? []) await reporter.report(report);
|
|
178
187
|
return report;
|
|
179
188
|
}
|
|
189
|
+
function serializeEvalReport(report) {
|
|
190
|
+
return `${serializeCanonical(report)}\n`;
|
|
191
|
+
}
|
|
192
|
+
function parseEvalReport(json) {
|
|
193
|
+
const value = JSON.parse(json);
|
|
194
|
+
if (typeof value !== "object" || value === null || value.schemaVersion !== "fevex.eval.v1" || typeof value.suiteName !== "string" || typeof value.targetVersion !== "string" || typeof value.agentName !== "string" || typeof value.dataset !== "object" || value.dataset === null || typeof value.dataset.name !== "string" || typeof value.dataset.version !== "string" || !Array.isArray(value.cases) || !Array.isArray(value.scorerIds) || !value.scorerIds.every((scorerId) => typeof scorerId === "string" && Boolean(scorerId)) || typeof value.summary !== "object" || value.summary === null) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
195
|
+
const report = value;
|
|
196
|
+
const caseIds = /* @__PURE__ */ new Set();
|
|
197
|
+
for (const testCase of report.cases) {
|
|
198
|
+
if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
199
|
+
caseIds.add(testCase.caseId);
|
|
200
|
+
for (const score of testCase.scores) if (typeof score !== "object" || score === null || typeof score.scorerId !== "string" || !report.scorerIds.includes(score.scorerId) || ![
|
|
201
|
+
"passed",
|
|
202
|
+
"failed",
|
|
203
|
+
"skipped",
|
|
204
|
+
"error"
|
|
205
|
+
].includes(score.status) || score.score !== void 0 && (!Number.isFinite(score.score) || score.score < 0 || score.score > 1)) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
206
|
+
}
|
|
207
|
+
return value;
|
|
208
|
+
}
|
|
209
|
+
function jsonReporter(write) {
|
|
210
|
+
return { report: (report) => write(serializeEvalReport(report)) };
|
|
211
|
+
}
|
|
212
|
+
function textReporter(write) {
|
|
213
|
+
return { report(report) {
|
|
214
|
+
return write(`${[
|
|
215
|
+
`${report.suiteName} (${report.targetVersion})`,
|
|
216
|
+
`dataset ${report.dataset.name}@${report.dataset.version}`,
|
|
217
|
+
...report.cases.map((testCase) => {
|
|
218
|
+
const passed = testCase.scores.filter(({ status }) => status === "passed").length;
|
|
219
|
+
const failed = testCase.scores.filter(({ status }) => status === "failed" || status === "error").length;
|
|
220
|
+
return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed`;
|
|
221
|
+
}),
|
|
222
|
+
`summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores`
|
|
223
|
+
].join("\n")}\n`);
|
|
224
|
+
} };
|
|
225
|
+
}
|
|
226
|
+
function assertRatio(value, name) {
|
|
227
|
+
if (value !== void 0 && (!Number.isFinite(value) || value < 0)) throw new TypeError(`${name} must be non-negative`);
|
|
228
|
+
}
|
|
229
|
+
function compareEvaluationReports(baseline, current, options = {}) {
|
|
230
|
+
if (baseline.suiteName !== current.suiteName || baseline.dataset.name !== current.dataset.name || baseline.dataset.version !== current.dataset.version) throw new TypeError("Evaluation reports must use the same suite and dataset version");
|
|
231
|
+
assertRatio(options.maxScoreDrop, "maxScoreDrop");
|
|
232
|
+
assertRatio(options.maxLatencyIncreaseRatio, "maxLatencyIncreaseRatio");
|
|
233
|
+
assertRatio(options.maxCostIncreaseRatio, "maxCostIncreaseRatio");
|
|
234
|
+
const maxScoreDrop = options.maxScoreDrop ?? 0;
|
|
235
|
+
const regressions = [];
|
|
236
|
+
const currentCases = new Map(current.cases.map((testCase) => [testCase.caseId, testCase]));
|
|
237
|
+
for (const baselineCase of baseline.cases) {
|
|
238
|
+
const currentCase = currentCases.get(baselineCase.caseId);
|
|
239
|
+
if (!currentCase) {
|
|
240
|
+
regressions.push({
|
|
241
|
+
kind: "case_removed",
|
|
242
|
+
caseId: baselineCase.caseId,
|
|
243
|
+
message: `Case "${baselineCase.caseId}" was removed`
|
|
244
|
+
});
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
if (baselineCase.status === "completed" && currentCase.status === "failed") regressions.push({
|
|
248
|
+
kind: "case_failed",
|
|
249
|
+
caseId: baselineCase.caseId,
|
|
250
|
+
message: `Case "${baselineCase.caseId}" no longer completes`
|
|
251
|
+
});
|
|
252
|
+
const currentScores = new Map(currentCase.scores.map((score) => [score.scorerId, score]));
|
|
253
|
+
for (const baselineScore of baselineCase.scores) {
|
|
254
|
+
const currentScore = currentScores.get(baselineScore.scorerId);
|
|
255
|
+
if (!currentScore) {
|
|
256
|
+
regressions.push({
|
|
257
|
+
kind: "scorer_removed",
|
|
258
|
+
caseId: baselineCase.caseId,
|
|
259
|
+
scorerId: baselineScore.scorerId,
|
|
260
|
+
message: `Scorer "${baselineScore.scorerId}" was removed from "${baselineCase.caseId}"`
|
|
261
|
+
});
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (baselineScore.status === "passed" && currentScore.status !== "passed") regressions.push({
|
|
265
|
+
kind: "score_failed",
|
|
266
|
+
caseId: baselineCase.caseId,
|
|
267
|
+
scorerId: baselineScore.scorerId,
|
|
268
|
+
message: `Scorer "${baselineScore.scorerId}" changed from pass to fail`
|
|
269
|
+
});
|
|
270
|
+
else if (baselineScore.score !== void 0 && currentScore.score !== void 0 && baselineScore.score - currentScore.score > maxScoreDrop) regressions.push({
|
|
271
|
+
kind: "score_dropped",
|
|
272
|
+
caseId: baselineCase.caseId,
|
|
273
|
+
scorerId: baselineScore.scorerId,
|
|
274
|
+
message: `Scorer "${baselineScore.scorerId}" dropped from ${baselineScore.score} to ${currentScore.score}`
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
if (options.maxLatencyIncreaseRatio !== void 0 && baselineCase.latencyMs > 0 && currentCase.latencyMs / baselineCase.latencyMs - 1 > options.maxLatencyIncreaseRatio) regressions.push({
|
|
278
|
+
kind: "latency_increased",
|
|
279
|
+
caseId: baselineCase.caseId,
|
|
280
|
+
message: `Latency increased from ${baselineCase.latencyMs}ms to ${currentCase.latencyMs}ms`
|
|
281
|
+
});
|
|
282
|
+
if (options.maxCostIncreaseRatio !== void 0 && baselineCase.cost && currentCase.cost) {
|
|
283
|
+
if (baselineCase.cost.currency !== currentCase.cost.currency || baselineCase.cost.amount > 0 && currentCase.cost.amount / baselineCase.cost.amount - 1 > options.maxCostIncreaseRatio) regressions.push({
|
|
284
|
+
kind: "cost_increased",
|
|
285
|
+
caseId: baselineCase.caseId,
|
|
286
|
+
message: `Cost changed from ${baselineCase.cost.amount} ${baselineCase.cost.currency} to ${currentCase.cost.amount} ${currentCase.cost.currency}`
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return {
|
|
291
|
+
passed: regressions.length === 0,
|
|
292
|
+
baselineTargetVersion: baseline.targetVersion,
|
|
293
|
+
currentTargetVersion: current.targetVersion,
|
|
294
|
+
regressions
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
//#endregion
|
|
298
|
+
//#region src/evals/scorers.ts
|
|
180
299
|
function exactOutputScorer(id = "output.exact") {
|
|
181
300
|
return {
|
|
182
301
|
id,
|
|
@@ -185,7 +304,7 @@ function exactOutputScorer(id = "output.exact") {
|
|
|
185
304
|
skipped: true,
|
|
186
305
|
reason: "No expected output"
|
|
187
306
|
};
|
|
188
|
-
const passed =
|
|
307
|
+
const passed = serializeCanonical(output) === serializeCanonical(testCase.expected.output);
|
|
189
308
|
return {
|
|
190
309
|
passed,
|
|
191
310
|
score: passed ? 1 : 0
|
|
@@ -272,7 +391,7 @@ function maxTokensScorer(limits, id = "tokens.max") {
|
|
|
272
391
|
}
|
|
273
392
|
function maxCostScorer(maxAmount, currency, id = "cost.max") {
|
|
274
393
|
if (!Number.isFinite(maxAmount) || maxAmount < 0) throw new TypeError("maxAmount must be non-negative");
|
|
275
|
-
|
|
394
|
+
if (!currency.trim()) throw new TypeError("currency must be non-empty");
|
|
276
395
|
return {
|
|
277
396
|
id,
|
|
278
397
|
score({ cost }) {
|
|
@@ -299,121 +418,5 @@ function maxCostScorer(maxAmount, currency, id = "cost.max") {
|
|
|
299
418
|
}
|
|
300
419
|
};
|
|
301
420
|
}
|
|
302
|
-
function canonicalize(value) {
|
|
303
|
-
if (Array.isArray(value)) return value.map(canonicalize);
|
|
304
|
-
if (value === null || typeof value !== "object") return value;
|
|
305
|
-
return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
|
|
306
|
-
}
|
|
307
|
-
function serializeCanonical(value) {
|
|
308
|
-
return JSON.stringify(canonicalize(value), null, 2);
|
|
309
|
-
}
|
|
310
|
-
function serializeEvalReport(report) {
|
|
311
|
-
return `${serializeCanonical(report)}\n`;
|
|
312
|
-
}
|
|
313
|
-
function parseEvalReport(json) {
|
|
314
|
-
const value = JSON.parse(json);
|
|
315
|
-
if (typeof value !== "object" || value === null || value.schemaVersion !== "fevex.eval.v1" || typeof value.suiteName !== "string" || typeof value.targetVersion !== "string" || typeof value.agentName !== "string" || typeof value.dataset !== "object" || value.dataset === null || typeof value.dataset.name !== "string" || typeof value.dataset.version !== "string" || !Array.isArray(value.cases) || !Array.isArray(value.scorerIds) || !value.scorerIds.every((scorerId) => typeof scorerId === "string" && Boolean(scorerId)) || typeof value.summary !== "object" || value.summary === null) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
316
|
-
const report = value;
|
|
317
|
-
const caseIds = /* @__PURE__ */ new Set();
|
|
318
|
-
for (const testCase of report.cases) {
|
|
319
|
-
if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
320
|
-
caseIds.add(testCase.caseId);
|
|
321
|
-
for (const score of testCase.scores) if (typeof score !== "object" || score === null || typeof score.scorerId !== "string" || !report.scorerIds.includes(score.scorerId) || ![
|
|
322
|
-
"passed",
|
|
323
|
-
"failed",
|
|
324
|
-
"skipped",
|
|
325
|
-
"error"
|
|
326
|
-
].includes(score.status) || score.score !== void 0 && (!Number.isFinite(score.score) || score.score < 0 || score.score > 1)) throw new TypeError("Invalid fevex.eval.v1 report");
|
|
327
|
-
}
|
|
328
|
-
return value;
|
|
329
|
-
}
|
|
330
|
-
function jsonReporter(write) {
|
|
331
|
-
return { report: (report) => write(serializeEvalReport(report)) };
|
|
332
|
-
}
|
|
333
|
-
function textReporter(write) {
|
|
334
|
-
return { report(report) {
|
|
335
|
-
return write(`${[
|
|
336
|
-
`${report.suiteName} (${report.targetVersion})`,
|
|
337
|
-
`dataset ${report.dataset.name}@${report.dataset.version}`,
|
|
338
|
-
...report.cases.map((testCase) => {
|
|
339
|
-
const passed = testCase.scores.filter(({ status }) => status === "passed").length;
|
|
340
|
-
const failed = testCase.scores.filter(({ status }) => status === "failed" || status === "error").length;
|
|
341
|
-
return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed`;
|
|
342
|
-
}),
|
|
343
|
-
`summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores`
|
|
344
|
-
].join("\n")}\n`);
|
|
345
|
-
} };
|
|
346
|
-
}
|
|
347
|
-
function assertRatio(value, name) {
|
|
348
|
-
if (value !== void 0 && (!Number.isFinite(value) || value < 0)) throw new TypeError(`${name} must be non-negative`);
|
|
349
|
-
}
|
|
350
|
-
function compareEvaluationReports(baseline, current, options = {}) {
|
|
351
|
-
if (baseline.suiteName !== current.suiteName || baseline.dataset.name !== current.dataset.name || baseline.dataset.version !== current.dataset.version) throw new TypeError("Evaluation reports must use the same suite and dataset version");
|
|
352
|
-
assertRatio(options.maxScoreDrop, "maxScoreDrop");
|
|
353
|
-
assertRatio(options.maxLatencyIncreaseRatio, "maxLatencyIncreaseRatio");
|
|
354
|
-
assertRatio(options.maxCostIncreaseRatio, "maxCostIncreaseRatio");
|
|
355
|
-
const maxScoreDrop = options.maxScoreDrop ?? 0;
|
|
356
|
-
const regressions = [];
|
|
357
|
-
const currentCases = new Map(current.cases.map((testCase) => [testCase.caseId, testCase]));
|
|
358
|
-
for (const baselineCase of baseline.cases) {
|
|
359
|
-
const currentCase = currentCases.get(baselineCase.caseId);
|
|
360
|
-
if (!currentCase) {
|
|
361
|
-
regressions.push({
|
|
362
|
-
kind: "case_removed",
|
|
363
|
-
caseId: baselineCase.caseId,
|
|
364
|
-
message: `Case "${baselineCase.caseId}" was removed`
|
|
365
|
-
});
|
|
366
|
-
continue;
|
|
367
|
-
}
|
|
368
|
-
if (baselineCase.status === "completed" && currentCase.status === "failed") regressions.push({
|
|
369
|
-
kind: "case_failed",
|
|
370
|
-
caseId: baselineCase.caseId,
|
|
371
|
-
message: `Case "${baselineCase.caseId}" no longer completes`
|
|
372
|
-
});
|
|
373
|
-
const currentScores = new Map(currentCase.scores.map((score) => [score.scorerId, score]));
|
|
374
|
-
for (const baselineScore of baselineCase.scores) {
|
|
375
|
-
const currentScore = currentScores.get(baselineScore.scorerId);
|
|
376
|
-
if (!currentScore) {
|
|
377
|
-
regressions.push({
|
|
378
|
-
kind: "scorer_removed",
|
|
379
|
-
caseId: baselineCase.caseId,
|
|
380
|
-
scorerId: baselineScore.scorerId,
|
|
381
|
-
message: `Scorer "${baselineScore.scorerId}" was removed from "${baselineCase.caseId}"`
|
|
382
|
-
});
|
|
383
|
-
continue;
|
|
384
|
-
}
|
|
385
|
-
if (baselineScore.status === "passed" && currentScore.status !== "passed") regressions.push({
|
|
386
|
-
kind: "score_failed",
|
|
387
|
-
caseId: baselineCase.caseId,
|
|
388
|
-
scorerId: baselineScore.scorerId,
|
|
389
|
-
message: `Scorer "${baselineScore.scorerId}" changed from pass to fail`
|
|
390
|
-
});
|
|
391
|
-
else if (baselineScore.score !== void 0 && currentScore.score !== void 0 && baselineScore.score - currentScore.score > maxScoreDrop) regressions.push({
|
|
392
|
-
kind: "score_dropped",
|
|
393
|
-
caseId: baselineCase.caseId,
|
|
394
|
-
scorerId: baselineScore.scorerId,
|
|
395
|
-
message: `Scorer "${baselineScore.scorerId}" dropped from ${baselineScore.score} to ${currentScore.score}`
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
if (options.maxLatencyIncreaseRatio !== void 0 && baselineCase.latencyMs > 0 && currentCase.latencyMs / baselineCase.latencyMs - 1 > options.maxLatencyIncreaseRatio) regressions.push({
|
|
399
|
-
kind: "latency_increased",
|
|
400
|
-
caseId: baselineCase.caseId,
|
|
401
|
-
message: `Latency increased from ${baselineCase.latencyMs}ms to ${currentCase.latencyMs}ms`
|
|
402
|
-
});
|
|
403
|
-
if (options.maxCostIncreaseRatio !== void 0 && baselineCase.cost && currentCase.cost) {
|
|
404
|
-
if (baselineCase.cost.currency !== currentCase.cost.currency || baselineCase.cost.amount > 0 && currentCase.cost.amount / baselineCase.cost.amount - 1 > options.maxCostIncreaseRatio) regressions.push({
|
|
405
|
-
kind: "cost_increased",
|
|
406
|
-
caseId: baselineCase.caseId,
|
|
407
|
-
message: `Cost changed from ${baselineCase.cost.amount} ${baselineCase.cost.currency} to ${currentCase.cost.amount} ${currentCase.cost.currency}`
|
|
408
|
-
});
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
return {
|
|
412
|
-
passed: regressions.length === 0,
|
|
413
|
-
baselineTargetVersion: baseline.targetVersion,
|
|
414
|
-
currentTargetVersion: current.targetVersion,
|
|
415
|
-
regressions
|
|
416
|
-
};
|
|
417
|
-
}
|
|
418
421
|
//#endregion
|
|
419
422
|
export { compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { f as RunId, t as AgentEvent } from "./types-CuoRx12Q.mjs";
|
|
2
|
+
import { t as ModelGateway } from "./index-CTIwrVf9.mjs";
|
|
3
|
+
import { A as RunStore, D as RunRequest, E as RunRecord, F as TeamRun, M as SessionId, O as RunResult, S as ResumeRunResolution, U as WorkflowRun, a as AgentRun, b as ListEventsOptions, f as CredentialStore, j as Session, t as AgentDefinition } from "./index-Dx_LotWZ.mjs";
|
|
4
|
+
import { r as ContextProvider, s as MemoryStore } from "./index-DeXUtyd9.mjs";
|
|
5
|
+
import { r as ObservabilityOptions } from "./index-B4PIkqOU.mjs";
|
|
6
|
+
import { r as Sandbox } from "./index-CrOId2px.mjs";
|
|
7
|
+
import { g as ToolDefinition, s as ConnectionDefinition } from "./index-CqOjvtQ9.mjs";
|
|
8
|
+
import { r as PolicyDefinition } from "./index-dkFyTNh4.mjs";
|
|
9
|
+
import { n as WorkflowDefinition } from "./index-DviHtsLo.mjs";
|
|
10
|
+
import { t as TeamDefinition } from "./index-Bg9rr8Qe.mjs";
|
|
11
|
+
//#region src/fevex.d.ts
|
|
12
|
+
/** Provider-neutral definitions and adapters used to create one Fevex runtime. */
|
|
13
|
+
interface FevexConfig {
|
|
14
|
+
models: Record<string, ModelGateway>;
|
|
15
|
+
agents: AgentDefinition[];
|
|
16
|
+
workflows?: WorkflowDefinition[];
|
|
17
|
+
teams?: TeamDefinition[];
|
|
18
|
+
tools?: ToolDefinition[];
|
|
19
|
+
connections?: ConnectionDefinition[];
|
|
20
|
+
contextProviders?: ContextProvider[];
|
|
21
|
+
memoryStore?: MemoryStore;
|
|
22
|
+
runStore?: RunStore;
|
|
23
|
+
credentialStore?: CredentialStore;
|
|
24
|
+
sandbox?: Sandbox;
|
|
25
|
+
policies?: PolicyDefinition[];
|
|
26
|
+
onEvent?: (event: AgentEvent) => void;
|
|
27
|
+
observability?: ObservabilityOptions;
|
|
28
|
+
}
|
|
29
|
+
interface Fevex {
|
|
30
|
+
startAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<AgentRun<TOutput>>;
|
|
31
|
+
runAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
|
|
32
|
+
streamAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): AsyncIterable<AgentEvent>;
|
|
33
|
+
startWorkflow<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<WorkflowRun<TOutput>>;
|
|
34
|
+
runWorkflow<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
|
|
35
|
+
startTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<TeamRun<TOutput>>;
|
|
36
|
+
runTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
|
|
37
|
+
streamTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): AsyncIterable<AgentEvent>;
|
|
38
|
+
getTeamRun<TOutput = unknown>(runId: RunId): Promise<TeamRun<TOutput> | undefined>;
|
|
39
|
+
getRun<TOutput = unknown>(runId: RunId): Promise<RunRecord<TOutput> | undefined>;
|
|
40
|
+
listEvents(runId: RunId, options?: ListEventsOptions): Promise<AgentEvent[]>;
|
|
41
|
+
cancelRun(runId: RunId): Promise<boolean>;
|
|
42
|
+
resumeRun<TOutput = unknown>(runId: RunId, resolution: ResumeRunResolution): Promise<RunRecord<TOutput>>;
|
|
43
|
+
recoverRun<TOutput = unknown>(runId: RunId, options: {
|
|
44
|
+
actor: {
|
|
45
|
+
id: string;
|
|
46
|
+
type?: string;
|
|
47
|
+
};
|
|
48
|
+
}): Promise<RunRecord<TOutput>>;
|
|
49
|
+
compactSession(sessionId: SessionId, summary: string): Promise<Session>;
|
|
50
|
+
flushObservability(): Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Validates a configuration snapshot and creates an isolated runtime.
|
|
54
|
+
*
|
|
55
|
+
* The runtime does not own external adapters; callers remain responsible for
|
|
56
|
+
* closing model, connection, store, and exporter resources they provide.
|
|
57
|
+
*/
|
|
58
|
+
declare function createFevex(config: FevexConfig): Fevex;
|
|
59
|
+
//#endregion
|
|
60
|
+
export { FevexConfig as n, createFevex as r, Fevex as t };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CuoRx12Q.mjs";
|
|
2
|
+
import { c as ReasoningEffort } from "../index-CTIwrVf9.mjs";
|
|
3
|
+
import { E as RunRecord, F as TeamRun, M as SessionId, U as WorkflowRun, a as AgentRun } from "../index-Dx_LotWZ.mjs";
|
|
4
|
+
import { t as Fevex } from "../fevex-Ddyi6z6L.mjs";
|
|
5
|
+
//#region src/http/protocol.d.ts
|
|
6
|
+
declare const FEVEX_HTTP_PROTOCOL_VERSION = "3";
|
|
7
|
+
declare const FEVEX_HTTP_PROTOCOL_VERSION_HEADER = "Fevex-Protocol-Version";
|
|
8
|
+
interface StartRunHttpRequest<TInput extends JsonValue = JsonValue> {
|
|
9
|
+
input: TInput;
|
|
10
|
+
model?: string;
|
|
11
|
+
reasoning?: ReasoningEffort;
|
|
12
|
+
sessionId?: SessionId;
|
|
13
|
+
}
|
|
14
|
+
type ResumeRunHttpRequest = {
|
|
15
|
+
type: 'elicitation';
|
|
16
|
+
requestId: string;
|
|
17
|
+
value: JsonValue;
|
|
18
|
+
} | {
|
|
19
|
+
type: 'approval';
|
|
20
|
+
approvalId: string;
|
|
21
|
+
decision: 'approve' | 'reject';
|
|
22
|
+
} | {
|
|
23
|
+
type: 'tool_execution';
|
|
24
|
+
toolCallId: string;
|
|
25
|
+
decision: 'use_output';
|
|
26
|
+
output: JsonValue;
|
|
27
|
+
} | {
|
|
28
|
+
type: 'tool_execution';
|
|
29
|
+
toolCallId: string;
|
|
30
|
+
decision: 'retry';
|
|
31
|
+
} | {
|
|
32
|
+
type: 'timer';
|
|
33
|
+
} | {
|
|
34
|
+
type: 'event';
|
|
35
|
+
eventName: string;
|
|
36
|
+
payload?: JsonValue;
|
|
37
|
+
};
|
|
38
|
+
/** RFC 9457-compatible problem returned by the Fevex HTTP protocol. */
|
|
39
|
+
interface FevexProblemDetails {
|
|
40
|
+
type: string;
|
|
41
|
+
title: string;
|
|
42
|
+
status: number;
|
|
43
|
+
code: string;
|
|
44
|
+
detail?: string;
|
|
45
|
+
instance?: string;
|
|
46
|
+
}
|
|
47
|
+
interface FevexHttpHandlerContext {
|
|
48
|
+
context?: ExecutionContext;
|
|
49
|
+
}
|
|
50
|
+
interface FevexHttpHandlerOptions {
|
|
51
|
+
fevex: Fevex;
|
|
52
|
+
pollIntervalMs?: number;
|
|
53
|
+
/** Events per SSE read (default 100, maximum 1000). */
|
|
54
|
+
eventPageSize?: number;
|
|
55
|
+
onError?: (error: unknown, problem: FevexProblemDetails) => void;
|
|
56
|
+
}
|
|
57
|
+
type FevexHttpHandler = (request: Request, context?: FevexHttpHandlerContext) => Promise<Response>;
|
|
58
|
+
interface FevexHttpClientOptions {
|
|
59
|
+
baseUrl: string;
|
|
60
|
+
fetch?: FevexHttpFetch;
|
|
61
|
+
}
|
|
62
|
+
type FevexHttpFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
63
|
+
interface ObserveRunOptions {
|
|
64
|
+
after?: AgentEvent['id'];
|
|
65
|
+
signal?: AbortSignal;
|
|
66
|
+
}
|
|
67
|
+
interface FevexHttpClient {
|
|
68
|
+
startRun<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(agentName: string, request: StartRunHttpRequest<TInput>): Promise<AgentRun<TOutput>>;
|
|
69
|
+
startWorkflow<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(workflowName: string, request: StartRunHttpRequest<TInput>): Promise<WorkflowRun<TOutput>>;
|
|
70
|
+
startTeam<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(teamName: string, request: StartRunHttpRequest<TInput>): Promise<TeamRun<TOutput>>;
|
|
71
|
+
getRun<TOutput = JsonValue>(runId: string): Promise<RunRecord<TOutput>>;
|
|
72
|
+
observeRun(runId: string, options?: ObserveRunOptions): AsyncIterable<AgentEvent>;
|
|
73
|
+
resumeRun<TOutput = JsonValue>(runId: string, request: ResumeRunHttpRequest): Promise<RunRecord<TOutput>>;
|
|
74
|
+
recoverRun<TOutput = JsonValue>(runId: string): Promise<RunRecord<TOutput>>;
|
|
75
|
+
cancelRun(runId: string): Promise<void>;
|
|
76
|
+
}
|
|
77
|
+
//#endregion
|
|
78
|
+
//#region src/http/transport.d.ts
|
|
79
|
+
/** Exposes a Fevex instance through the versioned HTTP protocol. */
|
|
80
|
+
declare function createFevexHttpHandler(options: FevexHttpHandlerOptions): FevexHttpHandler;
|
|
81
|
+
declare class FevexHttpError extends Error {
|
|
82
|
+
readonly problem: FevexProblemDetails;
|
|
83
|
+
readonly name = "FevexHttpError";
|
|
84
|
+
constructor(problem: FevexProblemDetails);
|
|
85
|
+
get status(): number;
|
|
86
|
+
get code(): string;
|
|
87
|
+
}
|
|
88
|
+
declare function createFevexHttpClient(options: FevexHttpClientOptions): FevexHttpClient;
|
|
89
|
+
//#endregion
|
|
90
|
+
export { FEVEX_HTTP_PROTOCOL_VERSION, FEVEX_HTTP_PROTOCOL_VERSION_HEADER, FevexHttpClient, FevexHttpClientOptions, FevexHttpError, FevexHttpFetch, FevexHttpHandler, FevexHttpHandlerContext, FevexHttpHandlerOptions, FevexProblemDetails, ObserveRunOptions, ResumeRunHttpRequest, StartRunHttpRequest, createFevexHttpClient, createFevexHttpHandler };
|