@fevex/core 0.1.0-alpha.1 → 0.1.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +786 -13
  2. package/dist/agents/index.d.mts +2 -2
  3. package/dist/agents/index.mjs +1 -1
  4. package/dist/artifacts/index.d.mts +2 -0
  5. package/dist/artifacts/index.mjs +2 -0
  6. package/dist/artifacts-gUzemET7.mjs +440 -0
  7. package/dist/channels/index.d.mts +2 -0
  8. package/dist/channels/index.mjs +192 -0
  9. package/dist/evals/index.d.mts +19 -11
  10. package/dist/evals/index.mjs +125 -122
  11. package/dist/fevex-_KOsOsB8.d.mts +129 -0
  12. package/dist/http/index.d.mts +150 -0
  13. package/dist/http/index.mjs +692 -0
  14. package/dist/index-26ilbzAs.d.mts +50 -0
  15. package/dist/index-B8EcvWeN.d.mts +103 -0
  16. package/dist/index-BEqYWean.d.mts +118 -0
  17. package/dist/index-Btth7rdL.d.mts +58 -0
  18. package/dist/{index-QTkd6a7C.d.mts → index-CFzrvDQ0.d.mts} +4 -3
  19. package/dist/index-CIHWc5V1.d.mts +116 -0
  20. package/dist/index-D7r0puBi.d.mts +64 -0
  21. package/dist/index-DHcfIfh-.d.mts +224 -0
  22. package/dist/index-DUhNa7IR.d.mts +73 -0
  23. package/dist/index-lCCFokc6.d.mts +617 -0
  24. package/dist/index.d.mts +22 -10
  25. package/dist/index.mjs +4178 -671
  26. package/dist/knowledge/index.d.mts +2 -0
  27. package/dist/knowledge/index.mjs +289 -0
  28. package/dist/model-stream-JQdHDDcj.mjs +131 -0
  29. package/dist/models/index.d.mts +2 -46
  30. package/dist/observability/index.d.mts +2 -2
  31. package/dist/policies/index.d.mts +1 -1
  32. package/dist/policies/index.mjs +1 -1
  33. package/dist/run-error-Cr17wPGH.mjs +23 -0
  34. package/dist/{model-stream-u2xMsEaf.mjs → run-support-CmZ6Xo6H.mjs} +41 -130
  35. package/dist/runtime/index.d.mts +2 -2
  36. package/dist/runtime/index.mjs +271 -9
  37. package/dist/sandbox/index.d.mts +2 -0
  38. package/dist/sandbox/index.mjs +147 -0
  39. package/dist/teams/index.d.mts +2 -0
  40. package/dist/teams/index.mjs +2 -0
  41. package/dist/teams-Ctz676G4.mjs +82 -0
  42. package/dist/testing/index.d.mts +39 -9
  43. package/dist/testing/index.mjs +512 -28
  44. package/dist/tools/index.d.mts +3 -3
  45. package/dist/tools/index.mjs +442 -2
  46. package/dist/types-CckDlhYo.d.mts +575 -0
  47. package/dist/workflows/index.d.mts +2 -0
  48. package/dist/workflows/index.mjs +6 -0
  49. package/package.json +31 -3
  50. package/dist/exact-definition-NHBBFlFR.d.mts +0 -4
  51. package/dist/fevex-Dz6GBz9o.d.mts +0 -32
  52. package/dist/index-BzxHfyYV.d.mts +0 -23
  53. package/dist/index-CZGyhNIQ.d.mts +0 -47
  54. package/dist/index-C_Zpr13f.d.mts +0 -124
  55. package/dist/index-DGWOGXzc.d.mts +0 -181
  56. package/dist/index-ojCY3lwm.d.mts +0 -79
@@ -1,8 +1,12 @@
1
- import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../index-C_Zpr13f.mjs";
2
- import { ModelMetadata, ModelUsage } from "../models/index.mjs";
3
- import { l as TraceCost, t as CostCalculator } from "../index-ojCY3lwm.mjs";
4
- import { t as Fevex } from "../fevex-Dz6GBz9o.mjs";
5
- //#region src/evals/index.d.ts
1
+ import { r as ModelMetadata, s as ModelUsage } from "../index-D7r0puBi.mjs";
2
+ import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CckDlhYo.mjs";
3
+ import { t as CostCalculator, u as TraceCost } from "../index-BEqYWean.mjs";
4
+ import { t as Fevex } from "../fevex-_KOsOsB8.mjs";
5
+ //#region src/evals/canonical-json.d.ts
6
+ /** Serializes JSON-like data with stable object-key ordering. */
7
+ declare function serializeCanonical(value: unknown): string;
8
+ //#endregion
9
+ //#region src/evals/types.d.ts
6
10
  interface EvalExpectation {
7
11
  output?: JsonValue;
8
12
  tools?: string[];
@@ -121,7 +125,17 @@ interface RegressionOptions {
121
125
  maxLatencyIncreaseRatio?: number;
122
126
  maxCostIncreaseRatio?: number;
123
127
  }
128
+ //#endregion
129
+ //#region src/evals/evaluation.d.ts
130
+ /** Runs every case and scorer, producing a deterministic aggregate report. */
124
131
  declare function runEvaluation<TInput = unknown>(options: RunEvaluationOptions<TInput>): Promise<EvalReport>;
132
+ declare function serializeEvalReport(report: EvalReport): string;
133
+ declare function parseEvalReport(json: string): EvalReport;
134
+ declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
135
+ declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
136
+ declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
137
+ //#endregion
138
+ //#region src/evals/scorers.d.ts
125
139
  declare function exactOutputScorer(id?: string): EvalScorer;
126
140
  declare function toolSelectionScorer(options?: {
127
141
  id?: string;
@@ -135,11 +149,5 @@ declare function maxTokensScorer(limits: {
135
149
  total?: number;
136
150
  }, id?: string): EvalScorer;
137
151
  declare function maxCostScorer(maxAmount: number, currency: string, id?: string): EvalScorer;
138
- declare function serializeCanonical(value: unknown): string;
139
- declare function serializeEvalReport(report: EvalReport): string;
140
- declare function parseEvalReport(json: string): EvalReport;
141
- declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
142
- declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
143
- declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
144
152
  //#endregion
145
153
  export { EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalExpectation, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, Regression, RegressionOptions, RegressionReport, RunEvaluationOptions, compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
@@ -1,13 +1,21 @@
1
- //#region src/evals/index.ts
1
+ //#region src/evals/canonical-json.ts
2
+ function canonicalize(value) {
3
+ if (Array.isArray(value)) return value.map(canonicalize);
4
+ if (value === null || typeof value !== "object") return value;
5
+ return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
6
+ }
7
+ /** Serializes JSON-like data with stable object-key ordering. */
8
+ function serializeCanonical(value) {
9
+ return JSON.stringify(canonicalize(value), null, 2);
10
+ }
11
+ //#endregion
12
+ //#region src/evals/evaluation.ts
2
13
  function assertNonEmpty(value, name) {
3
14
  if (typeof value !== "string" || !value.trim()) throw new TypeError(`${name} must be non-empty`);
4
15
  }
5
16
  function errorMessage(error) {
6
17
  return error instanceof Error ? error.message : String(error);
7
18
  }
8
- function equalJson(left, right) {
9
- return serializeCanonical(left) === serializeCanonical(right);
10
- }
11
19
  function addUsage(target, usage) {
12
20
  if (!usage) return;
13
21
  for (const key of [
@@ -157,6 +165,7 @@ function summarize(cases) {
157
165
  latencyMs: cases.reduce((sum, testCase) => sum + testCase.latencyMs, 0)
158
166
  };
159
167
  }
168
+ /** Runs every case and scorer, producing a deterministic aggregate report. */
160
169
  async function runEvaluation(options) {
161
170
  validateOptions(options);
162
171
  const cases = [];
@@ -177,6 +186,116 @@ async function runEvaluation(options) {
177
186
  for (const reporter of options.reporters ?? []) await reporter.report(report);
178
187
  return report;
179
188
  }
189
+ function serializeEvalReport(report) {
190
+ return `${serializeCanonical(report)}\n`;
191
+ }
192
+ function parseEvalReport(json) {
193
+ const value = JSON.parse(json);
194
+ if (typeof value !== "object" || value === null || value.schemaVersion !== "fevex.eval.v1" || typeof value.suiteName !== "string" || typeof value.targetVersion !== "string" || typeof value.agentName !== "string" || typeof value.dataset !== "object" || value.dataset === null || typeof value.dataset.name !== "string" || typeof value.dataset.version !== "string" || !Array.isArray(value.cases) || !Array.isArray(value.scorerIds) || !value.scorerIds.every((scorerId) => typeof scorerId === "string" && Boolean(scorerId)) || typeof value.summary !== "object" || value.summary === null) throw new TypeError("Invalid fevex.eval.v1 report");
195
+ const report = value;
196
+ const caseIds = /* @__PURE__ */ new Set();
197
+ for (const testCase of report.cases) {
198
+ if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
199
+ caseIds.add(testCase.caseId);
200
+ for (const score of testCase.scores) if (typeof score !== "object" || score === null || typeof score.scorerId !== "string" || !report.scorerIds.includes(score.scorerId) || ![
201
+ "passed",
202
+ "failed",
203
+ "skipped",
204
+ "error"
205
+ ].includes(score.status) || score.score !== void 0 && (!Number.isFinite(score.score) || score.score < 0 || score.score > 1)) throw new TypeError("Invalid fevex.eval.v1 report");
206
+ }
207
+ return value;
208
+ }
209
+ function jsonReporter(write) {
210
+ return { report: (report) => write(serializeEvalReport(report)) };
211
+ }
212
+ function textReporter(write) {
213
+ return { report(report) {
214
+ return write(`${[
215
+ `${report.suiteName} (${report.targetVersion})`,
216
+ `dataset ${report.dataset.name}@${report.dataset.version}`,
217
+ ...report.cases.map((testCase) => {
218
+ const passed = testCase.scores.filter(({ status }) => status === "passed").length;
219
+ const failed = testCase.scores.filter(({ status }) => status === "failed" || status === "error").length;
220
+ return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed`;
221
+ }),
222
+ `summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores`
223
+ ].join("\n")}\n`);
224
+ } };
225
+ }
226
+ function assertRatio(value, name) {
227
+ if (value !== void 0 && (!Number.isFinite(value) || value < 0)) throw new TypeError(`${name} must be non-negative`);
228
+ }
229
+ function compareEvaluationReports(baseline, current, options = {}) {
230
+ if (baseline.suiteName !== current.suiteName || baseline.dataset.name !== current.dataset.name || baseline.dataset.version !== current.dataset.version) throw new TypeError("Evaluation reports must use the same suite and dataset version");
231
+ assertRatio(options.maxScoreDrop, "maxScoreDrop");
232
+ assertRatio(options.maxLatencyIncreaseRatio, "maxLatencyIncreaseRatio");
233
+ assertRatio(options.maxCostIncreaseRatio, "maxCostIncreaseRatio");
234
+ const maxScoreDrop = options.maxScoreDrop ?? 0;
235
+ const regressions = [];
236
+ const currentCases = new Map(current.cases.map((testCase) => [testCase.caseId, testCase]));
237
+ for (const baselineCase of baseline.cases) {
238
+ const currentCase = currentCases.get(baselineCase.caseId);
239
+ if (!currentCase) {
240
+ regressions.push({
241
+ kind: "case_removed",
242
+ caseId: baselineCase.caseId,
243
+ message: `Case "${baselineCase.caseId}" was removed`
244
+ });
245
+ continue;
246
+ }
247
+ if (baselineCase.status === "completed" && currentCase.status === "failed") regressions.push({
248
+ kind: "case_failed",
249
+ caseId: baselineCase.caseId,
250
+ message: `Case "${baselineCase.caseId}" no longer completes`
251
+ });
252
+ const currentScores = new Map(currentCase.scores.map((score) => [score.scorerId, score]));
253
+ for (const baselineScore of baselineCase.scores) {
254
+ const currentScore = currentScores.get(baselineScore.scorerId);
255
+ if (!currentScore) {
256
+ regressions.push({
257
+ kind: "scorer_removed",
258
+ caseId: baselineCase.caseId,
259
+ scorerId: baselineScore.scorerId,
260
+ message: `Scorer "${baselineScore.scorerId}" was removed from "${baselineCase.caseId}"`
261
+ });
262
+ continue;
263
+ }
264
+ if (baselineScore.status === "passed" && currentScore.status !== "passed") regressions.push({
265
+ kind: "score_failed",
266
+ caseId: baselineCase.caseId,
267
+ scorerId: baselineScore.scorerId,
268
+ message: `Scorer "${baselineScore.scorerId}" changed from pass to fail`
269
+ });
270
+ else if (baselineScore.score !== void 0 && currentScore.score !== void 0 && baselineScore.score - currentScore.score > maxScoreDrop) regressions.push({
271
+ kind: "score_dropped",
272
+ caseId: baselineCase.caseId,
273
+ scorerId: baselineScore.scorerId,
274
+ message: `Scorer "${baselineScore.scorerId}" dropped from ${baselineScore.score} to ${currentScore.score}`
275
+ });
276
+ }
277
+ if (options.maxLatencyIncreaseRatio !== void 0 && baselineCase.latencyMs > 0 && currentCase.latencyMs / baselineCase.latencyMs - 1 > options.maxLatencyIncreaseRatio) regressions.push({
278
+ kind: "latency_increased",
279
+ caseId: baselineCase.caseId,
280
+ message: `Latency increased from ${baselineCase.latencyMs}ms to ${currentCase.latencyMs}ms`
281
+ });
282
+ if (options.maxCostIncreaseRatio !== void 0 && baselineCase.cost && currentCase.cost) {
283
+ if (baselineCase.cost.currency !== currentCase.cost.currency || baselineCase.cost.amount > 0 && currentCase.cost.amount / baselineCase.cost.amount - 1 > options.maxCostIncreaseRatio) regressions.push({
284
+ kind: "cost_increased",
285
+ caseId: baselineCase.caseId,
286
+ message: `Cost changed from ${baselineCase.cost.amount} ${baselineCase.cost.currency} to ${currentCase.cost.amount} ${currentCase.cost.currency}`
287
+ });
288
+ }
289
+ }
290
+ return {
291
+ passed: regressions.length === 0,
292
+ baselineTargetVersion: baseline.targetVersion,
293
+ currentTargetVersion: current.targetVersion,
294
+ regressions
295
+ };
296
+ }
297
+ //#endregion
298
+ //#region src/evals/scorers.ts
180
299
  function exactOutputScorer(id = "output.exact") {
181
300
  return {
182
301
  id,
@@ -185,7 +304,7 @@ function exactOutputScorer(id = "output.exact") {
185
304
  skipped: true,
186
305
  reason: "No expected output"
187
306
  };
188
- const passed = equalJson(output, testCase.expected.output);
307
+ const passed = serializeCanonical(output) === serializeCanonical(testCase.expected.output);
189
308
  return {
190
309
  passed,
191
310
  score: passed ? 1 : 0
@@ -272,7 +391,7 @@ function maxTokensScorer(limits, id = "tokens.max") {
272
391
  }
273
392
  function maxCostScorer(maxAmount, currency, id = "cost.max") {
274
393
  if (!Number.isFinite(maxAmount) || maxAmount < 0) throw new TypeError("maxAmount must be non-negative");
275
- assertNonEmpty(currency, "currency");
394
+ if (!currency.trim()) throw new TypeError("currency must be non-empty");
276
395
  return {
277
396
  id,
278
397
  score({ cost }) {
@@ -299,121 +418,5 @@ function maxCostScorer(maxAmount, currency, id = "cost.max") {
299
418
  }
300
419
  };
301
420
  }
302
- function canonicalize(value) {
303
- if (Array.isArray(value)) return value.map(canonicalize);
304
- if (value === null || typeof value !== "object") return value;
305
- return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
306
- }
307
- function serializeCanonical(value) {
308
- return JSON.stringify(canonicalize(value), null, 2);
309
- }
310
- function serializeEvalReport(report) {
311
- return `${serializeCanonical(report)}\n`;
312
- }
313
- function parseEvalReport(json) {
314
- const value = JSON.parse(json);
315
- if (typeof value !== "object" || value === null || value.schemaVersion !== "fevex.eval.v1" || typeof value.suiteName !== "string" || typeof value.targetVersion !== "string" || typeof value.agentName !== "string" || typeof value.dataset !== "object" || value.dataset === null || typeof value.dataset.name !== "string" || typeof value.dataset.version !== "string" || !Array.isArray(value.cases) || !Array.isArray(value.scorerIds) || !value.scorerIds.every((scorerId) => typeof scorerId === "string" && Boolean(scorerId)) || typeof value.summary !== "object" || value.summary === null) throw new TypeError("Invalid fevex.eval.v1 report");
316
- const report = value;
317
- const caseIds = /* @__PURE__ */ new Set();
318
- for (const testCase of report.cases) {
319
- if (typeof testCase !== "object" || testCase === null || typeof testCase.caseId !== "string" || !testCase.caseId || caseIds.has(testCase.caseId) || !["completed", "failed"].includes(testCase.status) || !Array.isArray(testCase.tools) || !testCase.tools.every((tool) => typeof tool === "string") || !Number.isFinite(testCase.latencyMs) || testCase.latencyMs < 0 || !Array.isArray(testCase.scores)) throw new TypeError("Invalid fevex.eval.v1 report");
320
- caseIds.add(testCase.caseId);
321
- for (const score of testCase.scores) if (typeof score !== "object" || score === null || typeof score.scorerId !== "string" || !report.scorerIds.includes(score.scorerId) || ![
322
- "passed",
323
- "failed",
324
- "skipped",
325
- "error"
326
- ].includes(score.status) || score.score !== void 0 && (!Number.isFinite(score.score) || score.score < 0 || score.score > 1)) throw new TypeError("Invalid fevex.eval.v1 report");
327
- }
328
- return value;
329
- }
330
- function jsonReporter(write) {
331
- return { report: (report) => write(serializeEvalReport(report)) };
332
- }
333
- function textReporter(write) {
334
- return { report(report) {
335
- return write(`${[
336
- `${report.suiteName} (${report.targetVersion})`,
337
- `dataset ${report.dataset.name}@${report.dataset.version}`,
338
- ...report.cases.map((testCase) => {
339
- const passed = testCase.scores.filter(({ status }) => status === "passed").length;
340
- const failed = testCase.scores.filter(({ status }) => status === "failed" || status === "error").length;
341
- return `${testCase.caseId}: ${testCase.status}; ${passed} passed, ${failed} failed`;
342
- }),
343
- `summary: ${report.summary.completedCount}/${report.summary.caseCount} completed; ${report.summary.failedScoreCount + report.summary.errorScoreCount} failed scores`
344
- ].join("\n")}\n`);
345
- } };
346
- }
347
- function assertRatio(value, name) {
348
- if (value !== void 0 && (!Number.isFinite(value) || value < 0)) throw new TypeError(`${name} must be non-negative`);
349
- }
350
- function compareEvaluationReports(baseline, current, options = {}) {
351
- if (baseline.suiteName !== current.suiteName || baseline.dataset.name !== current.dataset.name || baseline.dataset.version !== current.dataset.version) throw new TypeError("Evaluation reports must use the same suite and dataset version");
352
- assertRatio(options.maxScoreDrop, "maxScoreDrop");
353
- assertRatio(options.maxLatencyIncreaseRatio, "maxLatencyIncreaseRatio");
354
- assertRatio(options.maxCostIncreaseRatio, "maxCostIncreaseRatio");
355
- const maxScoreDrop = options.maxScoreDrop ?? 0;
356
- const regressions = [];
357
- const currentCases = new Map(current.cases.map((testCase) => [testCase.caseId, testCase]));
358
- for (const baselineCase of baseline.cases) {
359
- const currentCase = currentCases.get(baselineCase.caseId);
360
- if (!currentCase) {
361
- regressions.push({
362
- kind: "case_removed",
363
- caseId: baselineCase.caseId,
364
- message: `Case "${baselineCase.caseId}" was removed`
365
- });
366
- continue;
367
- }
368
- if (baselineCase.status === "completed" && currentCase.status === "failed") regressions.push({
369
- kind: "case_failed",
370
- caseId: baselineCase.caseId,
371
- message: `Case "${baselineCase.caseId}" no longer completes`
372
- });
373
- const currentScores = new Map(currentCase.scores.map((score) => [score.scorerId, score]));
374
- for (const baselineScore of baselineCase.scores) {
375
- const currentScore = currentScores.get(baselineScore.scorerId);
376
- if (!currentScore) {
377
- regressions.push({
378
- kind: "scorer_removed",
379
- caseId: baselineCase.caseId,
380
- scorerId: baselineScore.scorerId,
381
- message: `Scorer "${baselineScore.scorerId}" was removed from "${baselineCase.caseId}"`
382
- });
383
- continue;
384
- }
385
- if (baselineScore.status === "passed" && currentScore.status !== "passed") regressions.push({
386
- kind: "score_failed",
387
- caseId: baselineCase.caseId,
388
- scorerId: baselineScore.scorerId,
389
- message: `Scorer "${baselineScore.scorerId}" changed from pass to fail`
390
- });
391
- else if (baselineScore.score !== void 0 && currentScore.score !== void 0 && baselineScore.score - currentScore.score > maxScoreDrop) regressions.push({
392
- kind: "score_dropped",
393
- caseId: baselineCase.caseId,
394
- scorerId: baselineScore.scorerId,
395
- message: `Scorer "${baselineScore.scorerId}" dropped from ${baselineScore.score} to ${currentScore.score}`
396
- });
397
- }
398
- if (options.maxLatencyIncreaseRatio !== void 0 && baselineCase.latencyMs > 0 && currentCase.latencyMs / baselineCase.latencyMs - 1 > options.maxLatencyIncreaseRatio) regressions.push({
399
- kind: "latency_increased",
400
- caseId: baselineCase.caseId,
401
- message: `Latency increased from ${baselineCase.latencyMs}ms to ${currentCase.latencyMs}ms`
402
- });
403
- if (options.maxCostIncreaseRatio !== void 0 && baselineCase.cost && currentCase.cost) {
404
- if (baselineCase.cost.currency !== currentCase.cost.currency || baselineCase.cost.amount > 0 && currentCase.cost.amount / baselineCase.cost.amount - 1 > options.maxCostIncreaseRatio) regressions.push({
405
- kind: "cost_increased",
406
- caseId: baselineCase.caseId,
407
- message: `Cost changed from ${baselineCase.cost.amount} ${baselineCase.cost.currency} to ${currentCase.cost.amount} ${currentCase.cost.currency}`
408
- });
409
- }
410
- }
411
- return {
412
- passed: regressions.length === 0,
413
- baselineTargetVersion: baseline.targetVersion,
414
- currentTargetVersion: current.targetVersion,
415
- regressions
416
- };
417
- }
418
421
  //#endregion
419
422
  export { compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
@@ -0,0 +1,129 @@
1
+ import { S as ConnectionDefinition, j as ToolDefinition, o as ToolEmbeddingCache } from "./index-DHcfIfh-.mjs";
2
+ import { t as ModelGateway } from "./index-D7r0puBi.mjs";
3
+ import { F as ArtifactStore, f as RunId, o as ExecutionContext, t as AgentEvent } from "./types-CckDlhYo.mjs";
4
+ import { r as Sandbox } from "./index-DUhNa7IR.mjs";
5
+ import { B as ResumeRunResolution, I as ListEventsOptions, J as RunRequest, O as CredentialStore, Q as Session, R as PurgeRunsOptions, Y as RunResult, Z as RunStore, a as ToolSearchOptions, d as ContextCompactionStats, dt as WorkflowRun, et as SessionId, q as RunRecord, r as ContextBudget, rt as TeamRun, t as AgentDefinition, u as ContextCompactionOptions, x as AgentRun, z as PurgeRunsResult } from "./index-lCCFokc6.mjs";
6
+ import { c as ContextProvider, f as MemoryStore } from "./index-CIHWc5V1.mjs";
7
+ import { i as ObservabilityStats, r as ObservabilityOptions } from "./index-BEqYWean.mjs";
8
+ import { r as PolicyDefinition } from "./index-CFzrvDQ0.mjs";
9
+ import { n as WorkflowDefinition } from "./index-Btth7rdL.mjs";
10
+ import { t as TeamDefinition } from "./index-26ilbzAs.mjs";
11
+ //#region src/fevex.d.ts
12
+ interface FevexAdmissionRequest {
13
+ kind: 'agent' | 'workflow' | 'team';
14
+ name: string;
15
+ request: RunRequest;
16
+ activeRuns: number;
17
+ activeTenantRuns: number;
18
+ }
19
+ /** Optional host-level admission controls applied before a top-level run starts. */
20
+ interface FevexAdmission {
21
+ /** Maximum concurrent top-level runs in this runtime. */
22
+ maxActiveRuns?: number;
23
+ /** Maximum concurrent top-level runs for one tenant. */
24
+ maxActiveRunsPerTenant?: number;
25
+ /** Defaults to a non-empty `context.attributes.tenantId` string. */
26
+ tenantKey?: (context?: ExecutionContext) => string | undefined;
27
+ /** Custom admission check, called after the built-in limits. */
28
+ authorize?: (request: FevexAdmissionRequest) => boolean | Promise<boolean>;
29
+ }
30
+ interface FevexShutdownOptions {
31
+ /**
32
+ * Maximum time to wait for active runs before cancelling them. Defaults to
33
+ * 30 seconds. Use `0` to cancel immediately. If the deadline expires,
34
+ * shutdown allows up to 5 additional seconds for forced cancellation and the
35
+ * final trace flush.
36
+ */
37
+ timeoutMs?: number;
38
+ }
39
+ interface FevexShutdownResult {
40
+ /** Runs that were still active when the grace period expired. */
41
+ cancelledRunIds: RunId[];
42
+ /** True when the grace deadline expired before all active runs finished. */
43
+ timedOut: boolean;
44
+ /** Error surfaced while flushing pending traces, if any. */
45
+ exportError?: unknown;
46
+ /** Observability counters after the shutdown flush. */
47
+ observability: ObservabilityStats;
48
+ }
49
+ interface FevexLimits {
50
+ /**
51
+ * Maximum non-control events persisted for one run. Control events such as
52
+ * pause, resume, approval and terminal transitions are always persisted.
53
+ */
54
+ maxEventsPerRun?: number;
55
+ }
56
+ /** Provider-neutral definitions and adapters used to create one Fevex runtime. */
57
+ interface FevexConfig {
58
+ models: Record<string, ModelGateway>;
59
+ agents: AgentDefinition[];
60
+ workflows?: WorkflowDefinition[];
61
+ teams?: TeamDefinition[];
62
+ tools?: ToolDefinition[];
63
+ connections?: ConnectionDefinition[];
64
+ contextProviders?: ContextProvider[];
65
+ /** Default context budget for agents that do not declare one. */
66
+ contextBudget?: ContextBudget;
67
+ /** Optional context compactor and token estimator. */
68
+ compaction?: ContextCompactionOptions;
69
+ /** Default on-demand tool search for agents that do not declare one. */
70
+ toolSearch?: ToolSearchOptions;
71
+ /** Optional durable cache for semantic tool embeddings. */
72
+ toolSearchCache?: ToolEmbeddingCache;
73
+ memoryStore?: MemoryStore;
74
+ runStore?: RunStore;
75
+ credentialStore?: CredentialStore;
76
+ sandbox?: Sandbox;
77
+ /** Optional artifact store exposed to tools as `context.artifacts`. */
78
+ artifacts?: ArtifactStore;
79
+ policies?: PolicyDefinition[];
80
+ onEvent?: (event: AgentEvent) => void;
81
+ observability?: ObservabilityOptions;
82
+ admission?: FevexAdmission;
83
+ limits?: FevexLimits;
84
+ }
85
+ interface Fevex {
86
+ startAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<AgentRun<TOutput>>;
87
+ runAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
88
+ streamAgent<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): AsyncIterable<AgentEvent>;
89
+ startWorkflow<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<WorkflowRun<TOutput>>;
90
+ runWorkflow<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
91
+ startTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<TeamRun<TOutput>>;
92
+ runTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): Promise<RunResult<TOutput>>;
93
+ streamTeam<TInput = unknown, TOutput = unknown>(name: string, request: RunRequest<TInput, TOutput>): AsyncIterable<AgentEvent>;
94
+ getTeamRun<TOutput = unknown>(runId: RunId): Promise<TeamRun<TOutput> | undefined>;
95
+ getRun<TOutput = unknown>(runId: RunId): Promise<RunRecord<TOutput> | undefined>;
96
+ listEvents(runId: RunId, options?: ListEventsOptions): Promise<AgentEvent[]>;
97
+ cancelRun(runId: RunId): Promise<boolean>;
98
+ resumeRun<TOutput = unknown>(runId: RunId, resolution: ResumeRunResolution): Promise<RunRecord<TOutput>>;
99
+ recoverRun<TOutput = unknown>(runId: RunId, options: {
100
+ actor: {
101
+ id: string;
102
+ type?: string;
103
+ };
104
+ }): Promise<RunRecord<TOutput>>;
105
+ purgeRuns(options?: PurgeRunsOptions): Promise<PurgeRunsResult>;
106
+ compactSession(sessionId: SessionId, summary: string): Promise<Session>;
107
+ /** Waits for queued exports and rejects with failures accumulated since the previous flush. */
108
+ flushObservability(options?: {
109
+ timeoutMs?: number;
110
+ }): Promise<void>;
111
+ /** Cumulative trace export counters for this runtime. */
112
+ getObservabilityStats(): ObservabilityStats;
113
+ /** Cumulative context compaction counters for this runtime. */
114
+ getCompactionStats(): ContextCompactionStats;
115
+ /**
116
+ * Stops admitting new work, drains or cancels active runs, and flushes trace
117
+ * exports. The host still owns closing its stores and external adapters.
118
+ */
119
+ shutdown(options?: FevexShutdownOptions): Promise<FevexShutdownResult>;
120
+ }
121
+ /**
122
+ * Validates a configuration snapshot and creates an isolated runtime.
123
+ *
124
+ * The runtime does not own external adapters; callers remain responsible for
125
+ * closing model, connection, store, and exporter resources they provide.
126
+ */
127
+ declare function createFevex(config: FevexConfig): Fevex;
128
+ //#endregion
129
+ export { FevexLimits as a, createFevex as c, FevexConfig as i, FevexAdmission as n, FevexShutdownOptions as o, FevexAdmissionRequest as r, FevexShutdownResult as s, Fevex as t };
@@ -0,0 +1,150 @@
1
+ import { c as ReasoningEffort } from "../index-D7r0puBi.mjs";
2
+ import { D as ArtifactCatalog, F as ArtifactStore, f as RunId, k as ArtifactDescriptor, l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CckDlhYo.mjs";
3
+ import { dt as WorkflowRun, et as SessionId, q as RunRecord, rt as TeamRun, x as AgentRun } from "../index-lCCFokc6.mjs";
4
+ import { t as Fevex } from "../fevex-_KOsOsB8.mjs";
5
+ //#region src/http/protocol.d.ts
6
+ declare const FEVEX_HTTP_PROTOCOL_VERSION = "3";
7
+ declare const FEVEX_HTTP_PROTOCOL_VERSION_HEADER = "Fevex-Protocol-Version";
8
+ interface StartRunHttpRequest<TInput extends JsonValue = JsonValue> {
9
+ input: TInput;
10
+ model?: string;
11
+ reasoning?: ReasoningEffort;
12
+ sessionId?: SessionId;
13
+ /** Maximum wall-clock time for the run, including time spent paused. */
14
+ deadlineMs?: number;
15
+ }
16
+ type ResumeRunHttpRequest = {
17
+ type: 'elicitation';
18
+ requestId: string;
19
+ value: JsonValue;
20
+ } | {
21
+ type: 'approval';
22
+ approvalId: string;
23
+ decision: 'approve' | 'reject';
24
+ } | {
25
+ type: 'tool_execution';
26
+ toolCallId: string;
27
+ decision: 'use_output';
28
+ output: JsonValue;
29
+ } | {
30
+ type: 'tool_execution';
31
+ toolCallId: string;
32
+ decision: 'retry';
33
+ } | {
34
+ type: 'timer';
35
+ } | {
36
+ type: 'event';
37
+ eventName: string;
38
+ payload?: JsonValue;
39
+ };
40
+ /** RFC 9457-compatible problem returned by the Fevex HTTP protocol. */
41
+ interface FevexProblemDetails {
42
+ type: string;
43
+ title: string;
44
+ status: number;
45
+ code: string;
46
+ detail?: string;
47
+ instance?: string;
48
+ }
49
+ interface FevexHttpHandlerContext {
50
+ context?: ExecutionContext;
51
+ }
52
+ /** Host-facing resource access check. */
53
+ type FevexHttpAccess = {
54
+ action: 'start';
55
+ kind: 'agent' | 'workflow' | 'team';
56
+ name: string;
57
+ sessionId?: SessionId;
58
+ context?: ExecutionContext;
59
+ } | {
60
+ action: 'read' | 'observe' | 'cancel' | 'resume' | 'recover';
61
+ runId: RunId;
62
+ context?: ExecutionContext;
63
+ };
64
+ /** Host-facing artifact access check. */
65
+ type FevexArtifactAccess = {
66
+ action: 'read' | 'delete';
67
+ kind: 'artifact';
68
+ artifactId: string;
69
+ context?: ExecutionContext;
70
+ } | {
71
+ action: 'read';
72
+ kind: 'artifacts';
73
+ runId: RunId;
74
+ context?: ExecutionContext;
75
+ };
76
+ interface FevexHttpHandlerOptions {
77
+ fevex: Fevex;
78
+ pollIntervalMs?: number;
79
+ /** Events per SSE read (default 100, maximum 1000). */
80
+ eventPageSize?: number;
81
+ /** Maximum concurrent SSE streams (default unlimited). */
82
+ maxConnections?: number;
83
+ /**
84
+ * Authorizes a resource operation before the handler reads or mutates a run,
85
+ * or starts a run (new or existing session). Return false for a 403 response.
86
+ * Omit to preserve the host's existing trusted-network behavior.
87
+ */
88
+ authorize?: (access: FevexHttpAccess) => boolean | Promise<boolean>;
89
+ /**
90
+ * Byte store used by the artifact routes. When omitted, artifact routes
91
+ * return 501.
92
+ */
93
+ artifacts?: ArtifactStore;
94
+ /**
95
+ * Metadata catalog used to list artifacts for a run. When omitted, the
96
+ * run-artifact listing route returns 501.
97
+ */
98
+ artifactCatalog?: ArtifactCatalog;
99
+ /**
100
+ * Authorizes artifact reads, listings and deletions. Omit to preserve the
101
+ * host's existing trusted-network behavior.
102
+ */
103
+ authorizeArtifact?: (access: FevexArtifactAccess) => boolean | Promise<boolean>;
104
+ /**
105
+ * Called after a run starts so the host can persist run and session ownership.
106
+ * If it throws, the handler cancels the new run and returns 500. A process
107
+ * crash between start and this callback leaves the run unowned; hosts must
108
+ * treat unowned runs as inaccessible. Runs created outside the HTTP handler
109
+ * are not reported.
110
+ */
111
+ onRunStarted?: (run: RunRecord, context?: ExecutionContext) => void | Promise<void>;
112
+ onError?: (error: unknown, problem: FevexProblemDetails) => void;
113
+ }
114
+ type FevexHttpHandler = (request: Request, context?: FevexHttpHandlerContext) => Promise<Response>;
115
+ interface FevexHttpClientOptions {
116
+ baseUrl: string;
117
+ fetch?: FevexHttpFetch;
118
+ }
119
+ type FevexHttpFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
120
+ interface ObserveRunOptions {
121
+ after?: AgentEvent['id'];
122
+ signal?: AbortSignal;
123
+ }
124
+ interface FevexHttpClient {
125
+ startRun<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(agentName: string, request: StartRunHttpRequest<TInput>): Promise<AgentRun<TOutput>>;
126
+ startWorkflow<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(workflowName: string, request: StartRunHttpRequest<TInput>): Promise<WorkflowRun<TOutput>>;
127
+ startTeam<TInput extends JsonValue = JsonValue, TOutput = JsonValue>(teamName: string, request: StartRunHttpRequest<TInput>): Promise<TeamRun<TOutput>>;
128
+ getRun<TOutput = JsonValue>(runId: string): Promise<RunRecord<TOutput>>;
129
+ observeRun(runId: string, options?: ObserveRunOptions): AsyncIterable<AgentEvent>;
130
+ resumeRun<TOutput = JsonValue>(runId: string, request: ResumeRunHttpRequest): Promise<RunRecord<TOutput>>;
131
+ recoverRun<TOutput = JsonValue>(runId: string): Promise<RunRecord<TOutput>>;
132
+ cancelRun(runId: string): Promise<void>;
133
+ getArtifact(artifactId: string): Promise<Response>;
134
+ listArtifacts(runId: string): Promise<ArtifactDescriptor[]>;
135
+ deleteArtifact(artifactId: string): Promise<void>;
136
+ }
137
+ //#endregion
138
+ //#region src/http/transport.d.ts
139
+ /** Exposes a Fevex instance through the versioned HTTP protocol. */
140
+ declare function createFevexHttpHandler(options: FevexHttpHandlerOptions): FevexHttpHandler;
141
+ declare class FevexHttpError extends Error {
142
+ readonly problem: FevexProblemDetails;
143
+ readonly name = "FevexHttpError";
144
+ constructor(problem: FevexProblemDetails);
145
+ get status(): number;
146
+ get code(): string;
147
+ }
148
+ declare function createFevexHttpClient(options: FevexHttpClientOptions): FevexHttpClient;
149
+ //#endregion
150
+ export { FEVEX_HTTP_PROTOCOL_VERSION, FEVEX_HTTP_PROTOCOL_VERSION_HEADER, FevexArtifactAccess, FevexHttpAccess, FevexHttpClient, FevexHttpClientOptions, FevexHttpError, FevexHttpFetch, FevexHttpHandler, FevexHttpHandlerContext, FevexHttpHandlerOptions, FevexProblemDetails, ObserveRunOptions, ResumeRunHttpRequest, StartRunHttpRequest, createFevexHttpClient, createFevexHttpHandler };