@fevex/core 0.1.0-alpha.11 → 0.1.0-alpha.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +89 -14
  2. package/dist/agents/index.d.mts +1 -1
  3. package/dist/artifacts/index.d.mts +1 -1
  4. package/dist/channels/index.d.mts +1 -1
  5. package/dist/evals/index.d.mts +198 -5
  6. package/dist/evals/index.mjs +478 -12
  7. package/dist/{fevex-_KOsOsB8.d.mts → fevex-B4eNkGTa.d.mts} +19 -10
  8. package/dist/http/index.d.mts +4 -4
  9. package/dist/{index-DHcfIfh-.d.mts → index-AOouohIx.d.mts} +3 -3
  10. package/dist/{index-B8EcvWeN.d.mts → index-AVlWktA8.d.mts} +3 -3
  11. package/dist/{index-BEqYWean.d.mts → index-B7fxPgn_.d.mts} +5 -5
  12. package/dist/{index-Btth7rdL.d.mts → index-CAVMQT9j.d.mts} +3 -3
  13. package/dist/{index-D7r0puBi.d.mts → index-Cbe1ZiyN.d.mts} +1 -1
  14. package/dist/{index-CIHWc5V1.d.mts → index-D1LAhzgC.d.mts} +2 -2
  15. package/dist/{index-26ilbzAs.d.mts → index-DMmrhb8l.d.mts} +4 -4
  16. package/dist/index-KKq-_SiK.d.mts +43 -0
  17. package/dist/{index-lCCFokc6.d.mts → index-giFR5t0x.d.mts} +5 -3
  18. package/dist/{index-DUhNa7IR.d.mts → index-k-gBmfJP.d.mts} +1 -1
  19. package/dist/index.d.mts +13 -13
  20. package/dist/index.mjs +214 -31
  21. package/dist/knowledge/index.d.mts +1 -1
  22. package/dist/models/index.d.mts +1 -1
  23. package/dist/observability/index.d.mts +1 -1
  24. package/dist/policies/index.d.mts +2 -2
  25. package/dist/policies/index.mjs +39 -1
  26. package/dist/runtime/index.d.mts +1 -1
  27. package/dist/sandbox/index.d.mts +1 -1
  28. package/dist/teams/index.d.mts +1 -1
  29. package/dist/testing/index.d.mts +6 -6
  30. package/dist/tools/index.d.mts +2 -2
  31. package/dist/{types-CckDlhYo.d.mts → types-B6hD1o2I.d.mts} +19 -1
  32. package/dist/workflows/index.d.mts +1 -1
  33. package/package.json +1 -1
  34. package/dist/index-CFzrvDQ0.d.mts +0 -21
package/README.md CHANGED
@@ -447,9 +447,11 @@ model.started
447
447
  model.output.delta
448
448
  model.completed
449
449
  context.compacted
450
+ memory.completed
450
451
  tool.started
451
452
  tool.completed
452
453
  tool.failed
454
+ policy.evaluated
453
455
  run.completed
454
456
  run.failed
455
457
  run.cancelled
@@ -547,6 +549,7 @@ const started = await app.startAgent('support', {
547
549
  });
548
550
 
549
551
  const run = await app.getRun(started.id);
552
+ const finished = await app.waitForRun(started.id, { timeoutMs: 30_000 });
550
553
  const events = await app.listEvents(started.id);
551
554
  const laterEvents = await app.listEvents(started.id, {
552
555
  after: events.at(-1)?.id,
@@ -555,9 +558,11 @@ const laterEvents = await app.listEvents(started.id, {
555
558
  await app.cancelRun(started.id);
556
559
  ```
557
560
 
558
- `AgentRun`, `Session` and events are serializable snapshots. `listEvents` orders
559
- by `sequence`; `after` accepts an event ID from the same run as a stable cursor.
560
- Unknown cursors fail instead of silently replaying events.
561
+ `waitForRun` waits for the run to leave `running` (paused or terminal) without
562
+ polling, and accepts an optional `signal` and `timeoutMs`. `AgentRun`, `Session`
563
+ and events are serializable snapshots. `listEvents` orders by `sequence`; `after`
564
+ accepts an event ID from the same run as a stable cursor. Unknown cursors fail
565
+ instead of silently replaying events.
561
566
 
562
567
  `createFevex` uses `InMemoryRunStore` by default. Supply `runStore` to replace
563
568
  it. The default survives only while the process lives, and one run at a time
@@ -706,7 +711,23 @@ remain explicit and are intended for server deployments.
706
711
 
707
712
  Policies run before approval resolution, credentials and each effect or retry.
708
713
  Credential values are not stored or sent to the model, and tool outputs that
709
- contain a resolved value are rejected.
714
+ contain a resolved value are rejected. Every decision emits `policy.evaluated`
715
+ and appears in local traces and OpenTelemetry span events.
716
+
717
+ For a static baseline, `riskGuardrail()` denies `destructive` tools and requires
718
+ approval for `sensitive` tools. Tools without an explicit `risk` default to
719
+ `read`; set `undeclaredRisk: 'require_approval'` or `'deny'` to fail closed:
720
+
721
+ ```ts
722
+ import { riskGuardrail } from '@fevex/core/policies';
723
+
724
+ createFevex({
725
+ models,
726
+ agents,
727
+ tools,
728
+ policies: [riskGuardrail({ undeclaredRisk: 'require_approval' })],
729
+ });
730
+ ```
710
731
 
711
732
  ## Limits
712
733
 
@@ -750,7 +771,7 @@ tenant. The default tenant key is a non-empty `context.attributes.tenantId`;
750
771
  one runtime process.
751
772
 
752
773
  `createFevex({ limits: { maxEventsPerRun } })` caps non-control events per run.
753
- Pause, resume, approval, elicitation, wait and terminal events are always
774
+ Pause, resume, approval, policy, elicitation, wait and terminal events are always
754
775
  persisted so a run can always reach a durable state; use retention for a total
755
776
  storage cap.
756
777
 
@@ -814,7 +835,11 @@ console.log(app.getObservabilityStats());
814
835
 
815
836
  Workflow and team traces carry `kind` and a span per step. Agent steps expose
816
837
  `childRunId`; child agent traces carry `parentRunId` and `rootRunId`, and tool
817
- spans are nested under the model call that requested them. A dropped trace or a
838
+ spans are nested under the model call that requested them. Memory search and
839
+ write operations produce `search_memory` and `create_memory` GenAI spans. Policy
840
+ decisions are trace annotations and OpenTelemetry span events, with
841
+ `fevex.tool.risk` on tool spans and `gen_ai.memory.*` attributes on memory
842
+ spans. A dropped trace or a
818
843
  failed export is counted in `getObservabilityStats()`, and the next
819
844
  `flushObservability()` rejects with the failures accumulated since the previous
820
845
  flush. OpenTelemetry SDK setup, OTLP export and shutdown remain application
@@ -869,11 +894,60 @@ const report = await runEvaluation({
869
894
  ```
870
895
 
871
896
  Built-in deterministic scorers cover exact output, ordered or unordered tool
872
- selection, forbidden tools, max latency, max tokens and max cost. Use
873
- `serializeEvalReport` for a canonical, versionable JSON baseline, `jsonReporter`
874
- or `textReporter` with your own write callback, and `compareEvaluationReports`
875
- to fail CI when cases disappear, passing scores fail or scores drop beyond the
876
- configured tolerance.
897
+ selection, forbidden tools, max latency, max tokens and max cost. Cases can also
898
+ resolve paused approvals and assert the full trajectory:
899
+
900
+ ```ts
901
+ import {
902
+ riskGuardrailScorer,
903
+ runEvaluation,
904
+ trajectoryExpectationScorer,
905
+ } from '@fevex/core/evals';
906
+
907
+ const report = await runEvaluation({
908
+ app,
909
+ suiteName: 'support-regression',
910
+ targetVersion: 'v2',
911
+ agentName: 'support',
912
+ dataset: {
913
+ name: 'support',
914
+ version: '1',
915
+ cases: [{
916
+ id: 'sensitive-write',
917
+ input: 'Update account 42',
918
+ approvals: [{ toolName: 'accounts_update', decision: 'approve' }],
919
+ expected: {
920
+ output: { answer: 'Account 42 updated.' },
921
+ tools: ['accounts_update'],
922
+ trajectory: [
923
+ { type: 'policy', toolName: 'accounts_update', decision: 'require_approval' },
924
+ { type: 'approval', toolName: 'accounts_update', decision: 'approve' },
925
+ { type: 'tool', name: 'accounts_update', status: 'completed' },
926
+ ],
927
+ },
928
+ }],
929
+ },
930
+ scorers: [
931
+ trajectoryExpectationScorer({ allowAdditional: true }),
932
+ riskGuardrailScorer(),
933
+ ],
934
+ });
935
+ ```
936
+
937
+ Trajectory entries cover tool attempts, approval decisions, elicitation
938
+ resolutions and non-allow policy decisions. Cases can also answer elicitation
939
+ pauses with `elicitations`, matched by prompt when provided; the runtime still
940
+ validates each value against its `responseSchema`. Each case has a 10-minute deadline by default; set `deadlineMs`
941
+ on the suite or case to change it. Use `serializeEvalReport` for a canonical,
942
+ versionable JSON baseline,
943
+ `jsonReporter` or `textReporter` with your own write callback, and
944
+ `compareEvaluationReports` to fail CI when cases disappear, passing scores fail
945
+ or scores drop beyond the configured tolerance. Cases warn when an `approvals`
946
+ or `elicitations` fixture was never used; the gate rejects warnings by default.
947
+ `evaluateReleaseGate` adds minimum completion, pass, score, warning, latency and
948
+ cost thresholds; `assertReleaseGate` throws `ReleaseGateError` with the failed
949
+ checks. The repository runs a deterministic release suite with
950
+ `bun run eval:gate` and the built package with `bun run eval:gate:package`.
877
951
 
878
952
  ## Connections
879
953
 
@@ -1204,9 +1278,9 @@ the artifact available.
1204
1278
  | `@fevex/core/teams` | Multiagent team definitions |
1205
1279
  | `@fevex/core/runtime` | Runs, sessions and store contracts |
1206
1280
  | `@fevex/core/sandbox` | Sandbox contract and local development sandbox |
1207
- | `@fevex/core/policies` | Authorization policy contracts |
1281
+ | `@fevex/core/policies` | Authorization policy contracts and the risk guardrail |
1208
1282
  | `@fevex/core/observability` | Trace, bounded export queue, redaction and cost contracts |
1209
- | `@fevex/core/evals` | Datasets, scorers, reporters and regressions |
1283
+ | `@fevex/core/evals` | Datasets, trajectory and policy scorers, release gates and regressions |
1210
1284
  | `@fevex/core/testing` | Deterministic testing helpers |
1211
1285
 
1212
1286
  All subpaths ship in the same `@fevex/core` package.
@@ -1237,7 +1311,8 @@ The MVP intentionally supports:
1237
1311
  - bounded multi-step model-tool loops with sequential tool execution;
1238
1312
  - native provider streaming and durable, consultable runs;
1239
1313
  - synchronous event observers and `AsyncIterable` event streaming;
1240
- - local traces, OpenTelemetry export and deterministic evaluation suites;
1314
+ - local traces, OpenTelemetry export and deterministic evaluation suites with
1315
+ trajectory, policy, approval, cost and release-gate assertions;
1241
1316
  - optional Standard Schema validation;
1242
1317
  - immediate `AbortSignal` cancellation with cooperative underlying work.
1243
1318
 
@@ -1,2 +1,2 @@
1
- import { a as ToolSearchOptions, i as ToolSearchCheckpointOptions, n as AgentLimits, o as ToolSearchSemanticOptions, r as ContextBudget, s as defineAgent, t as AgentDefinition } from "../index-lCCFokc6.mjs";
1
+ import { a as ToolSearchOptions, i as ToolSearchCheckpointOptions, n as AgentLimits, o as ToolSearchSemanticOptions, r as ContextBudget, s as defineAgent, t as AgentDefinition } from "../index-giFR5t0x.mjs";
2
2
  export { AgentDefinition, AgentLimits, ContextBudget, ToolSearchCheckpointOptions, ToolSearchOptions, ToolSearchSemanticOptions, defineAgent };
@@ -1,2 +1,2 @@
1
- import { A as ArtifactError, C as FileSystemArtifactStore, D as ArtifactCatalog, E as CatalogedArtifactStoreOptions, F as ArtifactStore, I as ArtifactStoreContext, L as ArtifactWrite, M as ArtifactId, N as ArtifactQuery, O as ArtifactContent, P as ArtifactRecord, R as artifactTenantId, S as InMemoryArtifactCatalog, T as CatalogedArtifactStore, _ as S3ArtifactStore, b as InMemoryArtifactStore, j as ArtifactErrorCode, k as ArtifactDescriptor, v as S3ArtifactStoreOptions, w as FileSystemArtifactStoreOptions, x as InMemoryArtifactStoreOptions, y as S3ObjectClient, z as assertArtifactId } from "../types-CckDlhYo.mjs";
1
+ import { A as ArtifactError, C as FileSystemArtifactStore, D as ArtifactCatalog, E as CatalogedArtifactStoreOptions, F as ArtifactStore, I as ArtifactStoreContext, L as ArtifactWrite, M as ArtifactId, N as ArtifactQuery, O as ArtifactContent, P as ArtifactRecord, R as artifactTenantId, S as InMemoryArtifactCatalog, T as CatalogedArtifactStore, _ as S3ArtifactStore, b as InMemoryArtifactStore, j as ArtifactErrorCode, k as ArtifactDescriptor, v as S3ArtifactStoreOptions, w as FileSystemArtifactStoreOptions, x as InMemoryArtifactStoreOptions, y as S3ObjectClient, z as assertArtifactId } from "../types-B6hD1o2I.mjs";
2
2
  export { ArtifactCatalog, ArtifactContent, ArtifactDescriptor, ArtifactError, ArtifactErrorCode, ArtifactId, ArtifactQuery, ArtifactRecord, ArtifactStore, ArtifactStoreContext, ArtifactWrite, CatalogedArtifactStore, CatalogedArtifactStoreOptions, FileSystemArtifactStore, FileSystemArtifactStoreOptions, InMemoryArtifactCatalog, InMemoryArtifactStore, InMemoryArtifactStoreOptions, S3ArtifactStore, S3ArtifactStoreOptions, S3ObjectClient, artifactTenantId, assertArtifactId };
@@ -1,2 +1,2 @@
1
- import { a as ChannelDeliveryOptions, c as ChannelEvent, d as ChannelMessage, f as ChannelOutput, h as ChannelRunResult, i as ChannelContext, l as ChannelEventType, m as ChannelRunOptions, n as handleChannelInput, o as ChannelElicitationContext, p as ChannelRunEvent, r as ChannelAdapter, s as ChannelError, t as deliverChannelRun, u as ChannelFailurePhase } from "../index-B8EcvWeN.mjs";
1
+ import { a as ChannelDeliveryOptions, c as ChannelEvent, d as ChannelMessage, f as ChannelOutput, h as ChannelRunResult, i as ChannelContext, l as ChannelEventType, m as ChannelRunOptions, n as handleChannelInput, o as ChannelElicitationContext, p as ChannelRunEvent, r as ChannelAdapter, s as ChannelError, t as deliverChannelRun, u as ChannelFailurePhase } from "../index-AVlWktA8.mjs";
2
2
  export { ChannelAdapter, ChannelContext, ChannelDeliveryOptions, ChannelElicitationContext, ChannelError, ChannelEvent, ChannelEventType, ChannelFailurePhase, ChannelMessage, ChannelOutput, ChannelRunEvent, ChannelRunOptions, ChannelRunResult, deliverChannelRun, handleChannelInput };
@@ -1,7 +1,9 @@
1
- import { r as ModelMetadata, s as ModelUsage } from "../index-D7r0puBi.mjs";
2
- import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-CckDlhYo.mjs";
3
- import { t as CostCalculator, u as TraceCost } from "../index-BEqYWean.mjs";
4
- import { t as Fevex } from "../fevex-_KOsOsB8.mjs";
1
+ import { I as ToolRisk } from "../index-AOouohIx.mjs";
2
+ import { r as ModelMetadata, s as ModelUsage } from "../index-Cbe1ZiyN.mjs";
3
+ import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-B6hD1o2I.mjs";
4
+ import { t as CostCalculator, u as TraceCost } from "../index-B7fxPgn_.mjs";
5
+ import { n as PolicyDecision } from "../index-KKq-_SiK.mjs";
6
+ import { t as Fevex } from "../fevex-B4eNkGTa.mjs";
5
7
  //#region src/evals/canonical-json.d.ts
6
8
  /** Serializes JSON-like data with stable object-key ordering. */
7
9
  declare function serializeCanonical(value: unknown): string;
@@ -10,12 +12,98 @@ declare function serializeCanonical(value: unknown): string;
10
12
  interface EvalExpectation {
11
13
  output?: JsonValue;
12
14
  tools?: string[];
15
+ trajectory?: EvalTrajectoryExpectation[];
16
+ }
17
+ /** One decision point recorded while an evaluation case runs. */
18
+ type EvalTrajectoryEntry = {
19
+ type: 'tool';
20
+ step: number;
21
+ toolCallId: string;
22
+ name: string;
23
+ attempt: number;
24
+ risk: ToolRisk;
25
+ declaredRisk?: ToolRisk;
26
+ status: 'started' | 'completed' | 'failed' | 'unknown' | 'retrying';
27
+ } | {
28
+ type: 'approval';
29
+ toolCallId: string;
30
+ toolName: string;
31
+ decision: 'approve' | 'reject';
32
+ actorId: string;
33
+ } | {
34
+ type: 'elicitation';
35
+ requestId: string;
36
+ toolCallId: string;
37
+ prompt?: string;
38
+ actorId: string;
39
+ } | {
40
+ type: 'policy';
41
+ policyName: string;
42
+ toolName: string;
43
+ toolCallId?: string;
44
+ action: 'tool.execute' | 'approval.resolve';
45
+ decision: Exclude<PolicyDecision, 'allow'>;
46
+ risk: ToolRisk;
47
+ declaredRisk?: ToolRisk;
48
+ };
49
+ /** Expected trajectory step. Omitted fields match any value. */
50
+ type EvalTrajectoryExpectation = {
51
+ type: 'tool';
52
+ name: string;
53
+ status?: 'started' | 'completed' | 'failed' | 'unknown' | 'retrying';
54
+ attempt?: number;
55
+ risk?: ToolRisk;
56
+ declaredRisk?: ToolRisk;
57
+ } | {
58
+ type: 'approval';
59
+ toolName: string;
60
+ decision: 'approve' | 'reject';
61
+ actorId?: string;
62
+ } | {
63
+ type: 'elicitation';
64
+ requestId?: string;
65
+ toolCallId?: string;
66
+ prompt?: string;
67
+ actorId?: string;
68
+ } | {
69
+ type: 'policy';
70
+ toolName: string;
71
+ decision: Exclude<PolicyDecision, 'allow'>;
72
+ policyName?: string;
73
+ action?: 'tool.execute' | 'approval.resolve';
74
+ declaredRisk?: ToolRisk;
75
+ };
76
+ /** Approval resolution supplied by an evaluation case when a run pauses. */
77
+ interface EvalApproval {
78
+ /** Tool name required to match the paused approval. Omit to accept any tool. */
79
+ toolName?: string;
80
+ decision: 'approve' | 'reject';
81
+ actor?: {
82
+ id: string;
83
+ type?: string;
84
+ };
85
+ }
86
+ /** Elicitation value supplied by an evaluation case when a run pauses. */
87
+ interface EvalElicitation {
88
+ /** Prompt text required to match the paused request. Omit to accept any prompt. */
89
+ prompt?: string;
90
+ value: JsonValue;
91
+ actor?: {
92
+ id: string;
93
+ type?: string;
94
+ };
13
95
  }
14
96
  interface EvalCase<TInput = unknown> {
15
97
  id: string;
16
98
  input: TInput;
17
99
  expected?: EvalExpectation;
18
100
  context?: ExecutionContext;
101
+ /** Resolutions consumed in order when the run pauses for approval. */
102
+ approvals?: readonly EvalApproval[];
103
+ /** Values consumed in order when the run pauses for elicitation. */
104
+ elicitations?: readonly EvalElicitation[];
105
+ /** Maximum wall-clock time for this case, including pauses. */
106
+ deadlineMs?: number;
19
107
  }
20
108
  interface EvalDataset<TInput = unknown> {
21
109
  name: string;
@@ -26,6 +114,7 @@ interface EvalScorerContext<TInput = unknown> {
26
114
  testCase: EvalCase<TInput>;
27
115
  output?: JsonValue;
28
116
  tools: readonly string[];
117
+ trajectory: readonly EvalTrajectoryEntry[];
29
118
  events: readonly AgentEvent[];
30
119
  usage?: ModelUsage;
31
120
  cost?: TraceCost;
@@ -57,10 +146,12 @@ interface EvalCaseReport {
57
146
  status: 'completed' | 'failed';
58
147
  output?: JsonValue;
59
148
  tools: string[];
149
+ trajectory?: EvalTrajectoryEntry[];
60
150
  usage?: ModelUsage;
61
151
  cost?: TraceCost;
62
152
  latencyMs: number;
63
153
  scores: EvalScore[];
154
+ warnings?: string[];
64
155
  error?: string;
65
156
  }
66
157
  interface EvalSummary {
@@ -71,6 +162,8 @@ interface EvalSummary {
71
162
  failedScoreCount: number;
72
163
  skippedScoreCount: number;
73
164
  errorScoreCount: number;
165
+ /** Absent on reports created before case warnings existed. */
166
+ warningCount?: number;
74
167
  averageScore?: number;
75
168
  usage?: ModelUsage;
76
169
  cost?: TraceCost;
@@ -107,6 +200,8 @@ interface RunEvaluationOptions<TInput = unknown> {
107
200
  reporters?: readonly EvalReporter[];
108
201
  cost?: EvalCostOptions;
109
202
  signal?: AbortSignal;
203
+ /** Default per-case wall-clock deadline. Defaults to 10 minutes. */
204
+ deadlineMs?: number;
110
205
  }
111
206
  interface Regression {
112
207
  kind: 'case_removed' | 'case_failed' | 'scorer_removed' | 'score_failed' | 'score_dropped' | 'latency_increased' | 'cost_increased';
@@ -125,6 +220,59 @@ interface RegressionOptions {
125
220
  maxLatencyIncreaseRatio?: number;
126
221
  maxCostIncreaseRatio?: number;
127
222
  }
223
+ /** Minimum aggregate requirements for a release evaluation gate. */
224
+ interface EvalGateThresholds {
225
+ /** Fraction of cases that must complete. Defaults to 1. */
226
+ minCompletionRate?: number;
227
+ /** Fraction of non-skipped scores that must pass. Defaults to 1. */
228
+ minPassRate?: number;
229
+ /** Maximum failed scores across all cases. Defaults to 0. */
230
+ maxFailedScores?: number;
231
+ /** Maximum scorer errors across all cases. Defaults to 0. */
232
+ maxErrorScores?: number;
233
+ /** Maximum case warnings, such as unused resolution fixtures. Defaults to 0. */
234
+ maxWarnings?: number;
235
+ /** Minimum average score across numeric results. */
236
+ minAverageScore?: number;
237
+ /** Scorers that must be present in the report. */
238
+ requiredScorers?: readonly string[];
239
+ /** Maximum aggregate latency. */
240
+ maxLatencyMs?: number;
241
+ /** Maximum aggregate cost. Currency must match exactly. */
242
+ maxCost?: TraceCost;
243
+ }
244
+ interface ReleaseGateOptions extends EvalGateThresholds {
245
+ /** Optional baseline report; trajectory or policy regressions fail the gate. */
246
+ baseline?: EvalReport;
247
+ regression?: RegressionOptions;
248
+ }
249
+ interface EvalScorecard {
250
+ caseCount: number;
251
+ completedCount: number;
252
+ completionRate: number;
253
+ passedScoreCount: number;
254
+ failedScoreCount: number;
255
+ skippedScoreCount: number;
256
+ errorScoreCount: number;
257
+ warningCount: number;
258
+ passRate: number;
259
+ averageScore?: number;
260
+ latencyMs: number;
261
+ cost?: TraceCost;
262
+ }
263
+ interface EvalGateCheck {
264
+ name: string;
265
+ passed: boolean;
266
+ message: string;
267
+ expected?: JsonValue;
268
+ actual?: JsonValue;
269
+ }
270
+ interface ReleaseGateResult {
271
+ passed: boolean;
272
+ scorecard: EvalScorecard;
273
+ checks: EvalGateCheck[];
274
+ regressions: Regression[];
275
+ }
128
276
  //#endregion
129
277
  //#region src/evals/evaluation.d.ts
130
278
  /** Runs every case and scorer, producing a deterministic aggregate report. */
@@ -134,6 +282,18 @@ declare function parseEvalReport(json: string): EvalReport;
134
282
  declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
135
283
  declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
136
284
  declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
285
+ /**
286
+ * Evaluates minimum scorecard thresholds and an optional baseline. A trajectory
287
+ * or policy regression appears as a failed score and blocks the gate even when
288
+ * the final output is correct.
289
+ */
290
+ declare function evaluateReleaseGate(report: EvalReport, options?: ReleaseGateOptions): ReleaseGateResult;
291
+ declare class ReleaseGateError extends Error {
292
+ readonly result: ReleaseGateResult;
293
+ readonly name = "ReleaseGateError";
294
+ constructor(result: ReleaseGateResult);
295
+ }
296
+ declare function assertReleaseGate(report: EvalReport, options?: ReleaseGateOptions): ReleaseGateResult;
137
297
  //#endregion
138
298
  //#region src/evals/scorers.d.ts
139
299
  declare function exactOutputScorer(id?: string): EvalScorer;
@@ -149,5 +309,38 @@ declare function maxTokensScorer(limits: {
149
309
  total?: number;
150
310
  }, id?: string): EvalScorer;
151
311
  declare function maxCostScorer(maxAmount: number, currency: string, id?: string): EvalScorer;
312
+ interface TrajectoryScorerOptions {
313
+ id?: string;
314
+ /** Defaults to true. Set false to assert the same steps in any order. */
315
+ ordered?: boolean;
316
+ /** Defaults to false. Set true to allow unasserted steps. */
317
+ allowAdditional?: boolean;
318
+ }
319
+ /** Asserts the ordered tool, approval and non-allow policy decisions of a run. */
320
+ declare function trajectoryScorer(expected: readonly EvalTrajectoryExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
321
+ /** Reads the expected trajectory from each dataset case. */
322
+ declare function trajectoryExpectationScorer(options?: TrajectoryScorerOptions): EvalScorer;
323
+ type ApprovalExpectation = Extract<EvalTrajectoryExpectation, {
324
+ type: 'approval';
325
+ }>;
326
+ type ElicitationExpectation = Extract<EvalTrajectoryExpectation, {
327
+ type: 'elicitation';
328
+ }>;
329
+ type PolicyExpectation = Extract<EvalTrajectoryExpectation, {
330
+ type: 'policy';
331
+ }>;
332
+ declare function approvalScorer(expected: readonly ApprovalExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
333
+ declare function elicitationScorer(expected: readonly ElicitationExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
334
+ declare function policyScorer(expected: readonly PolicyExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
335
+ interface RiskGuardrailScorerOptions {
336
+ /** Risks that must never execute. Defaults to `destructive`. */
337
+ deny?: readonly ToolRisk[];
338
+ /** Risks that must have an approved approval before execution. Defaults to `sensitive`. */
339
+ requireApproval?: readonly ToolRisk[];
340
+ /** Expected handling of tools that omit `risk`. Defaults to `allow`. */
341
+ undeclaredRisk?: 'allow' | 'require_approval' | 'deny';
342
+ }
343
+ /** Asserts that risky tools respected the configured guardrail. */
344
+ declare function riskGuardrailScorer(options?: RiskGuardrailScorerOptions, id?: string): EvalScorer;
152
345
  //#endregion
153
- export { EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalExpectation, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, Regression, RegressionOptions, RegressionReport, RunEvaluationOptions, compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
346
+ export { EvalApproval, EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalElicitation, EvalExpectation, EvalGateCheck, EvalGateThresholds, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorecard, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, EvalTrajectoryEntry, EvalTrajectoryExpectation, Regression, RegressionOptions, RegressionReport, ReleaseGateError, ReleaseGateOptions, ReleaseGateResult, RiskGuardrailScorerOptions, RunEvaluationOptions, TrajectoryScorerOptions, approvalScorer, assertReleaseGate, compareEvaluationReports, elicitationScorer, evaluateReleaseGate, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, policyScorer, riskGuardrailScorer, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer, trajectoryExpectationScorer, trajectoryScorer };