@fevex/core 0.1.0-alpha.11 → 0.1.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -14
- package/dist/agents/index.d.mts +1 -1
- package/dist/artifacts/index.d.mts +1 -1
- package/dist/channels/index.d.mts +1 -1
- package/dist/evals/index.d.mts +198 -5
- package/dist/evals/index.mjs +478 -12
- package/dist/{fevex-_KOsOsB8.d.mts → fevex-B4eNkGTa.d.mts} +19 -10
- package/dist/http/index.d.mts +4 -4
- package/dist/{index-DHcfIfh-.d.mts → index-AOouohIx.d.mts} +3 -3
- package/dist/{index-B8EcvWeN.d.mts → index-AVlWktA8.d.mts} +3 -3
- package/dist/{index-BEqYWean.d.mts → index-B7fxPgn_.d.mts} +5 -5
- package/dist/{index-Btth7rdL.d.mts → index-CAVMQT9j.d.mts} +3 -3
- package/dist/{index-D7r0puBi.d.mts → index-Cbe1ZiyN.d.mts} +1 -1
- package/dist/{index-CIHWc5V1.d.mts → index-D1LAhzgC.d.mts} +2 -2
- package/dist/{index-26ilbzAs.d.mts → index-DMmrhb8l.d.mts} +4 -4
- package/dist/index-KKq-_SiK.d.mts +43 -0
- package/dist/{index-lCCFokc6.d.mts → index-giFR5t0x.d.mts} +5 -3
- package/dist/{index-DUhNa7IR.d.mts → index-k-gBmfJP.d.mts} +1 -1
- package/dist/index.d.mts +13 -13
- package/dist/index.mjs +214 -31
- package/dist/knowledge/index.d.mts +1 -1
- package/dist/models/index.d.mts +1 -1
- package/dist/observability/index.d.mts +1 -1
- package/dist/policies/index.d.mts +2 -2
- package/dist/policies/index.mjs +39 -1
- package/dist/runtime/index.d.mts +1 -1
- package/dist/sandbox/index.d.mts +1 -1
- package/dist/teams/index.d.mts +1 -1
- package/dist/testing/index.d.mts +6 -6
- package/dist/tools/index.d.mts +2 -2
- package/dist/{types-CckDlhYo.d.mts → types-B6hD1o2I.d.mts} +19 -1
- package/dist/workflows/index.d.mts +1 -1
- package/package.json +1 -1
- package/dist/index-CFzrvDQ0.d.mts +0 -21
package/README.md
CHANGED
|
@@ -447,9 +447,11 @@ model.started
|
|
|
447
447
|
model.output.delta
|
|
448
448
|
model.completed
|
|
449
449
|
context.compacted
|
|
450
|
+
memory.completed
|
|
450
451
|
tool.started
|
|
451
452
|
tool.completed
|
|
452
453
|
tool.failed
|
|
454
|
+
policy.evaluated
|
|
453
455
|
run.completed
|
|
454
456
|
run.failed
|
|
455
457
|
run.cancelled
|
|
@@ -547,6 +549,7 @@ const started = await app.startAgent('support', {
|
|
|
547
549
|
});
|
|
548
550
|
|
|
549
551
|
const run = await app.getRun(started.id);
|
|
552
|
+
const finished = await app.waitForRun(started.id, { timeoutMs: 30_000 });
|
|
550
553
|
const events = await app.listEvents(started.id);
|
|
551
554
|
const laterEvents = await app.listEvents(started.id, {
|
|
552
555
|
after: events.at(-1)?.id,
|
|
@@ -555,9 +558,11 @@ const laterEvents = await app.listEvents(started.id, {
|
|
|
555
558
|
await app.cancelRun(started.id);
|
|
556
559
|
```
|
|
557
560
|
|
|
558
|
-
`
|
|
559
|
-
|
|
560
|
-
|
|
561
|
+
`waitForRun` waits for the run to leave `running` (paused or terminal) without
|
|
562
|
+
polling, and accepts an optional `signal` and `timeoutMs`. `AgentRun`, `Session`
|
|
563
|
+
and events are serializable snapshots. `listEvents` orders by `sequence`; `after`
|
|
564
|
+
accepts an event ID from the same run as a stable cursor. Unknown cursors fail
|
|
565
|
+
instead of silently replaying events.
|
|
561
566
|
|
|
562
567
|
`createFevex` uses `InMemoryRunStore` by default. Supply `runStore` to replace
|
|
563
568
|
it. The default survives only while the process lives, and one run at a time
|
|
@@ -706,7 +711,23 @@ remain explicit and are intended for server deployments.
|
|
|
706
711
|
|
|
707
712
|
Policies run before approval resolution, credentials and each effect or retry.
|
|
708
713
|
Credential values are not stored or sent to the model, and tool outputs that
|
|
709
|
-
contain a resolved value are rejected.
|
|
714
|
+
contain a resolved value are rejected. Every decision emits `policy.evaluated`
|
|
715
|
+
and appears in local traces and OpenTelemetry span events.
|
|
716
|
+
|
|
717
|
+
For a static baseline, `riskGuardrail()` denies `destructive` tools and requires
|
|
718
|
+
approval for `sensitive` tools. Tools without an explicit `risk` default to
|
|
719
|
+
`read`; set `undeclaredRisk: 'require_approval'` or `'deny'` to fail closed:
|
|
720
|
+
|
|
721
|
+
```ts
|
|
722
|
+
import { riskGuardrail } from '@fevex/core/policies';
|
|
723
|
+
|
|
724
|
+
createFevex({
|
|
725
|
+
models,
|
|
726
|
+
agents,
|
|
727
|
+
tools,
|
|
728
|
+
policies: [riskGuardrail({ undeclaredRisk: 'require_approval' })],
|
|
729
|
+
});
|
|
730
|
+
```
|
|
710
731
|
|
|
711
732
|
## Limits
|
|
712
733
|
|
|
@@ -750,7 +771,7 @@ tenant. The default tenant key is a non-empty `context.attributes.tenantId`;
|
|
|
750
771
|
one runtime process.
|
|
751
772
|
|
|
752
773
|
`createFevex({ limits: { maxEventsPerRun } })` caps non-control events per run.
|
|
753
|
-
Pause, resume, approval, elicitation, wait and terminal events are always
|
|
774
|
+
Pause, resume, approval, policy, elicitation, wait and terminal events are always
|
|
754
775
|
persisted so a run can always reach a durable state; use retention for a total
|
|
755
776
|
storage cap.
|
|
756
777
|
|
|
@@ -814,7 +835,11 @@ console.log(app.getObservabilityStats());
|
|
|
814
835
|
|
|
815
836
|
Workflow and team traces carry `kind` and a span per step. Agent steps expose
|
|
816
837
|
`childRunId`; child agent traces carry `parentRunId` and `rootRunId`, and tool
|
|
817
|
-
spans are nested under the model call that requested them.
|
|
838
|
+
spans are nested under the model call that requested them. Memory search and
|
|
839
|
+
write operations produce `search_memory` and `create_memory` GenAI spans. Policy
|
|
840
|
+
decisions are trace annotations and OpenTelemetry span events, with
|
|
841
|
+
`fevex.tool.risk` on tool spans and `gen_ai.memory.*` attributes on memory
|
|
842
|
+
spans. A dropped trace or a
|
|
818
843
|
failed export is counted in `getObservabilityStats()`, and the next
|
|
819
844
|
`flushObservability()` rejects with the failures accumulated since the previous
|
|
820
845
|
flush. OpenTelemetry SDK setup, OTLP export and shutdown remain application
|
|
@@ -869,11 +894,60 @@ const report = await runEvaluation({
|
|
|
869
894
|
```
|
|
870
895
|
|
|
871
896
|
Built-in deterministic scorers cover exact output, ordered or unordered tool
|
|
872
|
-
selection, forbidden tools, max latency, max tokens and max cost.
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
897
|
+
selection, forbidden tools, max latency, max tokens and max cost. Cases can also
|
|
898
|
+
resolve paused approvals and assert the full trajectory:
|
|
899
|
+
|
|
900
|
+
```ts
|
|
901
|
+
import {
|
|
902
|
+
riskGuardrailScorer,
|
|
903
|
+
runEvaluation,
|
|
904
|
+
trajectoryExpectationScorer,
|
|
905
|
+
} from '@fevex/core/evals';
|
|
906
|
+
|
|
907
|
+
const report = await runEvaluation({
|
|
908
|
+
app,
|
|
909
|
+
suiteName: 'support-regression',
|
|
910
|
+
targetVersion: 'v2',
|
|
911
|
+
agentName: 'support',
|
|
912
|
+
dataset: {
|
|
913
|
+
name: 'support',
|
|
914
|
+
version: '1',
|
|
915
|
+
cases: [{
|
|
916
|
+
id: 'sensitive-write',
|
|
917
|
+
input: 'Update account 42',
|
|
918
|
+
approvals: [{ toolName: 'accounts_update', decision: 'approve' }],
|
|
919
|
+
expected: {
|
|
920
|
+
output: { answer: 'Account 42 updated.' },
|
|
921
|
+
tools: ['accounts_update'],
|
|
922
|
+
trajectory: [
|
|
923
|
+
{ type: 'policy', toolName: 'accounts_update', decision: 'require_approval' },
|
|
924
|
+
{ type: 'approval', toolName: 'accounts_update', decision: 'approve' },
|
|
925
|
+
{ type: 'tool', name: 'accounts_update', status: 'completed' },
|
|
926
|
+
],
|
|
927
|
+
},
|
|
928
|
+
}],
|
|
929
|
+
},
|
|
930
|
+
scorers: [
|
|
931
|
+
trajectoryExpectationScorer({ allowAdditional: true }),
|
|
932
|
+
riskGuardrailScorer(),
|
|
933
|
+
],
|
|
934
|
+
});
|
|
935
|
+
```
|
|
936
|
+
|
|
937
|
+
Trajectory entries cover tool attempts, approval decisions, elicitation
|
|
938
|
+
resolutions and non-allow policy decisions. Cases can also answer elicitation
|
|
939
|
+
pauses with `elicitations`, matched by prompt when provided; the runtime still
|
|
940
|
+
validates each value against its `responseSchema`. Each case has a 10-minute deadline by default; set `deadlineMs`
|
|
941
|
+
on the suite or case to change it. Use `serializeEvalReport` for a canonical,
|
|
942
|
+
versionable JSON baseline,
|
|
943
|
+
`jsonReporter` or `textReporter` with your own write callback, and
|
|
944
|
+
`compareEvaluationReports` to fail CI when cases disappear, passing scores fail
|
|
945
|
+
or scores drop beyond the configured tolerance. Cases warn when an `approvals`
|
|
946
|
+
or `elicitations` fixture was never used; the gate rejects warnings by default.
|
|
947
|
+
`evaluateReleaseGate` adds minimum completion, pass, score, warning, latency and
|
|
948
|
+
cost thresholds; `assertReleaseGate` throws `ReleaseGateError` with the failed
|
|
949
|
+
checks. The repository runs a deterministic release suite with
|
|
950
|
+
`bun run eval:gate` and the built package with `bun run eval:gate:package`.
|
|
877
951
|
|
|
878
952
|
## Connections
|
|
879
953
|
|
|
@@ -1204,9 +1278,9 @@ the artifact available.
|
|
|
1204
1278
|
| `@fevex/core/teams` | Multiagent team definitions |
|
|
1205
1279
|
| `@fevex/core/runtime` | Runs, sessions and store contracts |
|
|
1206
1280
|
| `@fevex/core/sandbox` | Sandbox contract and local development sandbox |
|
|
1207
|
-
| `@fevex/core/policies` | Authorization policy contracts |
|
|
1281
|
+
| `@fevex/core/policies` | Authorization policy contracts and the risk guardrail |
|
|
1208
1282
|
| `@fevex/core/observability` | Trace, bounded export queue, redaction and cost contracts |
|
|
1209
|
-
| `@fevex/core/evals` | Datasets, scorers,
|
|
1283
|
+
| `@fevex/core/evals` | Datasets, trajectory and policy scorers, release gates and regressions |
|
|
1210
1284
|
| `@fevex/core/testing` | Deterministic testing helpers |
|
|
1211
1285
|
|
|
1212
1286
|
All subpaths ship in the same `@fevex/core` package.
|
|
@@ -1237,7 +1311,8 @@ The MVP intentionally supports:
|
|
|
1237
1311
|
- bounded multi-step model-tool loops with sequential tool execution;
|
|
1238
1312
|
- native provider streaming and durable, consultable runs;
|
|
1239
1313
|
- synchronous event observers and `AsyncIterable` event streaming;
|
|
1240
|
-
- local traces, OpenTelemetry export and deterministic evaluation suites
|
|
1314
|
+
- local traces, OpenTelemetry export and deterministic evaluation suites with
|
|
1315
|
+
trajectory, policy, approval, cost and release-gate assertions;
|
|
1241
1316
|
- optional Standard Schema validation;
|
|
1242
1317
|
- immediate `AbortSignal` cancellation with cooperative underlying work.
|
|
1243
1318
|
|
package/dist/agents/index.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as ToolSearchOptions, i as ToolSearchCheckpointOptions, n as AgentLimits, o as ToolSearchSemanticOptions, r as ContextBudget, s as defineAgent, t as AgentDefinition } from "../index-
|
|
1
|
+
import { a as ToolSearchOptions, i as ToolSearchCheckpointOptions, n as AgentLimits, o as ToolSearchSemanticOptions, r as ContextBudget, s as defineAgent, t as AgentDefinition } from "../index-giFR5t0x.mjs";
|
|
2
2
|
export { AgentDefinition, AgentLimits, ContextBudget, ToolSearchCheckpointOptions, ToolSearchOptions, ToolSearchSemanticOptions, defineAgent };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { A as ArtifactError, C as FileSystemArtifactStore, D as ArtifactCatalog, E as CatalogedArtifactStoreOptions, F as ArtifactStore, I as ArtifactStoreContext, L as ArtifactWrite, M as ArtifactId, N as ArtifactQuery, O as ArtifactContent, P as ArtifactRecord, R as artifactTenantId, S as InMemoryArtifactCatalog, T as CatalogedArtifactStore, _ as S3ArtifactStore, b as InMemoryArtifactStore, j as ArtifactErrorCode, k as ArtifactDescriptor, v as S3ArtifactStoreOptions, w as FileSystemArtifactStoreOptions, x as InMemoryArtifactStoreOptions, y as S3ObjectClient, z as assertArtifactId } from "../types-
|
|
1
|
+
import { A as ArtifactError, C as FileSystemArtifactStore, D as ArtifactCatalog, E as CatalogedArtifactStoreOptions, F as ArtifactStore, I as ArtifactStoreContext, L as ArtifactWrite, M as ArtifactId, N as ArtifactQuery, O as ArtifactContent, P as ArtifactRecord, R as artifactTenantId, S as InMemoryArtifactCatalog, T as CatalogedArtifactStore, _ as S3ArtifactStore, b as InMemoryArtifactStore, j as ArtifactErrorCode, k as ArtifactDescriptor, v as S3ArtifactStoreOptions, w as FileSystemArtifactStoreOptions, x as InMemoryArtifactStoreOptions, y as S3ObjectClient, z as assertArtifactId } from "../types-B6hD1o2I.mjs";
|
|
2
2
|
export { ArtifactCatalog, ArtifactContent, ArtifactDescriptor, ArtifactError, ArtifactErrorCode, ArtifactId, ArtifactQuery, ArtifactRecord, ArtifactStore, ArtifactStoreContext, ArtifactWrite, CatalogedArtifactStore, CatalogedArtifactStoreOptions, FileSystemArtifactStore, FileSystemArtifactStoreOptions, InMemoryArtifactCatalog, InMemoryArtifactStore, InMemoryArtifactStoreOptions, S3ArtifactStore, S3ArtifactStoreOptions, S3ObjectClient, artifactTenantId, assertArtifactId };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { a as ChannelDeliveryOptions, c as ChannelEvent, d as ChannelMessage, f as ChannelOutput, h as ChannelRunResult, i as ChannelContext, l as ChannelEventType, m as ChannelRunOptions, n as handleChannelInput, o as ChannelElicitationContext, p as ChannelRunEvent, r as ChannelAdapter, s as ChannelError, t as deliverChannelRun, u as ChannelFailurePhase } from "../index-
|
|
1
|
+
import { a as ChannelDeliveryOptions, c as ChannelEvent, d as ChannelMessage, f as ChannelOutput, h as ChannelRunResult, i as ChannelContext, l as ChannelEventType, m as ChannelRunOptions, n as handleChannelInput, o as ChannelElicitationContext, p as ChannelRunEvent, r as ChannelAdapter, s as ChannelError, t as deliverChannelRun, u as ChannelFailurePhase } from "../index-AVlWktA8.mjs";
|
|
2
2
|
export { ChannelAdapter, ChannelContext, ChannelDeliveryOptions, ChannelElicitationContext, ChannelError, ChannelEvent, ChannelEventType, ChannelFailurePhase, ChannelMessage, ChannelOutput, ChannelRunEvent, ChannelRunOptions, ChannelRunResult, deliverChannelRun, handleChannelInput };
|
package/dist/evals/index.d.mts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { t as
|
|
1
|
+
import { I as ToolRisk } from "../index-AOouohIx.mjs";
|
|
2
|
+
import { r as ModelMetadata, s as ModelUsage } from "../index-Cbe1ZiyN.mjs";
|
|
3
|
+
import { l as JsonValue, o as ExecutionContext, t as AgentEvent } from "../types-B6hD1o2I.mjs";
|
|
4
|
+
import { t as CostCalculator, u as TraceCost } from "../index-B7fxPgn_.mjs";
|
|
5
|
+
import { n as PolicyDecision } from "../index-KKq-_SiK.mjs";
|
|
6
|
+
import { t as Fevex } from "../fevex-B4eNkGTa.mjs";
|
|
5
7
|
//#region src/evals/canonical-json.d.ts
|
|
6
8
|
/** Serializes JSON-like data with stable object-key ordering. */
|
|
7
9
|
declare function serializeCanonical(value: unknown): string;
|
|
@@ -10,12 +12,98 @@ declare function serializeCanonical(value: unknown): string;
|
|
|
10
12
|
interface EvalExpectation {
|
|
11
13
|
output?: JsonValue;
|
|
12
14
|
tools?: string[];
|
|
15
|
+
trajectory?: EvalTrajectoryExpectation[];
|
|
16
|
+
}
|
|
17
|
+
/** One decision point recorded while an evaluation case runs. */
|
|
18
|
+
type EvalTrajectoryEntry = {
|
|
19
|
+
type: 'tool';
|
|
20
|
+
step: number;
|
|
21
|
+
toolCallId: string;
|
|
22
|
+
name: string;
|
|
23
|
+
attempt: number;
|
|
24
|
+
risk: ToolRisk;
|
|
25
|
+
declaredRisk?: ToolRisk;
|
|
26
|
+
status: 'started' | 'completed' | 'failed' | 'unknown' | 'retrying';
|
|
27
|
+
} | {
|
|
28
|
+
type: 'approval';
|
|
29
|
+
toolCallId: string;
|
|
30
|
+
toolName: string;
|
|
31
|
+
decision: 'approve' | 'reject';
|
|
32
|
+
actorId: string;
|
|
33
|
+
} | {
|
|
34
|
+
type: 'elicitation';
|
|
35
|
+
requestId: string;
|
|
36
|
+
toolCallId: string;
|
|
37
|
+
prompt?: string;
|
|
38
|
+
actorId: string;
|
|
39
|
+
} | {
|
|
40
|
+
type: 'policy';
|
|
41
|
+
policyName: string;
|
|
42
|
+
toolName: string;
|
|
43
|
+
toolCallId?: string;
|
|
44
|
+
action: 'tool.execute' | 'approval.resolve';
|
|
45
|
+
decision: Exclude<PolicyDecision, 'allow'>;
|
|
46
|
+
risk: ToolRisk;
|
|
47
|
+
declaredRisk?: ToolRisk;
|
|
48
|
+
};
|
|
49
|
+
/** Expected trajectory step. Omitted fields match any value. */
|
|
50
|
+
type EvalTrajectoryExpectation = {
|
|
51
|
+
type: 'tool';
|
|
52
|
+
name: string;
|
|
53
|
+
status?: 'started' | 'completed' | 'failed' | 'unknown' | 'retrying';
|
|
54
|
+
attempt?: number;
|
|
55
|
+
risk?: ToolRisk;
|
|
56
|
+
declaredRisk?: ToolRisk;
|
|
57
|
+
} | {
|
|
58
|
+
type: 'approval';
|
|
59
|
+
toolName: string;
|
|
60
|
+
decision: 'approve' | 'reject';
|
|
61
|
+
actorId?: string;
|
|
62
|
+
} | {
|
|
63
|
+
type: 'elicitation';
|
|
64
|
+
requestId?: string;
|
|
65
|
+
toolCallId?: string;
|
|
66
|
+
prompt?: string;
|
|
67
|
+
actorId?: string;
|
|
68
|
+
} | {
|
|
69
|
+
type: 'policy';
|
|
70
|
+
toolName: string;
|
|
71
|
+
decision: Exclude<PolicyDecision, 'allow'>;
|
|
72
|
+
policyName?: string;
|
|
73
|
+
action?: 'tool.execute' | 'approval.resolve';
|
|
74
|
+
declaredRisk?: ToolRisk;
|
|
75
|
+
};
|
|
76
|
+
/** Approval resolution supplied by an evaluation case when a run pauses. */
|
|
77
|
+
interface EvalApproval {
|
|
78
|
+
/** Tool name required to match the paused approval. Omit to accept any tool. */
|
|
79
|
+
toolName?: string;
|
|
80
|
+
decision: 'approve' | 'reject';
|
|
81
|
+
actor?: {
|
|
82
|
+
id: string;
|
|
83
|
+
type?: string;
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** Elicitation value supplied by an evaluation case when a run pauses. */
|
|
87
|
+
interface EvalElicitation {
|
|
88
|
+
/** Prompt text required to match the paused request. Omit to accept any prompt. */
|
|
89
|
+
prompt?: string;
|
|
90
|
+
value: JsonValue;
|
|
91
|
+
actor?: {
|
|
92
|
+
id: string;
|
|
93
|
+
type?: string;
|
|
94
|
+
};
|
|
13
95
|
}
|
|
14
96
|
interface EvalCase<TInput = unknown> {
|
|
15
97
|
id: string;
|
|
16
98
|
input: TInput;
|
|
17
99
|
expected?: EvalExpectation;
|
|
18
100
|
context?: ExecutionContext;
|
|
101
|
+
/** Resolutions consumed in order when the run pauses for approval. */
|
|
102
|
+
approvals?: readonly EvalApproval[];
|
|
103
|
+
/** Values consumed in order when the run pauses for elicitation. */
|
|
104
|
+
elicitations?: readonly EvalElicitation[];
|
|
105
|
+
/** Maximum wall-clock time for this case, including pauses. */
|
|
106
|
+
deadlineMs?: number;
|
|
19
107
|
}
|
|
20
108
|
interface EvalDataset<TInput = unknown> {
|
|
21
109
|
name: string;
|
|
@@ -26,6 +114,7 @@ interface EvalScorerContext<TInput = unknown> {
|
|
|
26
114
|
testCase: EvalCase<TInput>;
|
|
27
115
|
output?: JsonValue;
|
|
28
116
|
tools: readonly string[];
|
|
117
|
+
trajectory: readonly EvalTrajectoryEntry[];
|
|
29
118
|
events: readonly AgentEvent[];
|
|
30
119
|
usage?: ModelUsage;
|
|
31
120
|
cost?: TraceCost;
|
|
@@ -57,10 +146,12 @@ interface EvalCaseReport {
|
|
|
57
146
|
status: 'completed' | 'failed';
|
|
58
147
|
output?: JsonValue;
|
|
59
148
|
tools: string[];
|
|
149
|
+
trajectory?: EvalTrajectoryEntry[];
|
|
60
150
|
usage?: ModelUsage;
|
|
61
151
|
cost?: TraceCost;
|
|
62
152
|
latencyMs: number;
|
|
63
153
|
scores: EvalScore[];
|
|
154
|
+
warnings?: string[];
|
|
64
155
|
error?: string;
|
|
65
156
|
}
|
|
66
157
|
interface EvalSummary {
|
|
@@ -71,6 +162,8 @@ interface EvalSummary {
|
|
|
71
162
|
failedScoreCount: number;
|
|
72
163
|
skippedScoreCount: number;
|
|
73
164
|
errorScoreCount: number;
|
|
165
|
+
/** Absent on reports created before case warnings existed. */
|
|
166
|
+
warningCount?: number;
|
|
74
167
|
averageScore?: number;
|
|
75
168
|
usage?: ModelUsage;
|
|
76
169
|
cost?: TraceCost;
|
|
@@ -107,6 +200,8 @@ interface RunEvaluationOptions<TInput = unknown> {
|
|
|
107
200
|
reporters?: readonly EvalReporter[];
|
|
108
201
|
cost?: EvalCostOptions;
|
|
109
202
|
signal?: AbortSignal;
|
|
203
|
+
/** Default per-case wall-clock deadline. Defaults to 10 minutes. */
|
|
204
|
+
deadlineMs?: number;
|
|
110
205
|
}
|
|
111
206
|
interface Regression {
|
|
112
207
|
kind: 'case_removed' | 'case_failed' | 'scorer_removed' | 'score_failed' | 'score_dropped' | 'latency_increased' | 'cost_increased';
|
|
@@ -125,6 +220,59 @@ interface RegressionOptions {
|
|
|
125
220
|
maxLatencyIncreaseRatio?: number;
|
|
126
221
|
maxCostIncreaseRatio?: number;
|
|
127
222
|
}
|
|
223
|
+
/** Minimum aggregate requirements for a release evaluation gate. */
|
|
224
|
+
interface EvalGateThresholds {
|
|
225
|
+
/** Fraction of cases that must complete. Defaults to 1. */
|
|
226
|
+
minCompletionRate?: number;
|
|
227
|
+
/** Fraction of non-skipped scores that must pass. Defaults to 1. */
|
|
228
|
+
minPassRate?: number;
|
|
229
|
+
/** Maximum failed scores across all cases. Defaults to 0. */
|
|
230
|
+
maxFailedScores?: number;
|
|
231
|
+
/** Maximum scorer errors across all cases. Defaults to 0. */
|
|
232
|
+
maxErrorScores?: number;
|
|
233
|
+
/** Maximum case warnings, such as unused resolution fixtures. Defaults to 0. */
|
|
234
|
+
maxWarnings?: number;
|
|
235
|
+
/** Minimum average score across numeric results. */
|
|
236
|
+
minAverageScore?: number;
|
|
237
|
+
/** Scorers that must be present in the report. */
|
|
238
|
+
requiredScorers?: readonly string[];
|
|
239
|
+
/** Maximum aggregate latency. */
|
|
240
|
+
maxLatencyMs?: number;
|
|
241
|
+
/** Maximum aggregate cost. Currency must match exactly. */
|
|
242
|
+
maxCost?: TraceCost;
|
|
243
|
+
}
|
|
244
|
+
interface ReleaseGateOptions extends EvalGateThresholds {
|
|
245
|
+
/** Optional baseline report; trajectory or policy regressions fail the gate. */
|
|
246
|
+
baseline?: EvalReport;
|
|
247
|
+
regression?: RegressionOptions;
|
|
248
|
+
}
|
|
249
|
+
interface EvalScorecard {
|
|
250
|
+
caseCount: number;
|
|
251
|
+
completedCount: number;
|
|
252
|
+
completionRate: number;
|
|
253
|
+
passedScoreCount: number;
|
|
254
|
+
failedScoreCount: number;
|
|
255
|
+
skippedScoreCount: number;
|
|
256
|
+
errorScoreCount: number;
|
|
257
|
+
warningCount: number;
|
|
258
|
+
passRate: number;
|
|
259
|
+
averageScore?: number;
|
|
260
|
+
latencyMs: number;
|
|
261
|
+
cost?: TraceCost;
|
|
262
|
+
}
|
|
263
|
+
interface EvalGateCheck {
|
|
264
|
+
name: string;
|
|
265
|
+
passed: boolean;
|
|
266
|
+
message: string;
|
|
267
|
+
expected?: JsonValue;
|
|
268
|
+
actual?: JsonValue;
|
|
269
|
+
}
|
|
270
|
+
interface ReleaseGateResult {
|
|
271
|
+
passed: boolean;
|
|
272
|
+
scorecard: EvalScorecard;
|
|
273
|
+
checks: EvalGateCheck[];
|
|
274
|
+
regressions: Regression[];
|
|
275
|
+
}
|
|
128
276
|
//#endregion
|
|
129
277
|
//#region src/evals/evaluation.d.ts
|
|
130
278
|
/** Runs every case and scorer, producing a deterministic aggregate report. */
|
|
@@ -134,6 +282,18 @@ declare function parseEvalReport(json: string): EvalReport;
|
|
|
134
282
|
declare function jsonReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
135
283
|
declare function textReporter(write: (output: string) => void | Promise<void>): EvalReporter;
|
|
136
284
|
declare function compareEvaluationReports(baseline: EvalReport, current: EvalReport, options?: RegressionOptions): RegressionReport;
|
|
285
|
+
/**
|
|
286
|
+
* Evaluates minimum scorecard thresholds and an optional baseline. A trajectory
|
|
287
|
+
* or policy regression appears as a failed score and blocks the gate even when
|
|
288
|
+
* the final output is correct.
|
|
289
|
+
*/
|
|
290
|
+
declare function evaluateReleaseGate(report: EvalReport, options?: ReleaseGateOptions): ReleaseGateResult;
|
|
291
|
+
declare class ReleaseGateError extends Error {
|
|
292
|
+
readonly result: ReleaseGateResult;
|
|
293
|
+
readonly name = "ReleaseGateError";
|
|
294
|
+
constructor(result: ReleaseGateResult);
|
|
295
|
+
}
|
|
296
|
+
declare function assertReleaseGate(report: EvalReport, options?: ReleaseGateOptions): ReleaseGateResult;
|
|
137
297
|
//#endregion
|
|
138
298
|
//#region src/evals/scorers.d.ts
|
|
139
299
|
declare function exactOutputScorer(id?: string): EvalScorer;
|
|
@@ -149,5 +309,38 @@ declare function maxTokensScorer(limits: {
|
|
|
149
309
|
total?: number;
|
|
150
310
|
}, id?: string): EvalScorer;
|
|
151
311
|
declare function maxCostScorer(maxAmount: number, currency: string, id?: string): EvalScorer;
|
|
312
|
+
interface TrajectoryScorerOptions {
|
|
313
|
+
id?: string;
|
|
314
|
+
/** Defaults to true. Set false to assert the same steps in any order. */
|
|
315
|
+
ordered?: boolean;
|
|
316
|
+
/** Defaults to false. Set true to allow unasserted steps. */
|
|
317
|
+
allowAdditional?: boolean;
|
|
318
|
+
}
|
|
319
|
+
/** Asserts the ordered tool, approval and non-allow policy decisions of a run. */
|
|
320
|
+
declare function trajectoryScorer(expected: readonly EvalTrajectoryExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
|
|
321
|
+
/** Reads the expected trajectory from each dataset case. */
|
|
322
|
+
declare function trajectoryExpectationScorer(options?: TrajectoryScorerOptions): EvalScorer;
|
|
323
|
+
type ApprovalExpectation = Extract<EvalTrajectoryExpectation, {
|
|
324
|
+
type: 'approval';
|
|
325
|
+
}>;
|
|
326
|
+
type ElicitationExpectation = Extract<EvalTrajectoryExpectation, {
|
|
327
|
+
type: 'elicitation';
|
|
328
|
+
}>;
|
|
329
|
+
type PolicyExpectation = Extract<EvalTrajectoryExpectation, {
|
|
330
|
+
type: 'policy';
|
|
331
|
+
}>;
|
|
332
|
+
declare function approvalScorer(expected: readonly ApprovalExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
|
|
333
|
+
declare function elicitationScorer(expected: readonly ElicitationExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
|
|
334
|
+
declare function policyScorer(expected: readonly PolicyExpectation[], options?: TrajectoryScorerOptions): EvalScorer;
|
|
335
|
+
interface RiskGuardrailScorerOptions {
|
|
336
|
+
/** Risks that must never execute. Defaults to `destructive`. */
|
|
337
|
+
deny?: readonly ToolRisk[];
|
|
338
|
+
/** Risks that must have an approved approval before execution. Defaults to `sensitive`. */
|
|
339
|
+
requireApproval?: readonly ToolRisk[];
|
|
340
|
+
/** Expected handling of tools that omit `risk`. Defaults to `allow`. */
|
|
341
|
+
undeclaredRisk?: 'allow' | 'require_approval' | 'deny';
|
|
342
|
+
}
|
|
343
|
+
/** Asserts that risky tools respected the configured guardrail. */
|
|
344
|
+
declare function riskGuardrailScorer(options?: RiskGuardrailScorerOptions, id?: string): EvalScorer;
|
|
152
345
|
//#endregion
|
|
153
|
-
export { EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalExpectation, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, Regression, RegressionOptions, RegressionReport, RunEvaluationOptions, compareEvaluationReports, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer };
|
|
346
|
+
export { EvalApproval, EvalCase, EvalCaseReport, EvalCostOptions, EvalDataset, EvalElicitation, EvalExpectation, EvalGateCheck, EvalGateThresholds, EvalReport, EvalReporter, EvalScore, EvalScoreStatus, EvalScorecard, EvalScorer, EvalScorerContext, EvalScorerResult, EvalSummary, EvalTrajectoryEntry, EvalTrajectoryExpectation, Regression, RegressionOptions, RegressionReport, ReleaseGateError, ReleaseGateOptions, ReleaseGateResult, RiskGuardrailScorerOptions, RunEvaluationOptions, TrajectoryScorerOptions, approvalScorer, assertReleaseGate, compareEvaluationReports, elicitationScorer, evaluateReleaseGate, exactOutputScorer, forbiddenToolsScorer, jsonReporter, maxCostScorer, maxLatencyScorer, maxTokensScorer, parseEvalReport, policyScorer, riskGuardrailScorer, runEvaluation, serializeCanonical, serializeEvalReport, textReporter, toolSelectionScorer, trajectoryExpectationScorer, trajectoryScorer };
|