@runtypelabs/sdk 5.5.0 → 5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +998 -912
- package/dist/index.d.cts +787 -375
- package/dist/index.d.ts +787 -375
- package/dist/index.mjs +998 -912
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -2776,6 +2776,107 @@ interface paths {
|
|
|
2776
2776
|
patch?: never;
|
|
2777
2777
|
trace?: never;
|
|
2778
2778
|
};
|
|
2779
|
+
"/v1/agents/{id}/executions/{executionId}/events": {
|
|
2780
|
+
parameters: {
|
|
2781
|
+
query?: never;
|
|
2782
|
+
header?: never;
|
|
2783
|
+
path?: never;
|
|
2784
|
+
cookie?: never;
|
|
2785
|
+
};
|
|
2786
|
+
/**
|
|
2787
|
+
* Stream execution events
|
|
2788
|
+
* @description Reconnect to a durable Claude Managed agent turn: replay-and-tail its Server-Sent Events. Resolves the per-conversation session owner by (agentId, conversationId), replays buffered events with seq greater than `after`, then live-tails if the turn is still running. Each event carries an SSE `id:` line with its durable seq; reconnect after a disconnect (tab reload, sleep, stream timeout) by passing the last seen id as `after` to resume without missing or duplicating events. The vocabulary follows the same negotiation as the execution stream (`?events=`, `X-Persona-Version`, then the default-sse-event-format flag).
|
|
2789
|
+
*/
|
|
2790
|
+
get: {
|
|
2791
|
+
parameters: {
|
|
2792
|
+
query: {
|
|
2793
|
+
/** @description The conversation key that, with the agent id, resolves the durable session owner. */
|
|
2794
|
+
conversationId: string;
|
|
2795
|
+
/** @description Replay only events with seq greater than this value (the last seen SSE id). Defaults to 0. */
|
|
2796
|
+
after?: string;
|
|
2797
|
+
};
|
|
2798
|
+
header?: never;
|
|
2799
|
+
path: {
|
|
2800
|
+
id: string;
|
|
2801
|
+
executionId: string;
|
|
2802
|
+
};
|
|
2803
|
+
cookie?: never;
|
|
2804
|
+
};
|
|
2805
|
+
requestBody?: never;
|
|
2806
|
+
responses: {
|
|
2807
|
+
/** @description Server-Sent Events stream (buffered replay followed by a live tail) */
|
|
2808
|
+
200: {
|
|
2809
|
+
headers: {
|
|
2810
|
+
[name: string]: unknown;
|
|
2811
|
+
};
|
|
2812
|
+
content: {
|
|
2813
|
+
"text/event-stream": unknown;
|
|
2814
|
+
};
|
|
2815
|
+
};
|
|
2816
|
+
/** @description Invalid agent ID or missing conversationId */
|
|
2817
|
+
400: {
|
|
2818
|
+
headers: {
|
|
2819
|
+
[name: string]: unknown;
|
|
2820
|
+
};
|
|
2821
|
+
content: {
|
|
2822
|
+
"application/json": components["schemas"]["Error"];
|
|
2823
|
+
};
|
|
2824
|
+
};
|
|
2825
|
+
/** @description Unauthorized */
|
|
2826
|
+
401: {
|
|
2827
|
+
headers: {
|
|
2828
|
+
[name: string]: unknown;
|
|
2829
|
+
};
|
|
2830
|
+
content: {
|
|
2831
|
+
"application/json": components["schemas"]["Error"];
|
|
2832
|
+
};
|
|
2833
|
+
};
|
|
2834
|
+
/** @description Insufficient permissions */
|
|
2835
|
+
403: {
|
|
2836
|
+
headers: {
|
|
2837
|
+
[name: string]: unknown;
|
|
2838
|
+
};
|
|
2839
|
+
content: {
|
|
2840
|
+
"application/json": components["schemas"]["Error"];
|
|
2841
|
+
};
|
|
2842
|
+
};
|
|
2843
|
+
/** @description Agent not found */
|
|
2844
|
+
404: {
|
|
2845
|
+
headers: {
|
|
2846
|
+
[name: string]: unknown;
|
|
2847
|
+
};
|
|
2848
|
+
content: {
|
|
2849
|
+
"application/json": components["schemas"]["Error"];
|
|
2850
|
+
};
|
|
2851
|
+
};
|
|
2852
|
+
/** @description Internal server error */
|
|
2853
|
+
500: {
|
|
2854
|
+
headers: {
|
|
2855
|
+
[name: string]: unknown;
|
|
2856
|
+
};
|
|
2857
|
+
content: {
|
|
2858
|
+
"application/json": components["schemas"]["Error"];
|
|
2859
|
+
};
|
|
2860
|
+
};
|
|
2861
|
+
/** @description Failed to attach to the execution stream */
|
|
2862
|
+
502: {
|
|
2863
|
+
headers: {
|
|
2864
|
+
[name: string]: unknown;
|
|
2865
|
+
};
|
|
2866
|
+
content: {
|
|
2867
|
+
"application/json": components["schemas"]["Error"];
|
|
2868
|
+
};
|
|
2869
|
+
};
|
|
2870
|
+
};
|
|
2871
|
+
};
|
|
2872
|
+
put?: never;
|
|
2873
|
+
post?: never;
|
|
2874
|
+
delete?: never;
|
|
2875
|
+
options?: never;
|
|
2876
|
+
head?: never;
|
|
2877
|
+
patch?: never;
|
|
2878
|
+
trace?: never;
|
|
2879
|
+
};
|
|
2779
2880
|
"/v1/agents/{id}/executions/{executionId}/tool-calls": {
|
|
2780
2881
|
parameters: {
|
|
2781
2882
|
query?: never;
|
|
@@ -3380,6 +3481,189 @@ interface paths {
|
|
|
3380
3481
|
patch?: never;
|
|
3381
3482
|
trace?: never;
|
|
3382
3483
|
};
|
|
3484
|
+
"/v1/agents/{id}/runs": {
|
|
3485
|
+
parameters: {
|
|
3486
|
+
query?: never;
|
|
3487
|
+
header?: never;
|
|
3488
|
+
path?: never;
|
|
3489
|
+
cookie?: never;
|
|
3490
|
+
};
|
|
3491
|
+
/**
|
|
3492
|
+
* List managed agent runs
|
|
3493
|
+
* @description List the durable per-turn run summaries for a saved Claude Managed agent, newest first, optionally scoped to one conversation. Each row is written when a managed turn settles. Auth: agent ownership + AGENTS:READ (the same trust level as the events reconnect route).
|
|
3494
|
+
*/
|
|
3495
|
+
get: {
|
|
3496
|
+
parameters: {
|
|
3497
|
+
query?: {
|
|
3498
|
+
/** @description Scope to a single conversation key. */
|
|
3499
|
+
conversationId?: string;
|
|
3500
|
+
/** @description Max rows to return (default 50, capped at 200). */
|
|
3501
|
+
limit?: number;
|
|
3502
|
+
};
|
|
3503
|
+
header?: never;
|
|
3504
|
+
path: {
|
|
3505
|
+
id: string;
|
|
3506
|
+
};
|
|
3507
|
+
cookie?: never;
|
|
3508
|
+
};
|
|
3509
|
+
requestBody?: never;
|
|
3510
|
+
responses: {
|
|
3511
|
+
/** @description The agent run summaries */
|
|
3512
|
+
200: {
|
|
3513
|
+
headers: {
|
|
3514
|
+
[name: string]: unknown;
|
|
3515
|
+
};
|
|
3516
|
+
content: {
|
|
3517
|
+
"application/json": {
|
|
3518
|
+
runs: components["schemas"]["ManagedAgentRunSummary"][];
|
|
3519
|
+
};
|
|
3520
|
+
};
|
|
3521
|
+
};
|
|
3522
|
+
/** @description Invalid agent ID */
|
|
3523
|
+
400: {
|
|
3524
|
+
headers: {
|
|
3525
|
+
[name: string]: unknown;
|
|
3526
|
+
};
|
|
3527
|
+
content: {
|
|
3528
|
+
"application/json": components["schemas"]["Error"];
|
|
3529
|
+
};
|
|
3530
|
+
};
|
|
3531
|
+
/** @description Unauthorized */
|
|
3532
|
+
401: {
|
|
3533
|
+
headers: {
|
|
3534
|
+
[name: string]: unknown;
|
|
3535
|
+
};
|
|
3536
|
+
content: {
|
|
3537
|
+
"application/json": components["schemas"]["Error"];
|
|
3538
|
+
};
|
|
3539
|
+
};
|
|
3540
|
+
/** @description Insufficient permissions */
|
|
3541
|
+
403: {
|
|
3542
|
+
headers: {
|
|
3543
|
+
[name: string]: unknown;
|
|
3544
|
+
};
|
|
3545
|
+
content: {
|
|
3546
|
+
"application/json": components["schemas"]["Error"];
|
|
3547
|
+
};
|
|
3548
|
+
};
|
|
3549
|
+
/** @description Agent not found */
|
|
3550
|
+
404: {
|
|
3551
|
+
headers: {
|
|
3552
|
+
[name: string]: unknown;
|
|
3553
|
+
};
|
|
3554
|
+
content: {
|
|
3555
|
+
"application/json": components["schemas"]["Error"];
|
|
3556
|
+
};
|
|
3557
|
+
};
|
|
3558
|
+
/** @description Internal server error */
|
|
3559
|
+
500: {
|
|
3560
|
+
headers: {
|
|
3561
|
+
[name: string]: unknown;
|
|
3562
|
+
};
|
|
3563
|
+
content: {
|
|
3564
|
+
"application/json": components["schemas"]["Error"];
|
|
3565
|
+
};
|
|
3566
|
+
};
|
|
3567
|
+
};
|
|
3568
|
+
};
|
|
3569
|
+
put?: never;
|
|
3570
|
+
post?: never;
|
|
3571
|
+
delete?: never;
|
|
3572
|
+
options?: never;
|
|
3573
|
+
head?: never;
|
|
3574
|
+
patch?: never;
|
|
3575
|
+
trace?: never;
|
|
3576
|
+
};
|
|
3577
|
+
"/v1/agents/{id}/runs/{runId}": {
|
|
3578
|
+
parameters: {
|
|
3579
|
+
query?: never;
|
|
3580
|
+
header?: never;
|
|
3581
|
+
path?: never;
|
|
3582
|
+
cookie?: never;
|
|
3583
|
+
};
|
|
3584
|
+
/**
|
|
3585
|
+
* Get a managed agent run
|
|
3586
|
+
* @description Fetch one durable run summary plus its deliverable pointers (the swept `/mnt/session/outputs/` files; the bytes live in asset storage). Auth: agent ownership + AGENTS:READ.
|
|
3587
|
+
*/
|
|
3588
|
+
get: {
|
|
3589
|
+
parameters: {
|
|
3590
|
+
query?: never;
|
|
3591
|
+
header?: never;
|
|
3592
|
+
path: {
|
|
3593
|
+
id: string;
|
|
3594
|
+
runId: string;
|
|
3595
|
+
};
|
|
3596
|
+
cookie?: never;
|
|
3597
|
+
};
|
|
3598
|
+
requestBody?: never;
|
|
3599
|
+
responses: {
|
|
3600
|
+
/** @description The run summary and its deliverables */
|
|
3601
|
+
200: {
|
|
3602
|
+
headers: {
|
|
3603
|
+
[name: string]: unknown;
|
|
3604
|
+
};
|
|
3605
|
+
content: {
|
|
3606
|
+
"application/json": {
|
|
3607
|
+
outputs: components["schemas"]["ManagedAgentRunOutput"][];
|
|
3608
|
+
run: components["schemas"]["ManagedAgentRunSummary"];
|
|
3609
|
+
};
|
|
3610
|
+
};
|
|
3611
|
+
};
|
|
3612
|
+
/** @description Invalid agent ID */
|
|
3613
|
+
400: {
|
|
3614
|
+
headers: {
|
|
3615
|
+
[name: string]: unknown;
|
|
3616
|
+
};
|
|
3617
|
+
content: {
|
|
3618
|
+
"application/json": components["schemas"]["Error"];
|
|
3619
|
+
};
|
|
3620
|
+
};
|
|
3621
|
+
/** @description Unauthorized */
|
|
3622
|
+
401: {
|
|
3623
|
+
headers: {
|
|
3624
|
+
[name: string]: unknown;
|
|
3625
|
+
};
|
|
3626
|
+
content: {
|
|
3627
|
+
"application/json": components["schemas"]["Error"];
|
|
3628
|
+
};
|
|
3629
|
+
};
|
|
3630
|
+
/** @description Insufficient permissions */
|
|
3631
|
+
403: {
|
|
3632
|
+
headers: {
|
|
3633
|
+
[name: string]: unknown;
|
|
3634
|
+
};
|
|
3635
|
+
content: {
|
|
3636
|
+
"application/json": components["schemas"]["Error"];
|
|
3637
|
+
};
|
|
3638
|
+
};
|
|
3639
|
+
/** @description Agent or run not found */
|
|
3640
|
+
404: {
|
|
3641
|
+
headers: {
|
|
3642
|
+
[name: string]: unknown;
|
|
3643
|
+
};
|
|
3644
|
+
content: {
|
|
3645
|
+
"application/json": components["schemas"]["Error"];
|
|
3646
|
+
};
|
|
3647
|
+
};
|
|
3648
|
+
/** @description Internal server error */
|
|
3649
|
+
500: {
|
|
3650
|
+
headers: {
|
|
3651
|
+
[name: string]: unknown;
|
|
3652
|
+
};
|
|
3653
|
+
content: {
|
|
3654
|
+
"application/json": components["schemas"]["Error"];
|
|
3655
|
+
};
|
|
3656
|
+
};
|
|
3657
|
+
};
|
|
3658
|
+
};
|
|
3659
|
+
put?: never;
|
|
3660
|
+
post?: never;
|
|
3661
|
+
delete?: never;
|
|
3662
|
+
options?: never;
|
|
3663
|
+
head?: never;
|
|
3664
|
+
patch?: never;
|
|
3665
|
+
trace?: never;
|
|
3666
|
+
};
|
|
3383
3667
|
"/v1/agents/{id}/voice": {
|
|
3384
3668
|
parameters: {
|
|
3385
3669
|
query?: never;
|
|
@@ -9727,6 +10011,8 @@ interface paths {
|
|
|
9727
10011
|
};
|
|
9728
10012
|
virtual?: boolean;
|
|
9729
10013
|
};
|
|
10014
|
+
/** @description Strict mode (the `runtype eval --strict` gate): a soft grader miss fails its case like a gate miss. Default false — soft misses are reported per-outcome but do not fail the suite. */
|
|
10015
|
+
strict?: boolean;
|
|
9730
10016
|
suiteId?: string;
|
|
9731
10017
|
};
|
|
9732
10018
|
};
|
|
@@ -36715,7 +37001,9 @@ interface components {
|
|
|
36715
37001
|
/** @enum {string} */
|
|
36716
37002
|
type: "approval_complete";
|
|
36717
37003
|
} | {
|
|
37004
|
+
awaitReason?: string;
|
|
36718
37005
|
awaitedAt?: string;
|
|
37006
|
+
crawlId?: string;
|
|
36719
37007
|
executionId: string;
|
|
36720
37008
|
/** @enum {string} */
|
|
36721
37009
|
origin?: "webmcp" | "sdk";
|
|
@@ -36724,6 +37012,7 @@ interface components {
|
|
|
36724
37012
|
[key: string]: unknown;
|
|
36725
37013
|
};
|
|
36726
37014
|
seq: number;
|
|
37015
|
+
stepId?: string;
|
|
36727
37016
|
toolCallId?: string;
|
|
36728
37017
|
toolId?: string;
|
|
36729
37018
|
toolName?: string;
|
|
@@ -36883,7 +37172,9 @@ interface components {
|
|
|
36883
37172
|
type: "flow_error";
|
|
36884
37173
|
upgradeUrl?: string;
|
|
36885
37174
|
} | {
|
|
37175
|
+
awaitReason?: string;
|
|
36886
37176
|
awaitedAt: string;
|
|
37177
|
+
crawlId?: string;
|
|
36887
37178
|
executionId?: string;
|
|
36888
37179
|
flowId: string;
|
|
36889
37180
|
/** @enum {string} */
|
|
@@ -36893,6 +37184,7 @@ interface components {
|
|
|
36893
37184
|
[key: string]: unknown;
|
|
36894
37185
|
};
|
|
36895
37186
|
seq?: number;
|
|
37187
|
+
stepId?: string;
|
|
36896
37188
|
toolCallId?: string;
|
|
36897
37189
|
toolId?: string;
|
|
36898
37190
|
toolName?: string;
|
|
@@ -37184,6 +37476,39 @@ interface components {
|
|
|
37184
37476
|
valid: boolean;
|
|
37185
37477
|
warnings: components["schemas"]["FlowValidationIssue"][];
|
|
37186
37478
|
};
|
|
37479
|
+
ManagedAgentRunOutput: {
|
|
37480
|
+
anthropicFileId: string;
|
|
37481
|
+
/** @description Asset-storage key the bytes were persisted to. */
|
|
37482
|
+
assetRef: string | null;
|
|
37483
|
+
createdAt: string;
|
|
37484
|
+
filename: string;
|
|
37485
|
+
/** @description Deliverable pointer id (typeid, `mao_…`). */
|
|
37486
|
+
id: string;
|
|
37487
|
+
mimeType: string;
|
|
37488
|
+
sizeBytes: number;
|
|
37489
|
+
};
|
|
37490
|
+
ManagedAgentRunSummary: {
|
|
37491
|
+
agentId: string;
|
|
37492
|
+
conversationId: string;
|
|
37493
|
+
/** @description Cost in thousandths of a cent ($1 = 100,000). */
|
|
37494
|
+
costThousandths: number;
|
|
37495
|
+
createdAt: string;
|
|
37496
|
+
error: string | null;
|
|
37497
|
+
/** @description The DO-minted execution id for the turn. */
|
|
37498
|
+
executionId: string;
|
|
37499
|
+
/** @description The final assistant text (may be large or null). */
|
|
37500
|
+
finalText: string | null;
|
|
37501
|
+
/** @description Run id (typeid, `mar_…`). */
|
|
37502
|
+
id: string;
|
|
37503
|
+
inputTokens: number;
|
|
37504
|
+
/** @description The durable seq at settle (reconnect ceiling). */
|
|
37505
|
+
lastEventSeq: number;
|
|
37506
|
+
outputTokens: number;
|
|
37507
|
+
settledAt: string;
|
|
37508
|
+
startedAt: string;
|
|
37509
|
+
/** @description 'completed' | 'failed' | 'interrupted'. */
|
|
37510
|
+
status: string;
|
|
37511
|
+
};
|
|
37187
37512
|
Pagination: {
|
|
37188
37513
|
currentOffset: number;
|
|
37189
37514
|
currentPage?: number;
|
|
@@ -37329,6 +37654,11 @@ interface components {
|
|
|
37329
37654
|
passed: boolean;
|
|
37330
37655
|
reasoning?: string;
|
|
37331
37656
|
score?: number;
|
|
37657
|
+
/**
|
|
37658
|
+
* @description Grader severity. A soft miss is reported but only fails the suite under strict mode. Absent ⇒ gate.
|
|
37659
|
+
* @enum {string}
|
|
37660
|
+
*/
|
|
37661
|
+
severity?: "gate" | "soft";
|
|
37332
37662
|
}[];
|
|
37333
37663
|
outputExcerpt: string;
|
|
37334
37664
|
passed: boolean;
|
|
@@ -39925,6 +40255,414 @@ interface BillingSpendAnalyticsParams {
|
|
|
39925
40255
|
days?: number;
|
|
39926
40256
|
}
|
|
39927
40257
|
|
|
40258
|
+
/**
|
|
40259
|
+
* Eval config-as-code: `defineEval` + the grader builders.
|
|
40260
|
+
*
|
|
40261
|
+
* The authoring layer for code-colocated evals — define the evals for a flow or
|
|
40262
|
+
* agent right next to its `defineFlow` / `flows.ensure` definition. This module
|
|
40263
|
+
* is PURE and local (no I/O), the exact analog of `defineFlow` in
|
|
40264
|
+
* `flows-ensure.ts`: it validates and normalizes a loose `DefineEvalInput` into
|
|
40265
|
+
* a canonical `EvalDefinition` (target + cases + per-case graders) and computes
|
|
40266
|
+
* a content hash for hash-first convergence. The converge motion
|
|
40267
|
+
* (`client.evals.ensure` → `POST /eval/ensure`) and the `runtype eval` CLI build
|
|
40268
|
+
* on this contract in later increments.
|
|
40269
|
+
*
|
|
40270
|
+
* Grader types are MIRRORED INLINE from `@runtypelabs/shared`'s
|
|
40271
|
+
* `grader-types.ts` (the SDK is dependency-free by convention — see the same
|
|
40272
|
+
* pattern in `flows-ensure.ts`). The wire shapes must stay byte-identical to the
|
|
40273
|
+
* shared discriminated union so an eval authored here scores through the existing
|
|
40274
|
+
* `EvalScoringService` unchanged.
|
|
40275
|
+
*
|
|
40276
|
+
* Scope: the output + AI-judge union plus the trace graders (`called_tool` /
|
|
40277
|
+
* `tool_order` / `ran_step` / `completed` / `cost` / …), each scored server-side
|
|
40278
|
+
* by the same pure `runCheck` engine over the run's captured execution trace.
|
|
40279
|
+
* Severity (`.gate()` / `.soft()`) is deliberately NOT emitted here — it lands
|
|
40280
|
+
* with its own grader-engine extension. See
|
|
40281
|
+
* `docs/features/planning/2026-06-24-code-colocated-evals.md`.
|
|
40282
|
+
*/
|
|
40283
|
+
|
|
40284
|
+
/**
|
|
40285
|
+
* Per-grader severity (mirror of `@runtypelabs/shared`'s `GraderSeverity`). A
|
|
40286
|
+
* `gate` miss always fails the case; a `soft` miss is tracked-but-not-failing
|
|
40287
|
+
* unless the run is `--strict`. Absent ⇒ `gate`. Set it with the chainable
|
|
40288
|
+
* `.gate()` / `.soft()` handles on any grader builder.
|
|
40289
|
+
*/
|
|
40290
|
+
type GraderSeverity = 'gate' | 'soft';
|
|
40291
|
+
/** The deterministic check kinds (before severity is mixed in). */
|
|
40292
|
+
type CheckGraderShape = {
|
|
40293
|
+
kind: 'contains';
|
|
40294
|
+
value: string;
|
|
40295
|
+
caseSensitive?: boolean;
|
|
40296
|
+
} | {
|
|
40297
|
+
kind: 'not_contains';
|
|
40298
|
+
value: string;
|
|
40299
|
+
caseSensitive?: boolean;
|
|
40300
|
+
} | {
|
|
40301
|
+
kind: 'matches_expected';
|
|
40302
|
+
} | {
|
|
40303
|
+
kind: 'regex';
|
|
40304
|
+
pattern: string;
|
|
40305
|
+
flags?: string;
|
|
40306
|
+
} | {
|
|
40307
|
+
kind: 'valid_json';
|
|
40308
|
+
} | {
|
|
40309
|
+
kind: 'json_field';
|
|
40310
|
+
path: string;
|
|
40311
|
+
equals?: unknown;
|
|
40312
|
+
exists?: boolean;
|
|
40313
|
+
} | {
|
|
40314
|
+
kind: 'length';
|
|
40315
|
+
minChars?: number;
|
|
40316
|
+
maxChars?: number;
|
|
40317
|
+
} | {
|
|
40318
|
+
kind: 'latency';
|
|
40319
|
+
maxMs: number;
|
|
40320
|
+
} | {
|
|
40321
|
+
kind: 'no_error';
|
|
40322
|
+
} | {
|
|
40323
|
+
kind: 'called_tool';
|
|
40324
|
+
name: string;
|
|
40325
|
+
input?: unknown;
|
|
40326
|
+
output?: unknown;
|
|
40327
|
+
isError?: boolean;
|
|
40328
|
+
times?: number;
|
|
40329
|
+
} | {
|
|
40330
|
+
kind: 'not_called_tool';
|
|
40331
|
+
name: string;
|
|
40332
|
+
} | {
|
|
40333
|
+
kind: 'used_no_tools';
|
|
40334
|
+
} | {
|
|
40335
|
+
kind: 'max_tool_calls';
|
|
40336
|
+
max: number;
|
|
40337
|
+
} | {
|
|
40338
|
+
kind: 'tool_order';
|
|
40339
|
+
tools: string[];
|
|
40340
|
+
} | {
|
|
40341
|
+
kind: 'ran_step';
|
|
40342
|
+
name: string;
|
|
40343
|
+
} | {
|
|
40344
|
+
kind: 'step_order';
|
|
40345
|
+
steps: string[];
|
|
40346
|
+
} | {
|
|
40347
|
+
kind: 'completed';
|
|
40348
|
+
} | {
|
|
40349
|
+
kind: 'cost';
|
|
40350
|
+
maxUsd: number;
|
|
40351
|
+
};
|
|
40352
|
+
/** Deterministic, free, instant checks. Scored by the pure `runCheck` engine. */
|
|
40353
|
+
type CheckGrader = CheckGraderShape & {
|
|
40354
|
+
severity?: GraderSeverity;
|
|
40355
|
+
};
|
|
40356
|
+
/** Built-in AI-grader preset ids (mirror of `BUILT_IN_GRADER_IDS`). */
|
|
40357
|
+
type BuiltInGraderId = 'answersQuestion' | 'matchesExpected' | 'followsInstructions' | 'grounded' | 'rightTone' | 'safeToSend';
|
|
40358
|
+
/** LLM-as-judge over plain-language criteria. Scored by the api grader executor. */
|
|
40359
|
+
interface AIGrader {
|
|
40360
|
+
kind: 'ai';
|
|
40361
|
+
preset?: BuiltInGraderId;
|
|
40362
|
+
/** Plain language: "what does a good answer look like?" */
|
|
40363
|
+
criteria: string;
|
|
40364
|
+
/** Reference-guided when the case has `expected`. */
|
|
40365
|
+
useExpected?: boolean;
|
|
40366
|
+
/** Defaults to a cheap routed model at execution time. */
|
|
40367
|
+
model?: string;
|
|
40368
|
+
/** Pass cutoff on the 1-5 judge scale (default 4 server-side). Set via `.atLeast(n)`. */
|
|
40369
|
+
threshold?: number;
|
|
40370
|
+
/** Hard gate (default) vs soft. Set via `.gate()` / `.soft()`. */
|
|
40371
|
+
severity?: GraderSeverity;
|
|
40372
|
+
}
|
|
40373
|
+
type GraderConfig = CheckGrader | AIGrader;
|
|
40374
|
+
/** What a good answer looks like for a case (mirror of `CaseExpected`). */
|
|
40375
|
+
interface CaseExpected {
|
|
40376
|
+
text?: string;
|
|
40377
|
+
json?: unknown;
|
|
40378
|
+
facts?: string[];
|
|
40379
|
+
}
|
|
40380
|
+
/**
|
|
40381
|
+
* A grader plus chainable severity handles. The handles are defined
|
|
40382
|
+
* NON-ENUMERABLY, so `JSON.stringify`, `Object.keys`, and `computeEvalContentHash`
|
|
40383
|
+
* see only the data fields — a `Gradeable` serializes byte-identically to the
|
|
40384
|
+
* plain `GraderConfig` it wraps. `.atLeast(n)` is present only on AI graders.
|
|
40385
|
+
*/
|
|
40386
|
+
type Gradeable<T extends GraderConfig = GraderConfig> = T & {
|
|
40387
|
+
/** Mark as a hard gate (the default): a miss fails the case. */
|
|
40388
|
+
gate(): Gradeable<T>;
|
|
40389
|
+
/** Mark as soft: a miss is tracked but only fails the case under `--strict`. */
|
|
40390
|
+
soft(): Gradeable<T>;
|
|
40391
|
+
} & (T extends AIGrader ? {
|
|
40392
|
+
/** AI graders only: set the 1-5 judge pass cutoff (e.g. `judge(...).atLeast(3)`). */
|
|
40393
|
+
atLeast(threshold: number): Gradeable<T>;
|
|
40394
|
+
} : Record<never, never>);
|
|
40395
|
+
/** Output contains `value` (case-insensitive unless `caseSensitive`). */
|
|
40396
|
+
declare function contains(value: string, opts?: {
|
|
40397
|
+
caseSensitive?: boolean;
|
|
40398
|
+
}): Gradeable<CheckGrader>;
|
|
40399
|
+
/** Output does NOT contain `value`. */
|
|
40400
|
+
declare function notContains(value: string, opts?: {
|
|
40401
|
+
caseSensitive?: boolean;
|
|
40402
|
+
}): Gradeable<CheckGrader>;
|
|
40403
|
+
/** Output equals the case's `expected.text` (trim/lowercase/collapse-whitespace normalized). */
|
|
40404
|
+
declare function matchesExpected(): Gradeable<CheckGrader>;
|
|
40405
|
+
/** Output matches a regular expression. */
|
|
40406
|
+
declare function regex(pattern: string, flags?: string): Gradeable<CheckGrader>;
|
|
40407
|
+
/** Output parses as JSON. */
|
|
40408
|
+
declare function validJson(): Gradeable<CheckGrader>;
|
|
40409
|
+
/**
|
|
40410
|
+
* A dot-path field in the output's parsed JSON. With `equals`, asserts value
|
|
40411
|
+
* equality; otherwise asserts presence (`exists: false` asserts absence).
|
|
40412
|
+
*/
|
|
40413
|
+
declare function jsonField(path: string, opts?: {
|
|
40414
|
+
equals?: unknown;
|
|
40415
|
+
exists?: boolean;
|
|
40416
|
+
}): Gradeable<CheckGrader>;
|
|
40417
|
+
/** Output character length is within `[minChars, maxChars]`. */
|
|
40418
|
+
declare function length(opts: {
|
|
40419
|
+
minChars?: number;
|
|
40420
|
+
maxChars?: number;
|
|
40421
|
+
}): Gradeable<CheckGrader>;
|
|
40422
|
+
/** End-to-end latency is within `maxMs`. */
|
|
40423
|
+
declare function latency(maxMs: number): Gradeable<CheckGrader>;
|
|
40424
|
+
/** The case produced output without erroring. */
|
|
40425
|
+
declare function noError(): Gradeable<CheckGrader>;
|
|
40426
|
+
/**
|
|
40427
|
+
* A tool named `name` was called. Optional filters narrow the match: `input` /
|
|
40428
|
+
* `output` deep-equal a call's resolved input / result, `isError` matches a
|
|
40429
|
+
* call's error flag, and `times` asserts the matching count EXACTLY (omit for
|
|
40430
|
+
* "at least once").
|
|
40431
|
+
*/
|
|
40432
|
+
declare function calledTool(name: string, opts?: {
|
|
40433
|
+
input?: unknown;
|
|
40434
|
+
output?: unknown;
|
|
40435
|
+
isError?: boolean;
|
|
40436
|
+
times?: number;
|
|
40437
|
+
}): Gradeable<CheckGrader>;
|
|
40438
|
+
/** No tool named `name` was called. */
|
|
40439
|
+
declare function notCalledTool(name: string): Gradeable<CheckGrader>;
|
|
40440
|
+
/** The run made no tool calls at all. */
|
|
40441
|
+
declare function usedNoTools(): Gradeable<CheckGrader>;
|
|
40442
|
+
/** The run made at most `max` tool calls. */
|
|
40443
|
+
declare function maxToolCalls(max: number): Gradeable<CheckGrader>;
|
|
40444
|
+
/** `tools` appears as an ordered subsequence of the tool-call names. */
|
|
40445
|
+
declare function toolOrder(tools: string[]): Gradeable<CheckGrader>;
|
|
40446
|
+
/** A step named (or typed) `name` ran. */
|
|
40447
|
+
declare function ranStep(name: string): Gradeable<CheckGrader>;
|
|
40448
|
+
/** `steps` appears as an ordered subsequence of the steps that ran. */
|
|
40449
|
+
declare function stepOrder(steps: string[]): Gradeable<CheckGrader>;
|
|
40450
|
+
/** The run completed (finished without erroring and was not left paused). */
|
|
40451
|
+
declare function completed(): Gradeable<CheckGrader>;
|
|
40452
|
+
/** Total run cost was within `maxUsd` (US dollars). */
|
|
40453
|
+
declare function cost(maxUsd: number): Gradeable<CheckGrader>;
|
|
40454
|
+
/**
|
|
40455
|
+
* LLM-as-judge over free-form criteria. Soft, model-graded — reach for it only
|
|
40456
|
+
* when no deterministic check captures what "good" means. Chain `.atLeast(n)`
|
|
40457
|
+
* to set the 1-5 pass cutoff and `.soft()` to make a miss non-blocking.
|
|
40458
|
+
*/
|
|
40459
|
+
declare function judge(criteria: string, opts?: {
|
|
40460
|
+
useExpected?: boolean;
|
|
40461
|
+
model?: string;
|
|
40462
|
+
threshold?: number;
|
|
40463
|
+
preset?: BuiltInGraderId;
|
|
40464
|
+
}): Gradeable<AIGrader>;
|
|
40465
|
+
/**
|
|
40466
|
+
* The built-in AI-grader presets (criteria mirrored from `BUILT_IN_GRADERS`).
|
|
40467
|
+
* Each returns a ready `AIGrader`; pass an override string to `rightTone`.
|
|
40468
|
+
*/
|
|
40469
|
+
declare const judges: {
|
|
40470
|
+
readonly answersQuestion: () => Gradeable<AIGrader>;
|
|
40471
|
+
readonly matchesExpected: () => Gradeable<AIGrader>;
|
|
40472
|
+
readonly followsInstructions: () => Gradeable<AIGrader>;
|
|
40473
|
+
readonly grounded: () => Gradeable<AIGrader>;
|
|
40474
|
+
readonly rightTone: (voice?: string) => Gradeable<AIGrader>;
|
|
40475
|
+
readonly safeToSend: () => Gradeable<AIGrader>;
|
|
40476
|
+
};
|
|
40477
|
+
interface EvalMessage {
|
|
40478
|
+
role: 'user' | 'assistant' | 'system';
|
|
40479
|
+
content: string;
|
|
40480
|
+
}
|
|
40481
|
+
/** A case's input: flow variables and/or a scripted conversation to replay. */
|
|
40482
|
+
interface EvalCaseInput {
|
|
40483
|
+
variables?: Record<string, unknown>;
|
|
40484
|
+
messages?: EvalMessage[];
|
|
40485
|
+
}
|
|
40486
|
+
/** The target a suite evaluates — a saved flow or agent, by portable name. */
|
|
40487
|
+
type EvalTarget = {
|
|
40488
|
+
flow: string;
|
|
40489
|
+
} | {
|
|
40490
|
+
agent: string;
|
|
40491
|
+
};
|
|
40492
|
+
/** Loose per-case input to `defineEval`. */
|
|
40493
|
+
interface DefineEvalCaseInput {
|
|
40494
|
+
name: string;
|
|
40495
|
+
input?: EvalCaseInput;
|
|
40496
|
+
expected?: CaseExpected;
|
|
40497
|
+
/** Case-level graders, appended after any suite-level `graders`. */
|
|
40498
|
+
expect?: GraderConfig[];
|
|
40499
|
+
}
|
|
40500
|
+
/** Loose input to `defineEval` (validated + normalized into `EvalDefinition`). */
|
|
40501
|
+
interface DefineEvalInput {
|
|
40502
|
+
/**
|
|
40503
|
+
* Suite name — the converge identity (name + account scope), exactly like a
|
|
40504
|
+
* flow's name. Optional: defaults to a stable name derived from the target
|
|
40505
|
+
* (`flow:<name>` / `agent:<name>`). Give two suites for the same target
|
|
40506
|
+
* distinct names (e.g. 'smoke', 'regression') to keep them separate.
|
|
40507
|
+
*/
|
|
40508
|
+
name?: string;
|
|
40509
|
+
target: EvalTarget;
|
|
40510
|
+
/** Graders applied to EVERY case (suite-level). Run before each case's `expect`. */
|
|
40511
|
+
graders?: GraderConfig[];
|
|
40512
|
+
cases: DefineEvalCaseInput[];
|
|
40513
|
+
/**
|
|
40514
|
+
* Run without persisting a durable suite/batch to the dashboard (the 'virtual'
|
|
40515
|
+
* opt-out, mirroring `useVirtualFlow` / `storeResults:false`). Default false:
|
|
40516
|
+
* evals run via this surface are dashboard-visible by default.
|
|
40517
|
+
*/
|
|
40518
|
+
virtual?: boolean;
|
|
40519
|
+
}
|
|
40520
|
+
/** A normalized case: graders merged (suite-level then case-level) into `expect`. */
|
|
40521
|
+
interface EvalCaseDefinition {
|
|
40522
|
+
name: string;
|
|
40523
|
+
input: EvalCaseInput;
|
|
40524
|
+
expected?: CaseExpected;
|
|
40525
|
+
expect: GraderConfig[];
|
|
40526
|
+
}
|
|
40527
|
+
/** The canonical (wire) eval definition produced by `defineEval`. */
|
|
40528
|
+
interface EvalDefinition {
|
|
40529
|
+
/** Suite name — the converge identity. Derived from the target when omitted. */
|
|
40530
|
+
name: string;
|
|
40531
|
+
target: EvalTarget;
|
|
40532
|
+
cases: EvalCaseDefinition[];
|
|
40533
|
+
virtual: boolean;
|
|
40534
|
+
}
|
|
40535
|
+
/**
|
|
40536
|
+
* Pure-local declarative constructor for an eval definition. No I/O. Validates
|
|
40537
|
+
* structure, rejects unknown fields, merges suite-level `graders` into each
|
|
40538
|
+
* case's `expect`, and produces a canonical, environment-portable
|
|
40539
|
+
* `EvalDefinition`. The target names a saved flow/agent by name (portable across
|
|
40540
|
+
* environments, like `defineFlow`'s `flow:<name>` references).
|
|
40541
|
+
*
|
|
40542
|
+
* @example
|
|
40543
|
+
* ```typescript
|
|
40544
|
+
* import { defineEval, contains, judges } from '@runtypelabs/sdk'
|
|
40545
|
+
*
|
|
40546
|
+
* export default defineEval({
|
|
40547
|
+
* target: { flow: 'support-triage' },
|
|
40548
|
+
* graders: [contains('ticket')],
|
|
40549
|
+
* cases: [
|
|
40550
|
+
* {
|
|
40551
|
+
* name: 'billing routes to finance',
|
|
40552
|
+
* input: { variables: { message: 'I was double charged' } },
|
|
40553
|
+
* expect: [contains('finance'), judges.answersQuestion()],
|
|
40554
|
+
* },
|
|
40555
|
+
* ],
|
|
40556
|
+
* })
|
|
40557
|
+
* ```
|
|
40558
|
+
*/
|
|
40559
|
+
declare function defineEval(input: DefineEvalInput): EvalDefinition;
|
|
40560
|
+
/**
|
|
40561
|
+
* SHA-256 (hex) over the canonical normalized definition. Cases are sorted by
|
|
40562
|
+
* name (name is the case identity); grader order WITHIN a case is preserved
|
|
40563
|
+
* (reordering graders changes their result index, so it is a meaningful edit).
|
|
40564
|
+
* The hash is the basis for hash-first convergence in `client.evals.ensure`.
|
|
40565
|
+
*/
|
|
40566
|
+
declare function computeEvalContentHash(definition: EvalDefinition): Promise<string>;
|
|
40567
|
+
/** The converge outcome of `client.evals.ensure(definition)`. */
|
|
40568
|
+
interface EnsureEvalResult {
|
|
40569
|
+
result: 'unchanged' | 'created' | 'updated';
|
|
40570
|
+
/** The persisted eval suite id. */
|
|
40571
|
+
suiteId: string;
|
|
40572
|
+
/** The server-computed canonical hash (echo this — never your own). */
|
|
40573
|
+
contentHash: string;
|
|
40574
|
+
}
|
|
40575
|
+
/** The canonical definition + provenance returned by `client.evals.pull(name)`. */
|
|
40576
|
+
interface EvalPullResult {
|
|
40577
|
+
suiteId: string;
|
|
40578
|
+
definition: EvalDefinition;
|
|
40579
|
+
contentHash: string;
|
|
40580
|
+
lastModifiedSource: string | null;
|
|
40581
|
+
updatedAt: string | null;
|
|
40582
|
+
}
|
|
40583
|
+
/**
|
|
40584
|
+
* One grader's verdict for one case (mirror of `@runtypelabs/shared`'s
|
|
40585
|
+
* `GraderOutcome`). `graderIndex` is the position in the suite's grader list.
|
|
40586
|
+
*/
|
|
40587
|
+
interface GraderOutcome {
|
|
40588
|
+
graderIndex: number;
|
|
40589
|
+
kind: string;
|
|
40590
|
+
passed: boolean;
|
|
40591
|
+
/** The grader's severity (absent ⇒ gate). A soft miss only fails under `--strict`. */
|
|
40592
|
+
severity?: GraderSeverity;
|
|
40593
|
+
/** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). */
|
|
40594
|
+
score?: number;
|
|
40595
|
+
/** AI-grader verdict, or a check's human-readable reason. */
|
|
40596
|
+
reasoning?: string;
|
|
40597
|
+
}
|
|
40598
|
+
/** One case's run result: pass/fail plus each grader's outcome. */
|
|
40599
|
+
interface RunEvalCaseResult {
|
|
40600
|
+
name: string;
|
|
40601
|
+
passed: boolean;
|
|
40602
|
+
outcomes: GraderOutcome[];
|
|
40603
|
+
/** Truncated final-output snapshot (for surfacing a failing case in CI logs). */
|
|
40604
|
+
outputExcerpt: string;
|
|
40605
|
+
/** Whether producing the output threw. */
|
|
40606
|
+
errored: boolean;
|
|
40607
|
+
}
|
|
40608
|
+
/** The synchronous run + score result returned by `client.evals.runSuite(...)`. */
|
|
40609
|
+
interface RunEvalResult {
|
|
40610
|
+
/** The saved suite id, or `null` for an inline (virtual) run. */
|
|
40611
|
+
suiteId: string | null;
|
|
40612
|
+
name: string;
|
|
40613
|
+
targetType: 'flow' | 'agent';
|
|
40614
|
+
/** Suite score, 0..1 (passed cases / total cases). */
|
|
40615
|
+
score: number;
|
|
40616
|
+
/** True when every case passed every grader. */
|
|
40617
|
+
passed: boolean;
|
|
40618
|
+
totalCases: number;
|
|
40619
|
+
passedCases: number;
|
|
40620
|
+
cases: RunEvalCaseResult[];
|
|
40621
|
+
}
|
|
40622
|
+
/**
|
|
40623
|
+
* Run a saved suite by id (the post-`ensure` path) XOR an inline definition (the
|
|
40624
|
+
* `virtual` path — nothing is persisted). Exactly one must be provided.
|
|
40625
|
+
*
|
|
40626
|
+
* `strict` (default false) is the `runtype eval --strict` gate: a `soft` grader
|
|
40627
|
+
* miss fails its case like a `gate` miss. Without it, soft misses are reported
|
|
40628
|
+
* per-outcome but do not fail the suite.
|
|
40629
|
+
*/
|
|
40630
|
+
type RunEvalInput = {
|
|
40631
|
+
suiteId: string;
|
|
40632
|
+
strict?: boolean;
|
|
40633
|
+
} | {
|
|
40634
|
+
definition: EvalDefinition;
|
|
40635
|
+
strict?: boolean;
|
|
40636
|
+
};
|
|
40637
|
+
/**
|
|
40638
|
+
* Idempotently converge an eval suite definition onto the platform. Hash-first:
|
|
40639
|
+
* probes with a content hash, and only ships the full definition when the
|
|
40640
|
+
* server reports a miss (`definitionRequired`). Upserts the suite + replaces
|
|
40641
|
+
* its cases; never executes the eval (use the `runtype eval` CLI / `/eval`
|
|
40642
|
+
* submit surface to run it).
|
|
40643
|
+
*
|
|
40644
|
+
* `virtual: true` definitions are ephemeral and have nothing durable to
|
|
40645
|
+
* converge — ensure rejects them. Run a virtual eval directly instead.
|
|
40646
|
+
*/
|
|
40647
|
+
declare function ensureEval(client: RuntypeClient$1, definition: EvalDefinition): Promise<EnsureEvalResult>;
|
|
40648
|
+
/**
|
|
40649
|
+
* Pull the canonical definition + provenance for an eval suite by name — the
|
|
40650
|
+
* absorb-drift direction of the ensure protocol. The contentHash reflects the
|
|
40651
|
+
* live suite state.
|
|
40652
|
+
*/
|
|
40653
|
+
declare function pullEval(client: RuntypeClient$1, name: string): Promise<EvalPullResult>;
|
|
40654
|
+
/**
|
|
40655
|
+
* Run an eval suite synchronously and return the suite score + per-case grader
|
|
40656
|
+
* outcomes. Powers the `runtype eval` CI gate: a saved suite is run by id (after
|
|
40657
|
+
* `ensure`); a `virtual` definition is run inline without persisting anything.
|
|
40658
|
+
*
|
|
40659
|
+
* Synchronous and ephemeral — no batch is created and no scores are saved (use
|
|
40660
|
+
* the dashboard / `/eval/submit` for a durable, dashboard-visible run). Bounded:
|
|
40661
|
+
* suites over the server's per-run case limit must use the batch path. Not
|
|
40662
|
+
* supported here: `claude_managed` agents and inline/virtual agent targets.
|
|
40663
|
+
*/
|
|
40664
|
+
declare function runEvalSuite(client: RuntypeClient$1, input: RunEvalInput): Promise<RunEvalResult>;
|
|
40665
|
+
|
|
39928
40666
|
/**
|
|
39929
40667
|
* Flow config-as-code: `defineFlow`, `flows.ensure`, `flows.pull`.
|
|
39930
40668
|
*
|
|
@@ -39953,6 +40691,13 @@ interface BillingSpendAnalyticsParams {
|
|
|
39953
40691
|
* same flow. The flow definition surface is `{ name, steps }` — description
|
|
39954
40692
|
* is not part of the v1 ensure surface (the shared hash covers steps only).
|
|
39955
40693
|
*
|
|
40694
|
+
* A `defineFlow` may also carry inline `evals` — eval suites to converge
|
|
40695
|
+
* alongside the flow. These are SDK-orchestrated: they are NOT part of the
|
|
40696
|
+
* flow content hash and NEVER ride the `/flows/ensure` wire (its server schema
|
|
40697
|
+
* is `.strict()` `{ name, steps }`). After the flow converges, `ensureFlow`
|
|
40698
|
+
* converges each inline suite through the existing `/eval/ensure` endpoint
|
|
40699
|
+
* (`ensureEval`), so eval semantics stay confined to the eval endpoints.
|
|
40700
|
+
*
|
|
39956
40701
|
* See docs/adr/0003-agent-config-as-code-ensure.md for the design rationale.
|
|
39957
40702
|
*/
|
|
39958
40703
|
|
|
@@ -39972,10 +40717,26 @@ interface FlowDefinitionStep {
|
|
|
39972
40717
|
when?: string;
|
|
39973
40718
|
config?: Record<string, unknown>;
|
|
39974
40719
|
}
|
|
39975
|
-
/**
|
|
40720
|
+
/**
|
|
40721
|
+
* An inline eval suite attached to a `defineFlow`. It is a `DefineEvalInput`
|
|
40722
|
+
* whose `target` is optional: when omitted it defaults to `{ flow: <this
|
|
40723
|
+
* flow's name> }` (the enclosing flow), so the common "evals for THIS flow"
|
|
40724
|
+
* case needs no target. Each is normalized by the canonical `defineEval`.
|
|
40725
|
+
*/
|
|
40726
|
+
type FlowInlineEvalInput = Omit<DefineEvalInput, 'target'> & {
|
|
40727
|
+
target?: DefineEvalInput['target'];
|
|
40728
|
+
};
|
|
40729
|
+
/** `defineFlow` input: identity + the ordered step list (+ optional inline evals). */
|
|
39976
40730
|
interface DefineFlowInput {
|
|
39977
40731
|
name: string;
|
|
39978
40732
|
steps: FlowDefinitionStep[];
|
|
40733
|
+
/**
|
|
40734
|
+
* Eval suites to converge alongside this flow. Each is normalized through
|
|
40735
|
+
* the canonical `defineEval`; a suite with no `target` defaults to this flow.
|
|
40736
|
+
* NOT part of the flow content hash and NOT sent on the `/flows/ensure` wire
|
|
40737
|
+
* — `flows.ensure` converges these separately via `/eval/ensure`.
|
|
40738
|
+
*/
|
|
40739
|
+
evals?: FlowInlineEvalInput[];
|
|
39979
40740
|
}
|
|
39980
40741
|
/** The canonical (wire) definition produced by `defineFlow`. */
|
|
39981
40742
|
interface FlowDefinition {
|
|
@@ -39983,6 +40744,12 @@ interface FlowDefinition {
|
|
|
39983
40744
|
steps: Array<FlowDefinitionStep & {
|
|
39984
40745
|
order: number;
|
|
39985
40746
|
}>;
|
|
40747
|
+
/**
|
|
40748
|
+
* Normalized inline eval suites (a NON-wire field). Present only when the
|
|
40749
|
+
* definition declared at least one. `ensureFlow` strips this before sending
|
|
40750
|
+
* `{ name, steps }` to `/flows/ensure`, then converges each via `/eval/ensure`.
|
|
40751
|
+
*/
|
|
40752
|
+
evals?: EvalDefinition[];
|
|
39986
40753
|
}
|
|
39987
40754
|
/**
|
|
39988
40755
|
* Pure-local declarative constructor for a flow definition. No I/O.
|
|
@@ -40032,6 +40799,14 @@ interface EnsureFlowConverged {
|
|
|
40032
40799
|
versionId: string | null;
|
|
40033
40800
|
/** The server-computed canonical hash (echo this — never your own). */
|
|
40034
40801
|
contentHash: string;
|
|
40802
|
+
/**
|
|
40803
|
+
* Per-suite converge outcomes for the definition's inline `evals`, in
|
|
40804
|
+
* declaration order. Present only when the definition declared inline evals
|
|
40805
|
+
* AND the flow converged on the real (non-plan) path — each suite converges
|
|
40806
|
+
* via `/eval/ensure` after the flow itself. Absent when there are no inline
|
|
40807
|
+
* evals (and never populated on the dryRun/plan path).
|
|
40808
|
+
*/
|
|
40809
|
+
evals?: EnsureEvalResult[];
|
|
40035
40810
|
}
|
|
40036
40811
|
interface EnsureFlowPlan {
|
|
40037
40812
|
result: 'plan';
|
|
@@ -40161,11 +40936,19 @@ declare class FlowsNamespace {
|
|
|
40161
40936
|
* the steady state is one tiny probe request. Creates an immutable version
|
|
40162
40937
|
* snapshot on every change; never deletes; never executes the flow.
|
|
40163
40938
|
*
|
|
40939
|
+
* When the definition carries inline `evals`, each suite is converged via
|
|
40940
|
+
* `/eval/ensure` after the flow itself (real converge path only — not on
|
|
40941
|
+
* dryRun/`expectNoChanges`), and the outcomes are returned as `result.evals`.
|
|
40942
|
+
*
|
|
40164
40943
|
* @example
|
|
40165
40944
|
* ```typescript
|
|
40166
|
-
* const def = defineFlow({
|
|
40945
|
+
* const def = defineFlow({
|
|
40946
|
+
* name: 'Onboarding Digest',
|
|
40947
|
+
* steps: [...],
|
|
40948
|
+
* evals: [{ cases: [{ name: 'smoke', input: {...}, expect: [contains('ok')] }] }],
|
|
40949
|
+
* })
|
|
40167
40950
|
*
|
|
40168
|
-
* // Converge (CI/deploy).
|
|
40951
|
+
* // Converge the flow AND its inline eval suites (CI/deploy).
|
|
40169
40952
|
* const result = await Runtype.flows.ensure(def)
|
|
40170
40953
|
*
|
|
40171
40954
|
* // PR drift gate.
|
|
@@ -40574,377 +41357,6 @@ declare class BatchesNamespace {
|
|
|
40574
41357
|
}>;
|
|
40575
41358
|
}
|
|
40576
41359
|
|
|
40577
|
-
/**
|
|
40578
|
-
* Eval config-as-code: `defineEval` + the grader builders.
|
|
40579
|
-
*
|
|
40580
|
-
* The authoring layer for code-colocated evals — define the evals for a flow or
|
|
40581
|
-
* agent right next to its `defineFlow` / `flows.ensure` definition. This module
|
|
40582
|
-
* is PURE and local (no I/O), the exact analog of `defineFlow` in
|
|
40583
|
-
* `flows-ensure.ts`: it validates and normalizes a loose `DefineEvalInput` into
|
|
40584
|
-
* a canonical `EvalDefinition` (target + cases + per-case graders) and computes
|
|
40585
|
-
* a content hash for hash-first convergence. The converge motion
|
|
40586
|
-
* (`client.evals.ensure` → `POST /eval/ensure`) and the `runtype eval` CLI build
|
|
40587
|
-
* on this contract in later increments.
|
|
40588
|
-
*
|
|
40589
|
-
* Grader types are MIRRORED INLINE from `@runtypelabs/shared`'s
|
|
40590
|
-
* `grader-types.ts` (the SDK is dependency-free by convention — see the same
|
|
40591
|
-
* pattern in `flows-ensure.ts`). The wire shapes must stay byte-identical to the
|
|
40592
|
-
* shared discriminated union so an eval authored here scores through the existing
|
|
40593
|
-
* `EvalScoringService` unchanged.
|
|
40594
|
-
*
|
|
40595
|
-
* Scope: the output + AI-judge union plus the trace graders (`called_tool` /
|
|
40596
|
-
* `tool_order` / `ran_step` / `completed` / `cost` / …), each scored server-side
|
|
40597
|
-
* by the same pure `runCheck` engine over the run's captured execution trace.
|
|
40598
|
-
* Severity (`.gate()` / `.soft()`) is deliberately NOT emitted here — it lands
|
|
40599
|
-
* with its own grader-engine extension. See
|
|
40600
|
-
* `docs/features/planning/2026-06-24-code-colocated-evals.md`.
|
|
40601
|
-
*/
|
|
40602
|
-
|
|
40603
|
-
/** Deterministic, free, instant checks. Scored by the pure `runCheck` engine. */
|
|
40604
|
-
type CheckGrader = {
|
|
40605
|
-
kind: 'contains';
|
|
40606
|
-
value: string;
|
|
40607
|
-
caseSensitive?: boolean;
|
|
40608
|
-
} | {
|
|
40609
|
-
kind: 'not_contains';
|
|
40610
|
-
value: string;
|
|
40611
|
-
caseSensitive?: boolean;
|
|
40612
|
-
} | {
|
|
40613
|
-
kind: 'matches_expected';
|
|
40614
|
-
} | {
|
|
40615
|
-
kind: 'regex';
|
|
40616
|
-
pattern: string;
|
|
40617
|
-
flags?: string;
|
|
40618
|
-
} | {
|
|
40619
|
-
kind: 'valid_json';
|
|
40620
|
-
} | {
|
|
40621
|
-
kind: 'json_field';
|
|
40622
|
-
path: string;
|
|
40623
|
-
equals?: unknown;
|
|
40624
|
-
exists?: boolean;
|
|
40625
|
-
} | {
|
|
40626
|
-
kind: 'length';
|
|
40627
|
-
minChars?: number;
|
|
40628
|
-
maxChars?: number;
|
|
40629
|
-
} | {
|
|
40630
|
-
kind: 'latency';
|
|
40631
|
-
maxMs: number;
|
|
40632
|
-
} | {
|
|
40633
|
-
kind: 'no_error';
|
|
40634
|
-
} | {
|
|
40635
|
-
kind: 'called_tool';
|
|
40636
|
-
name: string;
|
|
40637
|
-
input?: unknown;
|
|
40638
|
-
output?: unknown;
|
|
40639
|
-
isError?: boolean;
|
|
40640
|
-
times?: number;
|
|
40641
|
-
} | {
|
|
40642
|
-
kind: 'not_called_tool';
|
|
40643
|
-
name: string;
|
|
40644
|
-
} | {
|
|
40645
|
-
kind: 'used_no_tools';
|
|
40646
|
-
} | {
|
|
40647
|
-
kind: 'max_tool_calls';
|
|
40648
|
-
max: number;
|
|
40649
|
-
} | {
|
|
40650
|
-
kind: 'tool_order';
|
|
40651
|
-
tools: string[];
|
|
40652
|
-
} | {
|
|
40653
|
-
kind: 'ran_step';
|
|
40654
|
-
name: string;
|
|
40655
|
-
} | {
|
|
40656
|
-
kind: 'step_order';
|
|
40657
|
-
steps: string[];
|
|
40658
|
-
} | {
|
|
40659
|
-
kind: 'completed';
|
|
40660
|
-
} | {
|
|
40661
|
-
kind: 'cost';
|
|
40662
|
-
maxUsd: number;
|
|
40663
|
-
};
|
|
40664
|
-
/** Built-in AI-grader preset ids (mirror of `BUILT_IN_GRADER_IDS`). */
|
|
40665
|
-
type BuiltInGraderId = 'answersQuestion' | 'matchesExpected' | 'followsInstructions' | 'grounded' | 'rightTone' | 'safeToSend';
|
|
40666
|
-
/** LLM-as-judge over plain-language criteria. Scored by the api grader executor. */
|
|
40667
|
-
interface AIGrader {
|
|
40668
|
-
kind: 'ai';
|
|
40669
|
-
preset?: BuiltInGraderId;
|
|
40670
|
-
/** Plain language: "what does a good answer look like?" */
|
|
40671
|
-
criteria: string;
|
|
40672
|
-
/** Reference-guided when the case has `expected`. */
|
|
40673
|
-
useExpected?: boolean;
|
|
40674
|
-
/** Defaults to a cheap routed model at execution time. */
|
|
40675
|
-
model?: string;
|
|
40676
|
-
/** Pass cutoff on the 1-5 judge scale (default 4 server-side). */
|
|
40677
|
-
threshold?: number;
|
|
40678
|
-
}
|
|
40679
|
-
type GraderConfig = CheckGrader | AIGrader;
|
|
40680
|
-
/** What a good answer looks like for a case (mirror of `CaseExpected`). */
|
|
40681
|
-
interface CaseExpected {
|
|
40682
|
-
text?: string;
|
|
40683
|
-
json?: unknown;
|
|
40684
|
-
facts?: string[];
|
|
40685
|
-
}
|
|
40686
|
-
/** Output contains `value` (case-insensitive unless `caseSensitive`). */
|
|
40687
|
-
declare function contains(value: string, opts?: {
|
|
40688
|
-
caseSensitive?: boolean;
|
|
40689
|
-
}): CheckGrader;
|
|
40690
|
-
/** Output does NOT contain `value`. */
|
|
40691
|
-
declare function notContains(value: string, opts?: {
|
|
40692
|
-
caseSensitive?: boolean;
|
|
40693
|
-
}): CheckGrader;
|
|
40694
|
-
/** Output equals the case's `expected.text` (trim/lowercase/collapse-whitespace normalized). */
|
|
40695
|
-
declare function matchesExpected(): CheckGrader;
|
|
40696
|
-
/** Output matches a regular expression. */
|
|
40697
|
-
declare function regex(pattern: string, flags?: string): CheckGrader;
|
|
40698
|
-
/** Output parses as JSON. */
|
|
40699
|
-
declare function validJson(): CheckGrader;
|
|
40700
|
-
/**
|
|
40701
|
-
* A dot-path field in the output's parsed JSON. With `equals`, asserts value
|
|
40702
|
-
* equality; otherwise asserts presence (`exists: false` asserts absence).
|
|
40703
|
-
*/
|
|
40704
|
-
declare function jsonField(path: string, opts?: {
|
|
40705
|
-
equals?: unknown;
|
|
40706
|
-
exists?: boolean;
|
|
40707
|
-
}): CheckGrader;
|
|
40708
|
-
/** Output character length is within `[minChars, maxChars]`. */
|
|
40709
|
-
declare function length(opts: {
|
|
40710
|
-
minChars?: number;
|
|
40711
|
-
maxChars?: number;
|
|
40712
|
-
}): CheckGrader;
|
|
40713
|
-
/** End-to-end latency is within `maxMs`. */
|
|
40714
|
-
declare function latency(maxMs: number): CheckGrader;
|
|
40715
|
-
/** The case produced output without erroring. */
|
|
40716
|
-
declare function noError(): CheckGrader;
|
|
40717
|
-
/**
|
|
40718
|
-
* A tool named `name` was called. Optional filters narrow the match: `input` /
|
|
40719
|
-
* `output` deep-equal a call's resolved input / result, `isError` matches a
|
|
40720
|
-
* call's error flag, and `times` asserts the matching count EXACTLY (omit for
|
|
40721
|
-
* "at least once").
|
|
40722
|
-
*/
|
|
40723
|
-
declare function calledTool(name: string, opts?: {
|
|
40724
|
-
input?: unknown;
|
|
40725
|
-
output?: unknown;
|
|
40726
|
-
isError?: boolean;
|
|
40727
|
-
times?: number;
|
|
40728
|
-
}): CheckGrader;
|
|
40729
|
-
/** No tool named `name` was called. */
|
|
40730
|
-
declare function notCalledTool(name: string): CheckGrader;
|
|
40731
|
-
/** The run made no tool calls at all. */
|
|
40732
|
-
declare function usedNoTools(): CheckGrader;
|
|
40733
|
-
/** The run made at most `max` tool calls. */
|
|
40734
|
-
declare function maxToolCalls(max: number): CheckGrader;
|
|
40735
|
-
/** `tools` appears as an ordered subsequence of the tool-call names. */
|
|
40736
|
-
declare function toolOrder(tools: string[]): CheckGrader;
|
|
40737
|
-
/** A step named (or typed) `name` ran. */
|
|
40738
|
-
declare function ranStep(name: string): CheckGrader;
|
|
40739
|
-
/** `steps` appears as an ordered subsequence of the steps that ran. */
|
|
40740
|
-
declare function stepOrder(steps: string[]): CheckGrader;
|
|
40741
|
-
/** The run completed (finished without erroring and was not left paused). */
|
|
40742
|
-
declare function completed(): CheckGrader;
|
|
40743
|
-
/** Total run cost was within `maxUsd` (US dollars). */
|
|
40744
|
-
declare function cost(maxUsd: number): CheckGrader;
|
|
40745
|
-
/**
|
|
40746
|
-
* LLM-as-judge over free-form criteria. Soft, model-graded — reach for it only
|
|
40747
|
-
* when no deterministic check captures what "good" means.
|
|
40748
|
-
*/
|
|
40749
|
-
declare function judge(criteria: string, opts?: {
|
|
40750
|
-
useExpected?: boolean;
|
|
40751
|
-
model?: string;
|
|
40752
|
-
threshold?: number;
|
|
40753
|
-
preset?: BuiltInGraderId;
|
|
40754
|
-
}): AIGrader;
|
|
40755
|
-
/**
|
|
40756
|
-
* The built-in AI-grader presets (criteria mirrored from `BUILT_IN_GRADERS`).
|
|
40757
|
-
* Each returns a ready `AIGrader`; pass an override string to `rightTone`.
|
|
40758
|
-
*/
|
|
40759
|
-
declare const judges: {
|
|
40760
|
-
readonly answersQuestion: () => AIGrader;
|
|
40761
|
-
readonly matchesExpected: () => AIGrader;
|
|
40762
|
-
readonly followsInstructions: () => AIGrader;
|
|
40763
|
-
readonly grounded: () => AIGrader;
|
|
40764
|
-
readonly rightTone: (voice?: string) => AIGrader;
|
|
40765
|
-
readonly safeToSend: () => AIGrader;
|
|
40766
|
-
};
|
|
40767
|
-
interface EvalMessage {
|
|
40768
|
-
role: 'user' | 'assistant' | 'system';
|
|
40769
|
-
content: string;
|
|
40770
|
-
}
|
|
40771
|
-
/** A case's input: flow variables and/or a scripted conversation to replay. */
|
|
40772
|
-
interface EvalCaseInput {
|
|
40773
|
-
variables?: Record<string, unknown>;
|
|
40774
|
-
messages?: EvalMessage[];
|
|
40775
|
-
}
|
|
40776
|
-
/** The target a suite evaluates — a saved flow or agent, by portable name. */
|
|
40777
|
-
type EvalTarget = {
|
|
40778
|
-
flow: string;
|
|
40779
|
-
} | {
|
|
40780
|
-
agent: string;
|
|
40781
|
-
};
|
|
40782
|
-
/** Loose per-case input to `defineEval`. */
|
|
40783
|
-
interface DefineEvalCaseInput {
|
|
40784
|
-
name: string;
|
|
40785
|
-
input?: EvalCaseInput;
|
|
40786
|
-
expected?: CaseExpected;
|
|
40787
|
-
/** Case-level graders, appended after any suite-level `graders`. */
|
|
40788
|
-
expect?: GraderConfig[];
|
|
40789
|
-
}
|
|
40790
|
-
/** Loose input to `defineEval` (validated + normalized into `EvalDefinition`). */
|
|
40791
|
-
interface DefineEvalInput {
|
|
40792
|
-
/**
|
|
40793
|
-
* Suite name — the converge identity (name + account scope), exactly like a
|
|
40794
|
-
* flow's name. Optional: defaults to a stable name derived from the target
|
|
40795
|
-
* (`flow:<name>` / `agent:<name>`). Give two suites for the same target
|
|
40796
|
-
* distinct names (e.g. 'smoke', 'regression') to keep them separate.
|
|
40797
|
-
*/
|
|
40798
|
-
name?: string;
|
|
40799
|
-
target: EvalTarget;
|
|
40800
|
-
/** Graders applied to EVERY case (suite-level). Run before each case's `expect`. */
|
|
40801
|
-
graders?: GraderConfig[];
|
|
40802
|
-
cases: DefineEvalCaseInput[];
|
|
40803
|
-
/**
|
|
40804
|
-
* Run without persisting a durable suite/batch to the dashboard (the 'virtual'
|
|
40805
|
-
* opt-out, mirroring `useVirtualFlow` / `storeResults:false`). Default false:
|
|
40806
|
-
* evals run via this surface are dashboard-visible by default.
|
|
40807
|
-
*/
|
|
40808
|
-
virtual?: boolean;
|
|
40809
|
-
}
|
|
40810
|
-
/** A normalized case: graders merged (suite-level then case-level) into `expect`. */
|
|
40811
|
-
interface EvalCaseDefinition {
|
|
40812
|
-
name: string;
|
|
40813
|
-
input: EvalCaseInput;
|
|
40814
|
-
expected?: CaseExpected;
|
|
40815
|
-
expect: GraderConfig[];
|
|
40816
|
-
}
|
|
40817
|
-
/** The canonical (wire) eval definition produced by `defineEval`. */
|
|
40818
|
-
interface EvalDefinition {
|
|
40819
|
-
/** Suite name — the converge identity. Derived from the target when omitted. */
|
|
40820
|
-
name: string;
|
|
40821
|
-
target: EvalTarget;
|
|
40822
|
-
cases: EvalCaseDefinition[];
|
|
40823
|
-
virtual: boolean;
|
|
40824
|
-
}
|
|
40825
|
-
/**
|
|
40826
|
-
* Pure-local declarative constructor for an eval definition. No I/O. Validates
|
|
40827
|
-
* structure, rejects unknown fields, merges suite-level `graders` into each
|
|
40828
|
-
* case's `expect`, and produces a canonical, environment-portable
|
|
40829
|
-
* `EvalDefinition`. The target names a saved flow/agent by name (portable across
|
|
40830
|
-
* environments, like `defineFlow`'s `flow:<name>` references).
|
|
40831
|
-
*
|
|
40832
|
-
* @example
|
|
40833
|
-
* ```typescript
|
|
40834
|
-
* import { defineEval, contains, judges } from '@runtypelabs/sdk'
|
|
40835
|
-
*
|
|
40836
|
-
* export default defineEval({
|
|
40837
|
-
* target: { flow: 'support-triage' },
|
|
40838
|
-
* graders: [contains('ticket')],
|
|
40839
|
-
* cases: [
|
|
40840
|
-
* {
|
|
40841
|
-
* name: 'billing routes to finance',
|
|
40842
|
-
* input: { variables: { message: 'I was double charged' } },
|
|
40843
|
-
* expect: [contains('finance'), judges.answersQuestion()],
|
|
40844
|
-
* },
|
|
40845
|
-
* ],
|
|
40846
|
-
* })
|
|
40847
|
-
* ```
|
|
40848
|
-
*/
|
|
40849
|
-
declare function defineEval(input: DefineEvalInput): EvalDefinition;
|
|
40850
|
-
/**
|
|
40851
|
-
* SHA-256 (hex) over the canonical normalized definition. Cases are sorted by
|
|
40852
|
-
* name (name is the case identity); grader order WITHIN a case is preserved
|
|
40853
|
-
* (reordering graders changes their result index, so it is a meaningful edit).
|
|
40854
|
-
* The hash is the basis for hash-first convergence in `client.evals.ensure`.
|
|
40855
|
-
*/
|
|
40856
|
-
declare function computeEvalContentHash(definition: EvalDefinition): Promise<string>;
|
|
40857
|
-
/** The converge outcome of `client.evals.ensure(definition)`. */
|
|
40858
|
-
interface EnsureEvalResult {
|
|
40859
|
-
result: 'unchanged' | 'created' | 'updated';
|
|
40860
|
-
/** The persisted eval suite id. */
|
|
40861
|
-
suiteId: string;
|
|
40862
|
-
/** The server-computed canonical hash (echo this — never your own). */
|
|
40863
|
-
contentHash: string;
|
|
40864
|
-
}
|
|
40865
|
-
/** The canonical definition + provenance returned by `client.evals.pull(name)`. */
|
|
40866
|
-
interface EvalPullResult {
|
|
40867
|
-
suiteId: string;
|
|
40868
|
-
definition: EvalDefinition;
|
|
40869
|
-
contentHash: string;
|
|
40870
|
-
lastModifiedSource: string | null;
|
|
40871
|
-
updatedAt: string | null;
|
|
40872
|
-
}
|
|
40873
|
-
/**
|
|
40874
|
-
* One grader's verdict for one case (mirror of `@runtypelabs/shared`'s
|
|
40875
|
-
* `GraderOutcome`). `graderIndex` is the position in the suite's grader list.
|
|
40876
|
-
*/
|
|
40877
|
-
interface GraderOutcome {
|
|
40878
|
-
graderIndex: number;
|
|
40879
|
-
kind: string;
|
|
40880
|
-
passed: boolean;
|
|
40881
|
-
/** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). */
|
|
40882
|
-
score?: number;
|
|
40883
|
-
/** AI-grader verdict, or a check's human-readable reason. */
|
|
40884
|
-
reasoning?: string;
|
|
40885
|
-
}
|
|
40886
|
-
/** One case's run result: pass/fail plus each grader's outcome. */
|
|
40887
|
-
interface RunEvalCaseResult {
|
|
40888
|
-
name: string;
|
|
40889
|
-
passed: boolean;
|
|
40890
|
-
outcomes: GraderOutcome[];
|
|
40891
|
-
/** Truncated final-output snapshot (for surfacing a failing case in CI logs). */
|
|
40892
|
-
outputExcerpt: string;
|
|
40893
|
-
/** Whether producing the output threw. */
|
|
40894
|
-
errored: boolean;
|
|
40895
|
-
}
|
|
40896
|
-
/** The synchronous run + score result returned by `client.evals.runSuite(...)`. */
|
|
40897
|
-
interface RunEvalResult {
|
|
40898
|
-
/** The saved suite id, or `null` for an inline (virtual) run. */
|
|
40899
|
-
suiteId: string | null;
|
|
40900
|
-
name: string;
|
|
40901
|
-
targetType: 'flow' | 'agent';
|
|
40902
|
-
/** Suite score, 0..1 (passed cases / total cases). */
|
|
40903
|
-
score: number;
|
|
40904
|
-
/** True when every case passed every grader. */
|
|
40905
|
-
passed: boolean;
|
|
40906
|
-
totalCases: number;
|
|
40907
|
-
passedCases: number;
|
|
40908
|
-
cases: RunEvalCaseResult[];
|
|
40909
|
-
}
|
|
40910
|
-
/**
|
|
40911
|
-
* Run a saved suite by id (the post-`ensure` path) XOR an inline definition (the
|
|
40912
|
-
* `virtual` path — nothing is persisted). Exactly one must be provided.
|
|
40913
|
-
*/
|
|
40914
|
-
type RunEvalInput = {
|
|
40915
|
-
suiteId: string;
|
|
40916
|
-
} | {
|
|
40917
|
-
definition: EvalDefinition;
|
|
40918
|
-
};
|
|
40919
|
-
/**
|
|
40920
|
-
* Idempotently converge an eval suite definition onto the platform. Hash-first:
|
|
40921
|
-
* probes with a content hash, and only ships the full definition when the
|
|
40922
|
-
* server reports a miss (`definitionRequired`). Upserts the suite + replaces
|
|
40923
|
-
* its cases; never executes the eval (use the `runtype eval` CLI / `/eval`
|
|
40924
|
-
* submit surface to run it).
|
|
40925
|
-
*
|
|
40926
|
-
* `virtual: true` definitions are ephemeral and have nothing durable to
|
|
40927
|
-
* converge — ensure rejects them. Run a virtual eval directly instead.
|
|
40928
|
-
*/
|
|
40929
|
-
declare function ensureEval(client: RuntypeClient$1, definition: EvalDefinition): Promise<EnsureEvalResult>;
|
|
40930
|
-
/**
|
|
40931
|
-
* Pull the canonical definition + provenance for an eval suite by name — the
|
|
40932
|
-
* absorb-drift direction of the ensure protocol. The contentHash reflects the
|
|
40933
|
-
* live suite state.
|
|
40934
|
-
*/
|
|
40935
|
-
declare function pullEval(client: RuntypeClient$1, name: string): Promise<EvalPullResult>;
|
|
40936
|
-
/**
|
|
40937
|
-
* Run an eval suite synchronously and return the suite score + per-case grader
|
|
40938
|
-
* outcomes. Powers the `runtype eval` CI gate: a saved suite is run by id (after
|
|
40939
|
-
* `ensure`); a `virtual` definition is run inline without persisting anything.
|
|
40940
|
-
*
|
|
40941
|
-
* Synchronous and ephemeral — no batch is created and no scores are saved (use
|
|
40942
|
-
* the dashboard / `/eval/submit` for a durable, dashboard-visible run). Bounded:
|
|
40943
|
-
* suites over the server's per-run case limit must use the batch path. Not
|
|
40944
|
-
* supported here: `claude_managed` agents and inline/virtual agent targets.
|
|
40945
|
-
*/
|
|
40946
|
-
declare function runEvalSuite(client: RuntypeClient$1, input: RunEvalInput): Promise<RunEvalResult>;
|
|
40947
|
-
|
|
40948
41360
|
/**
|
|
40949
41361
|
* EvalsNamespace - Static namespace for evaluation operations
|
|
40950
41362
|
*
|
|
@@ -47802,4 +48214,4 @@ declare function getLikelySupportingCandidatePaths(bestCandidatePath: string | u
|
|
|
47802
48214
|
declare function getDefaultPlanPath(taskName: string): string;
|
|
47803
48215
|
declare function sanitizeTaskSlug(taskName: string): string;
|
|
47804
48216
|
|
|
47805
|
-
export { type AIGrader, type Agent, type AgentApprovalCompleteEvent, type AgentApprovalStartEvent, type AgentCompleteEvent, type AgentDefinition, type AgentDefinitionConfig, AgentDriftError, AgentEnsureConflictError, type AgentErrorEvent, type AgentEvent, type AgentEventType, type AgentExecuteRequest, type AgentExecuteResponse, type AgentIterationCompleteEvent, type AgentIterationStartEvent, type AgentMediaEvent, type AgentMessage, type AgentPausedEvent, type AgentPingEvent, type AgentPullResult, type AgentReflectionEvent, type AgentRuntimeToolDefinition, type AgentStartEvent, type AgentStreamCallbacks, type AgentStreamEvent, type AgentSubagentConfig, type AgentToolCompleteEvent, type AgentToolDeltaEvent, type AgentToolInputCompleteEvent, type AgentToolInputDeltaEvent, type AgentToolStartEvent, type AgentTurnCompleteEvent, type AgentTurnDeltaEvent, type AgentTurnStartEvent, type AgentVersionDetail, type AgentVersionListItem, type AgentVersionPublishResponse, AgentVersionsEndpoint, type AgentVersionsListResponse, AgentsEndpoint, AgentsNamespace, AnalyticsEndpoint, type ApiClient, type ApiKey, ApiKeysEndpoint, type ApiResponse, type App, type AppManifest, type AppVersion, type ApplyGeneratedProposalOptions, type ApplyGeneratedProposalResult, AppsEndpoint, type AssetReferenceContentPart, type AttachRuntimeToolsOptions, type BaseAgentEvent, BatchBuilder, type BatchClient, type BatchListParams, type BatchOptions, type BatchRequest, type BatchResult, type BatchScheduleConfig, type BatchStatus, BatchesNamespace, BillingEndpoint, type BillingSpendAnalyticsParams, type BindSkillInput, type BuiltInGraderId, type BuiltInTool, type BulkEditCondition, type BulkEditRequest, type BulkEditResponse, type BulkEditResult, type CaseExpected, ChatEndpoint, type CheckGrader, ClientBatchBuilder, type ClientConfig, type ClientConversation, ClientEvalBuilder, ClientFlowBuilder, type ClientToken, type ClientTokenConfig, type ClientTokenEnvironment, type ClientTokenVersionPin, ClientTokensEndpoint, type ClientToolDefinition, type ClientWidgetTheme, type ConditionalGetResult, type ConditionalStepConfig$1 as ConditionalStepConfig, type ContextErrorHandling, type ContextFallback, ContextTemplatesEndpoint, type Conversation, type ConversationListItem, type ConversationListParams, type ConversationMessage, type ConversationSource, ConversationsEndpoint, type ConversationsListResponse, type CreateApiKeyRequest, type CreateAppRequest, type CreateClientTokenRequest, type CreateClientTokenResponse, type CreateConversationRequest, type CreateFlowRequest, type CreateModelConfigRequest, type CreatePromptData, type CreatePromptRequest, type CreateProviderKeyRequest, type CreateRecordRequest, type CreateScheduleRequest, type CreateSecretRequest, type CreateToolRequest, type CustomMCPServer, type CustomMCPServerAuth, type CustomToolConfig, DEFAULT_RECOVERY_AFTER_EMPTY_SESSIONS, DEFAULT_STALL_STOP_AFTER, type DefineAgentInput, type DefineEvalCaseInput, type DefineEvalInput, type DefineFlowInput, type DefineProductInput, type DefineSkillInput, type DefineSurfaceInput, type DefineToolInput, type DeployCfSandboxRequest, type DeployCfSandboxResponse, type DeploySandboxRequest, type DeploySandboxResponse, type DiscoveredModel, type DispatchClient, DispatchEndpoint, type DispatchEnvironment, type DispatchEvent, type DispatchOptions$1 as DispatchOptions, type DispatchRequest, type EnsureAgentConverged, type EnsureAgentOptions, type EnsureAgentPlan, type EnsureAgentResult, type EnsureEvalResult, type EnsureFlowConverged, type EnsureFlowOptions, type EnsureFlowPlan, type EnsureFlowResult, type EnsureFpoOptions, type EnsureFpoResult, type EnsureProductConverged, type EnsureProductOptions, type EnsureProductPlan, type EnsureProductResult, type EnsureSkillConverged, type EnsureSkillOptions, type EnsureSkillPlan, type EnsureSkillResult, type EnsureSurfaceConverged, type EnsureSurfaceOptions, type EnsureSurfacePlan, type EnsureSurfaceResult, type EnsureToolConverged, type EnsureToolOptions, type EnsureToolPlan, type EnsureToolResult, type ErrorHandlingMode, EvalBuilder, type EvalCaseDefinition, type EvalCaseInput, type EvalClient, type EvalDefinition, EvalEndpoint, type EvalListParams, type EvalMessage, type EvalOptions, type EvalPullResult, type EvalRecord, type EvalRequest, type EvalResult, type EvalRunConfig, EvalRunner, type EvalStatus, type EvalTarget, EvalsNamespace, type ExecuteToolRequest, type ExecuteToolResponse, type ExecutionStreamEvent, type ExternalAgentContext, type ExternalToolConfig, type FallbackFailEvent, type FallbackStartEvent, type FallbackSuccessEvent, type FallbackTrigger, type FallbackTriggerType, type FallbacksExhaustedEvent, type FallbacksInitiatedEvent, type FetchGitHubStepConfig$1 as FetchGitHubStepConfig, type FetchUrlStepConfig$1 as FetchUrlStepConfig, type FieldFormat, type FileContentPart, type Flow, type FlowAttachment, FlowBuilder, type FlowCompleteEvent, type FlowConfig$1 as FlowConfig, type FlowDefinition, type FlowDefinitionStep, FlowDriftError, FlowEnsureConflictError, type FlowErrorEvent, type FlowFallback, type FlowListItem, type FlowPausedEvent, type FlowPullResult, FlowResult, type FlowStartEvent, type FlowStep, type FlowStepDefinition, type FlowStepType, FlowStepsEndpoint, type FlowStreamEvent, type FlowSummary, type FlowToolConfig, type FlowValidationClient, type FlowValidationIssue, type FlowValidationResult, type FlowVersionDetail, type FlowVersionListItem, type FlowVersionPublishResponse, FlowVersionsEndpoint, type FlowVersionsListResponse, FlowsEndpoint, FlowsNamespace, type FpoEntityOutcome, type FpoInput, type GenerateEmbeddingStepConfig$1 as GenerateEmbeddingStepConfig, type GeneratedRuntimeToolGateDecision, type GeneratedRuntimeToolGateOptions, type GraderConfig, type GraderOutcome, type ImageContentPart, type Integration, type IntegrationTool, IntegrationsEndpoint, type IntegrationsListResponse, type JSONSchema, type JsonArray, type JsonObject, type JsonPrimitive, type JsonValue, LEDGER_ARTIFACT_LINE_PREFIX, type ListConversationsResponse, type ListParams, type LocalToolConfig, type LocalToolDefinition, type LocalToolExecutionCompleteEvent, type LocalToolExecutionLoopSnapshotSlice, type LocalToolExecutionStartEvent, type LogEntry, type LogQueryParams, type LogQueryResponse, type LogQueryResult, type LogStatsParams, type LogStatsResponse, type LogStatsResult, LogsEndpoint, type Message$1 as Message, type MessageContent, type MessageFallback, type Metadata, type ModelConfig, ModelConfigsEndpoint, type ModelFallback, type ModelOverride, type ModelUsageDetail, type ModelUsageQueryParams, type ModelUsageResponse, type ModelUsageSummary, type ModelUsageTimeSeries, type PaginationResponse, type ProductDefinition, ProductDriftError, ProductEnsureConflictError, type ProductPullResult, ProductsNamespace, type Prompt$1 as Prompt, type PromptErrorHandling, type PromptFallback, type PromptListParams, type PromptStepConfig$1 as PromptStepConfig, PromptsEndpoint, PromptsNamespace, type ProviderApiKey, type ProviderKeyModel, ProviderKeysEndpoint, type PullFpoResult, RUNTYPE_CLIENT_KIND, type ReasoningConfig, type ReasoningContentPart, type ReasoningValue, type RecordConfig$1 as RecordConfig, type RecordCostAggregation, type RecordCostModelBreakdown, type RecordFilter, type RecordFilterCondition, type RecordFilterGroup, type RecordFilterOperator, type RecordListItem, type RecordListParams, type RecordStepResult, type RecordStepResultsParams, type RecordStepResultsResponse, type RecordWriteResponse, RecordsEndpoint, type RetrieveRecordStepConfig$1 as RetrieveRecordStepConfig, type RetryFallback, type RunEvalCaseResult, type RunEvalInput, type RunEvalResult, type RunTaskContextBudgetBreakdown, type RunTaskContextCompactionEvent, type RunTaskContextCompactionStrategy, type RunTaskContextNoticeEvent, type RunTaskContextSummaryEntry, type RunTaskContinuation, type RunTaskOffloadRecorder, type RunTaskOnContextCompaction, type RunTaskOnContextNotice, type RunTaskOnSession, type RunTaskOptions, type RunTaskResult, type RunTaskResumeState, type RunTaskSessionSummary, type RunTaskState, type RunTaskStateSlice, type RunTaskStatus, type RunTaskToolTraceSlice, type RuntimeCustomToolConfig, type RuntimeExternalToolConfig, type RuntimeFlowToolConfig, type RuntimeLocalToolConfig, type RuntimeSubagentToolConfig, type RuntimeTool, type RuntimeToolConfig, Runtype, type AgentSkillBinding as RuntypeAgentSkillBinding, RuntypeApiError, RuntypeClient, type ConditionalStepConfig as RuntypeConditionalStepConfig, type RuntypeConfig, type FetchGitHubStepConfig as RuntypeFetchGitHubStepConfig, type FetchUrlStepConfig as RuntypeFetchUrlStepConfig, RuntypeFlowBuilder, type FlowConfig as RuntypeFlowConfig, type GenerateEmbeddingStepConfig as RuntypeGenerateEmbeddingStepConfig, type Message as RuntypeMessage, type ModelOverride$1 as RuntypeModelOverride, type Prompt as RuntypePrompt, type PromptStepConfig as RuntypePromptStepConfig, type RuntypeRecord, type RecordConfig as RuntypeRecordConfig, type RetrieveRecordStepConfig as RuntypeRetrieveRecordStepConfig, type SearchStepConfig as RuntypeSearchStepConfig, type SendEmailStepConfig as RuntypeSendEmailStepConfig, type SendEventStepConfig as RuntypeSendEventStepConfig, type SendStreamStepConfig as RuntypeSendStreamStepConfig, type SendTextStepConfig as RuntypeSendTextStepConfig, type SetVariableStepConfig as RuntypeSetVariableStepConfig, type Skill as RuntypeSkill, type SkillCapabilities as RuntypeSkillCapabilities, type SkillFrontmatter as RuntypeSkillFrontmatter, type SkillManifest as RuntypeSkillManifest, type SkillProposal as RuntypeSkillProposal, type SkillRuntypeExtensions as RuntypeSkillRuntypeExtensions, type SkillScanFinding as RuntypeSkillScanFinding, type SkillScanResult as RuntypeSkillScanResult, type SkillScanVerdict as RuntypeSkillScanVerdict, type SkillVersion as RuntypeSkillVersion, type TransformDataStepConfig as RuntypeTransformDataStepConfig, type UpsertFlowConfig as RuntypeUpsertFlowConfig, type UpsertRecordStepConfig as RuntypeUpsertRecordStepConfig, type VectorSearchStepConfig as RuntypeVectorSearchStepConfig, type WaitUntilStepConfig as RuntypeWaitUntilStepConfig, SDK_USER_AGENT, SDK_VERSION, STEP_FIELD_REGISTRY, STEP_TYPE_TO_METHOD, type Schedule, type ScheduleExecutionOptions, type ScheduleListParams, type ScheduleMessage, type ScheduleMessageSet, type ScheduleMessages, type ScheduleMutationResponse, type ScheduleRun, type ScheduleRunNowResponse, type ScheduleStatusResponse, type ScheduleTarget, type ScheduleTrigger, SchedulesEndpoint, type SearchStepConfig$1 as SearchStepConfig, type Secret, type SecretCheckResponse, type SecretDeleteResponse, type SecretSetupUrlRequest, type SecretSetupUrlResponse, SecretsEndpoint, type SendEmailStepConfig$1 as SendEmailStepConfig, type SendEventStepConfig$1 as SendEventStepConfig, type SendStreamStepConfig$1 as SendStreamStepConfig, type SendTextStepConfig$1 as SendTextStepConfig, type SetVariableStepConfig$1 as SetVariableStepConfig, type SkillDefinition, SkillDriftError, SkillEnsureConflictError, type SkillListPage, type SkillListPagination, type SkillListParams, type SkillManifestInput, type SkillMarkdownInput, type SkillOrigin, type SkillProposalStatus, SkillProposalsNamespace, type SkillPullResult, type SkillStatus, type SkillTrustLevel, type SkillVersionStatus, type SkillWithVersion, type SkillWriteInput, SkillsNamespace, type SlackInstallRequest, type StepCompleteEvent, type StepDeltaEvent, type StepFallback, type StepFieldMeta, type StepStartEvent, type StepWaitingLocalEvent, type StreamCallbacks, type StreamConsumeOptions, type StreamEvent, type StreamEventOf, type SubagentToolConfig, type Surface, type SurfaceDefinition, type SurfaceDefinitionEnvironment, type SurfaceDefinitionStatus, type SurfaceDefinitionType, SurfaceDriftError, SurfaceEnsureConflictError, type SurfaceListParams, type SurfacePullResult, SurfacesEndpoint, SurfacesNamespace, type TextContentPart, type Tool, type ToolApprovalGrant, ToolApprovalGrantsEndpoint, type ToolConfig, type ToolDefinition, type ToolDefinitionType, ToolDriftError, ToolEnsureConflictError, type ToolPullResult, type ToolWithValidation, type ToolsConfig, ToolsEndpoint, ToolsNamespace, type TransformDataStepConfig$1 as TransformDataStepConfig, UNIFIED_EVENTS_QUERY, type UpdateAppRequest, type UpdateClientTokenRequest, type UpdateConversationRequest, type UpdatePromptData, type UpdateProviderKeyRequest, type UpdateScheduleRequest, type UpdateSecretRequest, type UpdateToolRequest, type UpdatedFlow, type UpsertFlowConfig$1 as UpsertFlowConfig, type UpsertOptions, type UpsertRecordStepConfig$1 as UpsertRecordStepConfig, type UserProfile, UsersEndpoint, type VectorSearchStepConfig$1 as VectorSearchStepConfig, type VersionType, type WaitUntilStepConfig$1 as WaitUntilStepConfig, type WorkflowCompileDeps, type WorkflowCompletionCriteriaConfig, type WorkflowConfig, type WorkflowConfigFactory, type WorkflowContext, type WorkflowDefinition, type WorkflowHookEntry, type WorkflowHookKind, type WorkflowHookRef, type WorkflowHookSignatures, type WorkflowMilestoneConfig, type WorkflowPhase, type WorkflowPolicyConfig, type WorkflowRecoveryConfig, type WorkflowSlot, type WorkflowStallPolicy, applyGeneratedRuntimeToolProposalToDispatchRequest, attachRuntimeToolsToDispatchRequest, buildEmptySessionNudge, buildGeneratedRuntimeToolGateOutput, buildLedgerOffloadReference, buildPolicyGuidance, buildSendViewOffloadMarker, calledTool, compileWorkflowConfig, completed, computeAgentContentHash, computeEvalContentHash, computeFlowContentHash, computeFpoContentHash, computeProductContentHash, computeSkillContentHash, computeSurfaceContentHash, computeToolContentHash, contains, cost, createAgentEventTranslator, createClient, createExternalTool, createFlowEventTranslator, defaultWorkflow, defaultWorkflowConfig, defineAgent, defineEval, defineFlow, defineFpo, definePlaybook, defineProduct, defineSkill, defineSurface, defineTool, deployWorkflow, ensureDefaultWorkflowHooks, ensureEval, ensureFpo, evaluateGeneratedRuntimeToolProposal, extractDeclaredToolResultChars, gameWorkflow, getDefaultPlanPath, getLikelySupportingCandidatePaths, interpolateWorkflowTemplate, isDiscoveryToolName, isMarathonArtifactPath, isPreservationSensitiveTask, isUnifiedEventType, isWorkflowHookRef, jsonField, judge, judges, latency, length, listWorkflowHooks, matchesExpected, maxToolCalls, noError, normalizeAgentDefinition, normalizeCandidatePath, normalizeFpoDefinition, normalizeProductDefinition, normalizeSkillDefinition, normalizeSurfaceDefinition, normalizeToolDefinition, notCalledTool, notContains, parseFinalBuffer, parseLedgerArtifactRelativePath, parseOffloadedOutputId, parseSSEChunk, processStream, pullEval, pullFpo, ranStep, regex, registerWorkflowHook, resolveStallStopAfter, resolveWorkflowHook, runEvalSuite, sanitizeTaskSlug, shouldInjectEmptySessionNudge, shouldRequestModelEscalation, stepOrder, streamEvents, toolOrder, unregisterWorkflowHook, usedNoTools, validJson, withUnifiedEvents };
|
|
48217
|
+
export { type AIGrader, type Agent, type AgentApprovalCompleteEvent, type AgentApprovalStartEvent, type AgentCompleteEvent, type AgentDefinition, type AgentDefinitionConfig, AgentDriftError, AgentEnsureConflictError, type AgentErrorEvent, type AgentEvent, type AgentEventType, type AgentExecuteRequest, type AgentExecuteResponse, type AgentIterationCompleteEvent, type AgentIterationStartEvent, type AgentMediaEvent, type AgentMessage, type AgentPausedEvent, type AgentPingEvent, type AgentPullResult, type AgentReflectionEvent, type AgentRuntimeToolDefinition, type AgentStartEvent, type AgentStreamCallbacks, type AgentStreamEvent, type AgentSubagentConfig, type AgentToolCompleteEvent, type AgentToolDeltaEvent, type AgentToolInputCompleteEvent, type AgentToolInputDeltaEvent, type AgentToolStartEvent, type AgentTurnCompleteEvent, type AgentTurnDeltaEvent, type AgentTurnStartEvent, type AgentVersionDetail, type AgentVersionListItem, type AgentVersionPublishResponse, AgentVersionsEndpoint, type AgentVersionsListResponse, AgentsEndpoint, AgentsNamespace, AnalyticsEndpoint, type ApiClient, type ApiKey, ApiKeysEndpoint, type ApiResponse, type App, type AppManifest, type AppVersion, type ApplyGeneratedProposalOptions, type ApplyGeneratedProposalResult, AppsEndpoint, type AssetReferenceContentPart, type AttachRuntimeToolsOptions, type BaseAgentEvent, BatchBuilder, type BatchClient, type BatchListParams, type BatchOptions, type BatchRequest, type BatchResult, type BatchScheduleConfig, type BatchStatus, BatchesNamespace, BillingEndpoint, type BillingSpendAnalyticsParams, type BindSkillInput, type BuiltInGraderId, type BuiltInTool, type BulkEditCondition, type BulkEditRequest, type BulkEditResponse, type BulkEditResult, type CaseExpected, ChatEndpoint, type CheckGrader, ClientBatchBuilder, type ClientConfig, type ClientConversation, ClientEvalBuilder, ClientFlowBuilder, type ClientToken, type ClientTokenConfig, type ClientTokenEnvironment, type ClientTokenVersionPin, ClientTokensEndpoint, type ClientToolDefinition, type ClientWidgetTheme, type ConditionalGetResult, type ConditionalStepConfig$1 as ConditionalStepConfig, type ContextErrorHandling, type ContextFallback, ContextTemplatesEndpoint, type Conversation, type ConversationListItem, type ConversationListParams, type ConversationMessage, type ConversationSource, ConversationsEndpoint, type ConversationsListResponse, type CreateApiKeyRequest, type CreateAppRequest, type CreateClientTokenRequest, type CreateClientTokenResponse, type CreateConversationRequest, type CreateFlowRequest, type CreateModelConfigRequest, type CreatePromptData, type CreatePromptRequest, type CreateProviderKeyRequest, type CreateRecordRequest, type CreateScheduleRequest, type CreateSecretRequest, type CreateToolRequest, type CustomMCPServer, type CustomMCPServerAuth, type CustomToolConfig, DEFAULT_RECOVERY_AFTER_EMPTY_SESSIONS, DEFAULT_STALL_STOP_AFTER, type DefineAgentInput, type DefineEvalCaseInput, type DefineEvalInput, type DefineFlowInput, type DefineProductInput, type DefineSkillInput, type DefineSurfaceInput, type DefineToolInput, type DeployCfSandboxRequest, type DeployCfSandboxResponse, type DeploySandboxRequest, type DeploySandboxResponse, type DiscoveredModel, type DispatchClient, DispatchEndpoint, type DispatchEnvironment, type DispatchEvent, type DispatchOptions$1 as DispatchOptions, type DispatchRequest, type EnsureAgentConverged, type EnsureAgentOptions, type EnsureAgentPlan, type EnsureAgentResult, type EnsureEvalResult, type EnsureFlowConverged, type EnsureFlowOptions, type EnsureFlowPlan, type EnsureFlowResult, type EnsureFpoOptions, type EnsureFpoResult, type EnsureProductConverged, type EnsureProductOptions, type EnsureProductPlan, type EnsureProductResult, type EnsureSkillConverged, type EnsureSkillOptions, type EnsureSkillPlan, type EnsureSkillResult, type EnsureSurfaceConverged, type EnsureSurfaceOptions, type EnsureSurfacePlan, type EnsureSurfaceResult, type EnsureToolConverged, type EnsureToolOptions, type EnsureToolPlan, type EnsureToolResult, type ErrorHandlingMode, EvalBuilder, type EvalCaseDefinition, type EvalCaseInput, type EvalClient, type EvalDefinition, EvalEndpoint, type EvalListParams, type EvalMessage, type EvalOptions, type EvalPullResult, type EvalRecord, type EvalRequest, type EvalResult, type EvalRunConfig, EvalRunner, type EvalStatus, type EvalTarget, EvalsNamespace, type ExecuteToolRequest, type ExecuteToolResponse, type ExecutionStreamEvent, type ExternalAgentContext, type ExternalToolConfig, type FallbackFailEvent, type FallbackStartEvent, type FallbackSuccessEvent, type FallbackTrigger, type FallbackTriggerType, type FallbacksExhaustedEvent, type FallbacksInitiatedEvent, type FetchGitHubStepConfig$1 as FetchGitHubStepConfig, type FetchUrlStepConfig$1 as FetchUrlStepConfig, type FieldFormat, type FileContentPart, type Flow, type FlowAttachment, FlowBuilder, type FlowCompleteEvent, type FlowConfig$1 as FlowConfig, type FlowDefinition, type FlowDefinitionStep, FlowDriftError, FlowEnsureConflictError, type FlowErrorEvent, type FlowFallback, type FlowInlineEvalInput, type FlowListItem, type FlowPausedEvent, type FlowPullResult, FlowResult, type FlowStartEvent, type FlowStep, type FlowStepDefinition, type FlowStepType, FlowStepsEndpoint, type FlowStreamEvent, type FlowSummary, type FlowToolConfig, type FlowValidationClient, type FlowValidationIssue, type FlowValidationResult, type FlowVersionDetail, type FlowVersionListItem, type FlowVersionPublishResponse, FlowVersionsEndpoint, type FlowVersionsListResponse, FlowsEndpoint, FlowsNamespace, type FpoEntityOutcome, type FpoInput, type GenerateEmbeddingStepConfig$1 as GenerateEmbeddingStepConfig, type GeneratedRuntimeToolGateDecision, type GeneratedRuntimeToolGateOptions, type Gradeable, type GraderConfig, type GraderOutcome, type GraderSeverity, type ImageContentPart, type Integration, type IntegrationTool, IntegrationsEndpoint, type IntegrationsListResponse, type JSONSchema, type JsonArray, type JsonObject, type JsonPrimitive, type JsonValue, LEDGER_ARTIFACT_LINE_PREFIX, type ListConversationsResponse, type ListParams, type LocalToolConfig, type LocalToolDefinition, type LocalToolExecutionCompleteEvent, type LocalToolExecutionLoopSnapshotSlice, type LocalToolExecutionStartEvent, type LogEntry, type LogQueryParams, type LogQueryResponse, type LogQueryResult, type LogStatsParams, type LogStatsResponse, type LogStatsResult, LogsEndpoint, type Message$1 as Message, type MessageContent, type MessageFallback, type Metadata, type ModelConfig, ModelConfigsEndpoint, type ModelFallback, type ModelOverride, type ModelUsageDetail, type ModelUsageQueryParams, type ModelUsageResponse, type ModelUsageSummary, type ModelUsageTimeSeries, type PaginationResponse, type ProductDefinition, ProductDriftError, ProductEnsureConflictError, type ProductPullResult, ProductsNamespace, type Prompt$1 as Prompt, type PromptErrorHandling, type PromptFallback, type PromptListParams, type PromptStepConfig$1 as PromptStepConfig, PromptsEndpoint, PromptsNamespace, type ProviderApiKey, type ProviderKeyModel, ProviderKeysEndpoint, type PullFpoResult, RUNTYPE_CLIENT_KIND, type ReasoningConfig, type ReasoningContentPart, type ReasoningValue, type RecordConfig$1 as RecordConfig, type RecordCostAggregation, type RecordCostModelBreakdown, type RecordFilter, type RecordFilterCondition, type RecordFilterGroup, type RecordFilterOperator, type RecordListItem, type RecordListParams, type RecordStepResult, type RecordStepResultsParams, type RecordStepResultsResponse, type RecordWriteResponse, RecordsEndpoint, type RetrieveRecordStepConfig$1 as RetrieveRecordStepConfig, type RetryFallback, type RunEvalCaseResult, type RunEvalInput, type RunEvalResult, type RunTaskContextBudgetBreakdown, type RunTaskContextCompactionEvent, type RunTaskContextCompactionStrategy, type RunTaskContextNoticeEvent, type RunTaskContextSummaryEntry, type RunTaskContinuation, type RunTaskOffloadRecorder, type RunTaskOnContextCompaction, type RunTaskOnContextNotice, type RunTaskOnSession, type RunTaskOptions, type RunTaskResult, type RunTaskResumeState, type RunTaskSessionSummary, type RunTaskState, type RunTaskStateSlice, type RunTaskStatus, type RunTaskToolTraceSlice, type RuntimeCustomToolConfig, type RuntimeExternalToolConfig, type RuntimeFlowToolConfig, type RuntimeLocalToolConfig, type RuntimeSubagentToolConfig, type RuntimeTool, type RuntimeToolConfig, Runtype, type AgentSkillBinding as RuntypeAgentSkillBinding, RuntypeApiError, RuntypeClient, type ConditionalStepConfig as RuntypeConditionalStepConfig, type RuntypeConfig, type FetchGitHubStepConfig as RuntypeFetchGitHubStepConfig, type FetchUrlStepConfig as RuntypeFetchUrlStepConfig, RuntypeFlowBuilder, type FlowConfig as RuntypeFlowConfig, type GenerateEmbeddingStepConfig as RuntypeGenerateEmbeddingStepConfig, type Message as RuntypeMessage, type ModelOverride$1 as RuntypeModelOverride, type Prompt as RuntypePrompt, type PromptStepConfig as RuntypePromptStepConfig, type RuntypeRecord, type RecordConfig as RuntypeRecordConfig, type RetrieveRecordStepConfig as RuntypeRetrieveRecordStepConfig, type SearchStepConfig as RuntypeSearchStepConfig, type SendEmailStepConfig as RuntypeSendEmailStepConfig, type SendEventStepConfig as RuntypeSendEventStepConfig, type SendStreamStepConfig as RuntypeSendStreamStepConfig, type SendTextStepConfig as RuntypeSendTextStepConfig, type SetVariableStepConfig as RuntypeSetVariableStepConfig, type Skill as RuntypeSkill, type SkillCapabilities as RuntypeSkillCapabilities, type SkillFrontmatter as RuntypeSkillFrontmatter, type SkillManifest as RuntypeSkillManifest, type SkillProposal as RuntypeSkillProposal, type SkillRuntypeExtensions as RuntypeSkillRuntypeExtensions, type SkillScanFinding as RuntypeSkillScanFinding, type SkillScanResult as RuntypeSkillScanResult, type SkillScanVerdict as RuntypeSkillScanVerdict, type SkillVersion as RuntypeSkillVersion, type TransformDataStepConfig as RuntypeTransformDataStepConfig, type UpsertFlowConfig as RuntypeUpsertFlowConfig, type UpsertRecordStepConfig as RuntypeUpsertRecordStepConfig, type VectorSearchStepConfig as RuntypeVectorSearchStepConfig, type WaitUntilStepConfig as RuntypeWaitUntilStepConfig, SDK_USER_AGENT, SDK_VERSION, STEP_FIELD_REGISTRY, STEP_TYPE_TO_METHOD, type Schedule, type ScheduleExecutionOptions, type ScheduleListParams, type ScheduleMessage, type ScheduleMessageSet, type ScheduleMessages, type ScheduleMutationResponse, type ScheduleRun, type ScheduleRunNowResponse, type ScheduleStatusResponse, type ScheduleTarget, type ScheduleTrigger, SchedulesEndpoint, type SearchStepConfig$1 as SearchStepConfig, type Secret, type SecretCheckResponse, type SecretDeleteResponse, type SecretSetupUrlRequest, type SecretSetupUrlResponse, SecretsEndpoint, type SendEmailStepConfig$1 as SendEmailStepConfig, type SendEventStepConfig$1 as SendEventStepConfig, type SendStreamStepConfig$1 as SendStreamStepConfig, type SendTextStepConfig$1 as SendTextStepConfig, type SetVariableStepConfig$1 as SetVariableStepConfig, type SkillDefinition, SkillDriftError, SkillEnsureConflictError, type SkillListPage, type SkillListPagination, type SkillListParams, type SkillManifestInput, type SkillMarkdownInput, type SkillOrigin, type SkillProposalStatus, SkillProposalsNamespace, type SkillPullResult, type SkillStatus, type SkillTrustLevel, type SkillVersionStatus, type SkillWithVersion, type SkillWriteInput, SkillsNamespace, type SlackInstallRequest, type StepCompleteEvent, type StepDeltaEvent, type StepFallback, type StepFieldMeta, type StepStartEvent, type StepWaitingLocalEvent, type StreamCallbacks, type StreamConsumeOptions, type StreamEvent, type StreamEventOf, type SubagentToolConfig, type Surface, type SurfaceDefinition, type SurfaceDefinitionEnvironment, type SurfaceDefinitionStatus, type SurfaceDefinitionType, SurfaceDriftError, SurfaceEnsureConflictError, type SurfaceListParams, type SurfacePullResult, SurfacesEndpoint, SurfacesNamespace, type TextContentPart, type Tool, type ToolApprovalGrant, ToolApprovalGrantsEndpoint, type ToolConfig, type ToolDefinition, type ToolDefinitionType, ToolDriftError, ToolEnsureConflictError, type ToolPullResult, type ToolWithValidation, type ToolsConfig, ToolsEndpoint, ToolsNamespace, type TransformDataStepConfig$1 as TransformDataStepConfig, UNIFIED_EVENTS_QUERY, type UpdateAppRequest, type UpdateClientTokenRequest, type UpdateConversationRequest, type UpdatePromptData, type UpdateProviderKeyRequest, type UpdateScheduleRequest, type UpdateSecretRequest, type UpdateToolRequest, type UpdatedFlow, type UpsertFlowConfig$1 as UpsertFlowConfig, type UpsertOptions, type UpsertRecordStepConfig$1 as UpsertRecordStepConfig, type UserProfile, UsersEndpoint, type VectorSearchStepConfig$1 as VectorSearchStepConfig, type VersionType, type WaitUntilStepConfig$1 as WaitUntilStepConfig, type WorkflowCompileDeps, type WorkflowCompletionCriteriaConfig, type WorkflowConfig, type WorkflowConfigFactory, type WorkflowContext, type WorkflowDefinition, type WorkflowHookEntry, type WorkflowHookKind, type WorkflowHookRef, type WorkflowHookSignatures, type WorkflowMilestoneConfig, type WorkflowPhase, type WorkflowPolicyConfig, type WorkflowRecoveryConfig, type WorkflowSlot, type WorkflowStallPolicy, applyGeneratedRuntimeToolProposalToDispatchRequest, attachRuntimeToolsToDispatchRequest, buildEmptySessionNudge, buildGeneratedRuntimeToolGateOutput, buildLedgerOffloadReference, buildPolicyGuidance, buildSendViewOffloadMarker, calledTool, compileWorkflowConfig, completed, computeAgentContentHash, computeEvalContentHash, computeFlowContentHash, computeFpoContentHash, computeProductContentHash, computeSkillContentHash, computeSurfaceContentHash, computeToolContentHash, contains, cost, createAgentEventTranslator, createClient, createExternalTool, createFlowEventTranslator, defaultWorkflow, defaultWorkflowConfig, defineAgent, defineEval, defineFlow, defineFpo, definePlaybook, defineProduct, defineSkill, defineSurface, defineTool, deployWorkflow, ensureDefaultWorkflowHooks, ensureEval, ensureFpo, evaluateGeneratedRuntimeToolProposal, extractDeclaredToolResultChars, gameWorkflow, getDefaultPlanPath, getLikelySupportingCandidatePaths, interpolateWorkflowTemplate, isDiscoveryToolName, isMarathonArtifactPath, isPreservationSensitiveTask, isUnifiedEventType, isWorkflowHookRef, jsonField, judge, judges, latency, length, listWorkflowHooks, matchesExpected, maxToolCalls, noError, normalizeAgentDefinition, normalizeCandidatePath, normalizeFpoDefinition, normalizeProductDefinition, normalizeSkillDefinition, normalizeSurfaceDefinition, normalizeToolDefinition, notCalledTool, notContains, parseFinalBuffer, parseLedgerArtifactRelativePath, parseOffloadedOutputId, parseSSEChunk, processStream, pullEval, pullFpo, ranStep, regex, registerWorkflowHook, resolveStallStopAfter, resolveWorkflowHook, runEvalSuite, sanitizeTaskSlug, shouldInjectEmptySessionNudge, shouldRequestModelEscalation, stepOrder, streamEvents, toolOrder, unregisterWorkflowHook, usedNoTools, validJson, withUnifiedEvents };
|