@runtypelabs/sdk 5.8.1 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +128 -180
- package/dist/index.d.cts +427 -4
- package/dist/index.d.ts +427 -4
- package/dist/index.mjs +128 -180
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -10490,6 +10490,94 @@ interface paths {
|
|
|
10490
10490
|
patch?: never;
|
|
10491
10491
|
trace?: never;
|
|
10492
10492
|
};
|
|
10493
|
+
"/v1/eval/graders/decompose": {
|
|
10494
|
+
parameters: {
|
|
10495
|
+
query?: never;
|
|
10496
|
+
header?: never;
|
|
10497
|
+
path?: never;
|
|
10498
|
+
cookie?: never;
|
|
10499
|
+
};
|
|
10500
|
+
get?: never;
|
|
10501
|
+
put?: never;
|
|
10502
|
+
/**
|
|
10503
|
+
* Decompose grader criteria
|
|
10504
|
+
* @description Split one plain-language AI-grader criterion carrying several obligations into focused, independently judgeable sub-checks. Authoring assist only: review the proposal, then save each accepted sub-check as its own AI grader (kind "ai") on the suite. Nothing is persisted by this endpoint.
|
|
10505
|
+
*/
|
|
10506
|
+
post: {
|
|
10507
|
+
parameters: {
|
|
10508
|
+
query?: never;
|
|
10509
|
+
header?: never;
|
|
10510
|
+
path?: never;
|
|
10511
|
+
cookie?: never;
|
|
10512
|
+
};
|
|
10513
|
+
requestBody?: {
|
|
10514
|
+
content: {
|
|
10515
|
+
"application/json": components["schemas"]["DecomposeCriteriaRequest"];
|
|
10516
|
+
};
|
|
10517
|
+
};
|
|
10518
|
+
responses: {
|
|
10519
|
+
/** @description Proposed sub-checks */
|
|
10520
|
+
200: {
|
|
10521
|
+
headers: {
|
|
10522
|
+
[name: string]: unknown;
|
|
10523
|
+
};
|
|
10524
|
+
content: {
|
|
10525
|
+
"application/json": components["schemas"]["DecomposeCriteriaResponse"];
|
|
10526
|
+
};
|
|
10527
|
+
};
|
|
10528
|
+
/** @description Invalid request */
|
|
10529
|
+
400: {
|
|
10530
|
+
headers: {
|
|
10531
|
+
[name: string]: unknown;
|
|
10532
|
+
};
|
|
10533
|
+
content: {
|
|
10534
|
+
"application/json": components["schemas"]["Error"];
|
|
10535
|
+
};
|
|
10536
|
+
};
|
|
10537
|
+
/** @description Unauthorized */
|
|
10538
|
+
401: {
|
|
10539
|
+
headers: {
|
|
10540
|
+
[name: string]: unknown;
|
|
10541
|
+
};
|
|
10542
|
+
content: {
|
|
10543
|
+
"application/json": components["schemas"]["Error"];
|
|
10544
|
+
};
|
|
10545
|
+
};
|
|
10546
|
+
/** @description Forbidden */
|
|
10547
|
+
403: {
|
|
10548
|
+
headers: {
|
|
10549
|
+
[name: string]: unknown;
|
|
10550
|
+
};
|
|
10551
|
+
content: {
|
|
10552
|
+
"application/json": components["schemas"]["Error"];
|
|
10553
|
+
};
|
|
10554
|
+
};
|
|
10555
|
+
/** @description Internal server error */
|
|
10556
|
+
500: {
|
|
10557
|
+
headers: {
|
|
10558
|
+
[name: string]: unknown;
|
|
10559
|
+
};
|
|
10560
|
+
content: {
|
|
10561
|
+
"application/json": components["schemas"]["Error"];
|
|
10562
|
+
};
|
|
10563
|
+
};
|
|
10564
|
+
/** @description The decomposition model did not return a usable proposal */
|
|
10565
|
+
502: {
|
|
10566
|
+
headers: {
|
|
10567
|
+
[name: string]: unknown;
|
|
10568
|
+
};
|
|
10569
|
+
content: {
|
|
10570
|
+
"application/json": components["schemas"]["Error"];
|
|
10571
|
+
};
|
|
10572
|
+
};
|
|
10573
|
+
};
|
|
10574
|
+
};
|
|
10575
|
+
delete?: never;
|
|
10576
|
+
options?: never;
|
|
10577
|
+
head?: never;
|
|
10578
|
+
patch?: never;
|
|
10579
|
+
trace?: never;
|
|
10580
|
+
};
|
|
10493
10581
|
"/v1/eval/group/{groupId}": {
|
|
10494
10582
|
parameters: {
|
|
10495
10583
|
query?: never;
|
|
@@ -11147,9 +11235,32 @@ interface paths {
|
|
|
11147
11235
|
};
|
|
11148
11236
|
/** @default {} */
|
|
11149
11237
|
input?: {
|
|
11238
|
+
checkpoint?: {
|
|
11239
|
+
forkMessageIndex: number;
|
|
11240
|
+
sourceConversationId?: string;
|
|
11241
|
+
sourceExecutionId?: string;
|
|
11242
|
+
};
|
|
11150
11243
|
messages?: {
|
|
11151
11244
|
content: string;
|
|
11152
11245
|
role: string;
|
|
11246
|
+
toolCalls?: {
|
|
11247
|
+
args?: unknown;
|
|
11248
|
+
toolCallId: string;
|
|
11249
|
+
toolName: string;
|
|
11250
|
+
}[];
|
|
11251
|
+
toolResults?: {
|
|
11252
|
+
result?: unknown;
|
|
11253
|
+
toolCallId: string;
|
|
11254
|
+
toolName: string;
|
|
11255
|
+
}[];
|
|
11256
|
+
}[];
|
|
11257
|
+
toolMocks?: {
|
|
11258
|
+
input?: unknown;
|
|
11259
|
+
isError?: boolean;
|
|
11260
|
+
output?: unknown;
|
|
11261
|
+
sourceToolExecutionId?: string;
|
|
11262
|
+
toolName: string;
|
|
11263
|
+
truncated?: boolean;
|
|
11153
11264
|
}[];
|
|
11154
11265
|
variables?: {
|
|
11155
11266
|
[key: string]: unknown;
|
|
@@ -11502,9 +11613,32 @@ interface paths {
|
|
|
11502
11613
|
};
|
|
11503
11614
|
/** @default {} */
|
|
11504
11615
|
input?: {
|
|
11616
|
+
checkpoint?: {
|
|
11617
|
+
forkMessageIndex: number;
|
|
11618
|
+
sourceConversationId?: string;
|
|
11619
|
+
sourceExecutionId?: string;
|
|
11620
|
+
};
|
|
11505
11621
|
messages?: {
|
|
11506
11622
|
content: string;
|
|
11507
11623
|
role: string;
|
|
11624
|
+
toolCalls?: {
|
|
11625
|
+
args?: unknown;
|
|
11626
|
+
toolCallId: string;
|
|
11627
|
+
toolName: string;
|
|
11628
|
+
}[];
|
|
11629
|
+
toolResults?: {
|
|
11630
|
+
result?: unknown;
|
|
11631
|
+
toolCallId: string;
|
|
11632
|
+
toolName: string;
|
|
11633
|
+
}[];
|
|
11634
|
+
}[];
|
|
11635
|
+
toolMocks?: {
|
|
11636
|
+
input?: unknown;
|
|
11637
|
+
isError?: boolean;
|
|
11638
|
+
output?: unknown;
|
|
11639
|
+
sourceToolExecutionId?: string;
|
|
11640
|
+
toolName: string;
|
|
11641
|
+
truncated?: boolean;
|
|
11508
11642
|
}[];
|
|
11509
11643
|
variables?: {
|
|
11510
11644
|
[key: string]: unknown;
|
|
@@ -11584,6 +11718,110 @@ interface paths {
|
|
|
11584
11718
|
patch?: never;
|
|
11585
11719
|
trace?: never;
|
|
11586
11720
|
};
|
|
11721
|
+
"/v1/eval/suites/{id}/cases/from-execution": {
|
|
11722
|
+
parameters: {
|
|
11723
|
+
query?: never;
|
|
11724
|
+
header?: never;
|
|
11725
|
+
path?: never;
|
|
11726
|
+
cookie?: never;
|
|
11727
|
+
};
|
|
11728
|
+
get?: never;
|
|
11729
|
+
put?: never;
|
|
11730
|
+
/**
|
|
11731
|
+
* Capture an eval case from a run
|
|
11732
|
+
* @description Fork an agent execution into a test case: freeze its conversation history up to a fork point, attach every recorded tool result as an editable mock, and save it with origin "saved_from_run". Capture only — reads the run and writes a case; it never re-executes anything. Restricted to agent executions the caller owns.
|
|
11733
|
+
*/
|
|
11734
|
+
post: {
|
|
11735
|
+
parameters: {
|
|
11736
|
+
query?: never;
|
|
11737
|
+
header?: never;
|
|
11738
|
+
path: {
|
|
11739
|
+
id: string;
|
|
11740
|
+
};
|
|
11741
|
+
cookie?: never;
|
|
11742
|
+
};
|
|
11743
|
+
requestBody?: {
|
|
11744
|
+
content: {
|
|
11745
|
+
"application/json": {
|
|
11746
|
+
/** @description Runtime execution id (from a log/trace row) of the agent run to fork from. */
|
|
11747
|
+
executionId: string;
|
|
11748
|
+
/** @description What "good" looks like for the graded next step (optional; e.g. pre-filled from a 👍). */
|
|
11749
|
+
expected?: {
|
|
11750
|
+
facts?: string[];
|
|
11751
|
+
json?: unknown;
|
|
11752
|
+
text?: string;
|
|
11753
|
+
};
|
|
11754
|
+
/** @description Where to fork the frozen history (index into the reconstructed message array). Defaults to just before the last recorded tool-call action, so the run’s final action is the graded next step. */
|
|
11755
|
+
forkMessageIndex?: number;
|
|
11756
|
+
/** @description Case name. Defaults to "Saved from run <executionId>". */
|
|
11757
|
+
name?: string;
|
|
11758
|
+
notes?: string;
|
|
11759
|
+
};
|
|
11760
|
+
};
|
|
11761
|
+
};
|
|
11762
|
+
responses: {
|
|
11763
|
+
/** @description Case captured */
|
|
11764
|
+
201: {
|
|
11765
|
+
headers: {
|
|
11766
|
+
[name: string]: unknown;
|
|
11767
|
+
};
|
|
11768
|
+
content: {
|
|
11769
|
+
"application/json": components["schemas"]["CapturedEvalCase"];
|
|
11770
|
+
};
|
|
11771
|
+
};
|
|
11772
|
+
/** @description Invalid request */
|
|
11773
|
+
400: {
|
|
11774
|
+
headers: {
|
|
11775
|
+
[name: string]: unknown;
|
|
11776
|
+
};
|
|
11777
|
+
content: {
|
|
11778
|
+
"application/json": components["schemas"]["Error"];
|
|
11779
|
+
};
|
|
11780
|
+
};
|
|
11781
|
+
/** @description Unauthorized */
|
|
11782
|
+
401: {
|
|
11783
|
+
headers: {
|
|
11784
|
+
[name: string]: unknown;
|
|
11785
|
+
};
|
|
11786
|
+
content: {
|
|
11787
|
+
"application/json": components["schemas"]["Error"];
|
|
11788
|
+
};
|
|
11789
|
+
};
|
|
11790
|
+
/** @description Forbidden */
|
|
11791
|
+
403: {
|
|
11792
|
+
headers: {
|
|
11793
|
+
[name: string]: unknown;
|
|
11794
|
+
};
|
|
11795
|
+
content: {
|
|
11796
|
+
"application/json": components["schemas"]["Error"];
|
|
11797
|
+
};
|
|
11798
|
+
};
|
|
11799
|
+
/** @description Not found */
|
|
11800
|
+
404: {
|
|
11801
|
+
headers: {
|
|
11802
|
+
[name: string]: unknown;
|
|
11803
|
+
};
|
|
11804
|
+
content: {
|
|
11805
|
+
"application/json": components["schemas"]["Error"];
|
|
11806
|
+
};
|
|
11807
|
+
};
|
|
11808
|
+
/** @description Internal server error */
|
|
11809
|
+
500: {
|
|
11810
|
+
headers: {
|
|
11811
|
+
[name: string]: unknown;
|
|
11812
|
+
};
|
|
11813
|
+
content: {
|
|
11814
|
+
"application/json": components["schemas"]["Error"];
|
|
11815
|
+
};
|
|
11816
|
+
};
|
|
11817
|
+
};
|
|
11818
|
+
};
|
|
11819
|
+
delete?: never;
|
|
11820
|
+
options?: never;
|
|
11821
|
+
head?: never;
|
|
11822
|
+
patch?: never;
|
|
11823
|
+
trace?: never;
|
|
11824
|
+
};
|
|
11587
11825
|
"/v1/eval/suites/{id}/cases/{caseId}": {
|
|
11588
11826
|
parameters: {
|
|
11589
11827
|
query?: never;
|
|
@@ -11685,9 +11923,32 @@ interface paths {
|
|
|
11685
11923
|
text?: string;
|
|
11686
11924
|
} | null;
|
|
11687
11925
|
input?: {
|
|
11926
|
+
checkpoint?: {
|
|
11927
|
+
forkMessageIndex: number;
|
|
11928
|
+
sourceConversationId?: string;
|
|
11929
|
+
sourceExecutionId?: string;
|
|
11930
|
+
};
|
|
11688
11931
|
messages?: {
|
|
11689
11932
|
content: string;
|
|
11690
11933
|
role: string;
|
|
11934
|
+
toolCalls?: {
|
|
11935
|
+
args?: unknown;
|
|
11936
|
+
toolCallId: string;
|
|
11937
|
+
toolName: string;
|
|
11938
|
+
}[];
|
|
11939
|
+
toolResults?: {
|
|
11940
|
+
result?: unknown;
|
|
11941
|
+
toolCallId: string;
|
|
11942
|
+
toolName: string;
|
|
11943
|
+
}[];
|
|
11944
|
+
}[];
|
|
11945
|
+
toolMocks?: {
|
|
11946
|
+
input?: unknown;
|
|
11947
|
+
isError?: boolean;
|
|
11948
|
+
output?: unknown;
|
|
11949
|
+
sourceToolExecutionId?: string;
|
|
11950
|
+
toolName: string;
|
|
11951
|
+
truncated?: boolean;
|
|
11691
11952
|
}[];
|
|
11692
11953
|
variables?: {
|
|
11693
11954
|
[key: string]: unknown;
|
|
@@ -18404,6 +18665,8 @@ interface paths {
|
|
|
18404
18665
|
content: {
|
|
18405
18666
|
"application/json": {
|
|
18406
18667
|
data: {
|
|
18668
|
+
/** @description True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. Absent on healthy responses. */
|
|
18669
|
+
degraded?: boolean;
|
|
18407
18670
|
entries: {
|
|
18408
18671
|
[key: string]: unknown;
|
|
18409
18672
|
}[];
|
|
@@ -18582,6 +18845,8 @@ interface paths {
|
|
|
18582
18845
|
byType: {
|
|
18583
18846
|
[key: string]: number;
|
|
18584
18847
|
};
|
|
18848
|
+
/** @description True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. Absent on healthy responses. */
|
|
18849
|
+
degraded?: boolean;
|
|
18585
18850
|
histogram: {
|
|
18586
18851
|
bucket: string;
|
|
18587
18852
|
count: number;
|
|
@@ -38568,6 +38833,11 @@ interface components {
|
|
|
38568
38833
|
/** @description Lifecycle status at acceptance (e.g. 'running'). */
|
|
38569
38834
|
status: string;
|
|
38570
38835
|
};
|
|
38836
|
+
CapturedEvalCase: {
|
|
38837
|
+
case: components["schemas"]["EvalCase"];
|
|
38838
|
+
/** @description True when every recorded tool output was captured in full. False when any output was truncated at capture — the case still saves, but tool outputs are partial ("⚠ tool outputs weren’t fully captured"). */
|
|
38839
|
+
replayable: boolean;
|
|
38840
|
+
};
|
|
38571
38841
|
DailyUsageResponse: {
|
|
38572
38842
|
daily?: {
|
|
38573
38843
|
atSpendLimit: boolean;
|
|
@@ -38608,6 +38878,18 @@ interface components {
|
|
|
38608
38878
|
tier: string;
|
|
38609
38879
|
};
|
|
38610
38880
|
};
|
|
38881
|
+
DecomposeCriteriaRequest: {
|
|
38882
|
+
/** @description One plain-language AI-grader criterion to split into focused sub-checks. */
|
|
38883
|
+
criteria: string;
|
|
38884
|
+
};
|
|
38885
|
+
DecomposeCriteriaResponse: {
|
|
38886
|
+
/** @description Resolved model id that produced the proposal. */
|
|
38887
|
+
model: string;
|
|
38888
|
+
/** @description Proposed focused checks (1-8), ordered as the obligations appear in the original criterion. A single entry means the criterion is already focused. Each accepted sub-check becomes a plain {kind:"ai"} grader row. */
|
|
38889
|
+
subChecks: {
|
|
38890
|
+
criteria: string;
|
|
38891
|
+
}[];
|
|
38892
|
+
};
|
|
38611
38893
|
EnrichedSkillProposal: components["schemas"]["SkillProposal"] & {
|
|
38612
38894
|
proposingAgent: {
|
|
38613
38895
|
id: string;
|
|
@@ -38717,6 +38999,12 @@ interface components {
|
|
|
38717
38999
|
caseId: string | null;
|
|
38718
39000
|
name: string;
|
|
38719
39001
|
outcomes: {
|
|
39002
|
+
/** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
|
|
39003
|
+
evidence?: {
|
|
39004
|
+
index: number;
|
|
39005
|
+
/** @enum {string} */
|
|
39006
|
+
kind: "message" | "tool_call";
|
|
39007
|
+
}[];
|
|
38720
39008
|
graderIndex: number;
|
|
38721
39009
|
kind: string;
|
|
38722
39010
|
passed: boolean;
|
|
@@ -38727,6 +39015,11 @@ interface components {
|
|
|
38727
39015
|
* @enum {string}
|
|
38728
39016
|
*/
|
|
38729
39017
|
severity?: "gate" | "soft";
|
|
39018
|
+
/**
|
|
39019
|
+
* @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
|
|
39020
|
+
* @enum {string}
|
|
39021
|
+
*/
|
|
39022
|
+
verdict?: "pass" | "fail" | "insufficient_evidence";
|
|
38730
39023
|
}[];
|
|
38731
39024
|
/** @description Truncated snapshot of the output that was graded (2000 chars max). */
|
|
38732
39025
|
outputExcerpt: string;
|
|
@@ -39836,6 +40129,12 @@ interface components {
|
|
|
39836
40129
|
errored: boolean;
|
|
39837
40130
|
name: string;
|
|
39838
40131
|
outcomes: {
|
|
40132
|
+
/** @description AI graders only: transcript entries the verdict rests on — indices into the run's trace steps (message) or tool calls (tool_call). */
|
|
40133
|
+
evidence?: {
|
|
40134
|
+
index: number;
|
|
40135
|
+
/** @enum {string} */
|
|
40136
|
+
kind: "message" | "tool_call";
|
|
40137
|
+
}[];
|
|
39839
40138
|
graderIndex: number;
|
|
39840
40139
|
kind: string;
|
|
39841
40140
|
passed: boolean;
|
|
@@ -39846,6 +40145,11 @@ interface components {
|
|
|
39846
40145
|
* @enum {string}
|
|
39847
40146
|
*/
|
|
39848
40147
|
severity?: "gate" | "soft";
|
|
40148
|
+
/**
|
|
40149
|
+
* @description AI graders only: the judge's binary verdict. insufficient_evidence means the judge declined to guess; it renders as a warning and never fails the case.
|
|
40150
|
+
* @enum {string}
|
|
40151
|
+
*/
|
|
40152
|
+
verdict?: "pass" | "fail" | "insufficient_evidence";
|
|
39849
40153
|
}[];
|
|
39850
40154
|
outputExcerpt: string;
|
|
39851
40155
|
passed: boolean;
|
|
@@ -42396,6 +42700,8 @@ interface LogQueryResult {
|
|
|
42396
42700
|
cursor: string | null;
|
|
42397
42701
|
hasMore: boolean;
|
|
42398
42702
|
};
|
|
42703
|
+
/** True when historical (R2 SQL) logs were unavailable and only recent hot-tier entries are included. */
|
|
42704
|
+
degraded?: boolean;
|
|
42399
42705
|
}
|
|
42400
42706
|
/** Logs are wrapped in a `{ success, data }` envelope by the API. */
|
|
42401
42707
|
interface LogQueryResponse {
|
|
@@ -42414,6 +42720,8 @@ interface LogStatsResult {
|
|
|
42414
42720
|
count: number;
|
|
42415
42721
|
errors: number;
|
|
42416
42722
|
}>;
|
|
42723
|
+
/** True when historical (R2 SQL) counts were unavailable and stats cover only the recent hot-tier window. */
|
|
42724
|
+
degraded?: boolean;
|
|
42417
42725
|
}
|
|
42418
42726
|
interface LogStatsResponse {
|
|
42419
42727
|
success: boolean;
|
|
@@ -42621,8 +42929,17 @@ interface AIGrader {
|
|
|
42621
42929
|
criteria: string;
|
|
42622
42930
|
/** Reference-guided when the case has `expected`. */
|
|
42623
42931
|
useExpected?: boolean;
|
|
42624
|
-
/** Defaults to a cheap routed model at execution time. */
|
|
42932
|
+
/** Defaults to a cheap routed model at execution time. Ignored when `judgeFlowId` is set. */
|
|
42625
42933
|
model?: string;
|
|
42934
|
+
/**
|
|
42935
|
+
* Custom judge flow: your flow (by id) runs as the judge instead of the
|
|
42936
|
+
* platform judge prompt. It receives `criteria`, `caseName`, `expected`, and
|
|
42937
|
+
* `transcript` as inputs and must output the platform verdict JSON
|
|
42938
|
+
* (`{"reasoning", "verdict": "pass"|"fail"|"insufficient_evidence", "evidence"}`).
|
|
42939
|
+
* A miswired flow (missing, failing, or returning malformed output) resolves
|
|
42940
|
+
* to an insufficient-evidence warning with the error, never a verdict.
|
|
42941
|
+
*/
|
|
42942
|
+
judgeFlowId?: string;
|
|
42626
42943
|
/** Pass cutoff on the 1-5 judge scale (default 4 server-side). Set via `.atLeast(n)`. */
|
|
42627
42944
|
threshold?: number;
|
|
42628
42945
|
/** Hard gate (default) vs soft. Set via `.gate()` / `.soft()`. */
|
|
@@ -42717,6 +43034,7 @@ declare function cost(maxUsd: number): Gradeable<CheckGrader>;
|
|
|
42717
43034
|
declare function judge(criteria: string, opts?: {
|
|
42718
43035
|
useExpected?: boolean;
|
|
42719
43036
|
model?: string;
|
|
43037
|
+
judgeFlowId?: string;
|
|
42720
43038
|
threshold?: number;
|
|
42721
43039
|
preset?: BuiltInGraderId;
|
|
42722
43040
|
}): Gradeable<AIGrader>;
|
|
@@ -42740,10 +43058,39 @@ interface EvalMessage {
|
|
|
42740
43058
|
role: 'user' | 'assistant' | 'system';
|
|
42741
43059
|
content: string;
|
|
42742
43060
|
}
|
|
43061
|
+
/**
|
|
43062
|
+
* Where a `saved_from_run` case was forked from (mirror of `CaseCheckpoint` in
|
|
43063
|
+
* `@runtypelabs/shared`). Present on cases captured from a run.
|
|
43064
|
+
*/
|
|
43065
|
+
interface EvalCaseCheckpoint {
|
|
43066
|
+
sourceExecutionId?: string;
|
|
43067
|
+
sourceConversationId?: string;
|
|
43068
|
+
/** Length of the frozen `messages` prefix kept before the graded next step. */
|
|
43069
|
+
forkMessageIndex: number;
|
|
43070
|
+
}
|
|
43071
|
+
/**
|
|
43072
|
+
* One recorded tool result attached to a captured case (mirror of `ToolMock` in
|
|
43073
|
+
* `@runtypelabs/shared`). A future next-step replay serves these by tool name;
|
|
43074
|
+
* editing `output` turns the case into a counterfactual.
|
|
43075
|
+
*/
|
|
43076
|
+
interface EvalToolMock {
|
|
43077
|
+
toolName: string;
|
|
43078
|
+
input?: unknown;
|
|
43079
|
+
output: unknown;
|
|
43080
|
+
/** True when the recorded call failed: `output` is its error payload and a replay serves it as a failure. */
|
|
43081
|
+
isError?: boolean;
|
|
43082
|
+
sourceToolExecutionId?: string;
|
|
43083
|
+
/** True when `output` was capped at capture (the case is then non-fully-replayable). */
|
|
43084
|
+
truncated?: boolean;
|
|
43085
|
+
}
|
|
42743
43086
|
/** A case's input: flow variables and/or a scripted conversation to replay. */
|
|
42744
43087
|
interface EvalCaseInput {
|
|
42745
43088
|
variables?: Record<string, unknown>;
|
|
42746
43089
|
messages?: EvalMessage[];
|
|
43090
|
+
/** Present on `saved_from_run` cases: where the run was forked. */
|
|
43091
|
+
checkpoint?: EvalCaseCheckpoint;
|
|
43092
|
+
/** Present on `saved_from_run` cases: recorded tool results, for replay/counterfactuals. */
|
|
43093
|
+
toolMocks?: EvalToolMock[];
|
|
42747
43094
|
}
|
|
42748
43095
|
/** The target a suite evaluates — a saved flow or agent, by portable name. */
|
|
42749
43096
|
type EvalTarget = {
|
|
@@ -42852,10 +43199,24 @@ interface GraderOutcome {
|
|
|
42852
43199
|
passed: boolean;
|
|
42853
43200
|
/** The grader's severity (absent ⇒ gate). A soft miss only fails under `--strict`. */
|
|
42854
43201
|
severity?: GraderSeverity;
|
|
42855
|
-
/** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). */
|
|
43202
|
+
/** 0..1 for scaled graders (AI graders normalize their 1-5 to 0..1). Absent for `insufficient_evidence`. */
|
|
42856
43203
|
score?: number;
|
|
42857
|
-
/** AI-grader verdict, or a check's human-readable reason. */
|
|
43204
|
+
/** AI-grader reasoning (critique-before-verdict), or a check's human-readable reason. */
|
|
42858
43205
|
reasoning?: string;
|
|
43206
|
+
/**
|
|
43207
|
+
* AI graders only: the trajectory judge's binary verdict.
|
|
43208
|
+
* `insufficient_evidence` means the judge declined to guess — it renders as
|
|
43209
|
+
* a warning and never fails the case.
|
|
43210
|
+
*/
|
|
43211
|
+
verdict?: 'pass' | 'fail' | 'insufficient_evidence';
|
|
43212
|
+
/**
|
|
43213
|
+
* AI graders only: transcript entries the verdict rests on — indices into
|
|
43214
|
+
* the run's trace steps (`message`) or tool calls (`tool_call`).
|
|
43215
|
+
*/
|
|
43216
|
+
evidence?: Array<{
|
|
43217
|
+
kind: 'message' | 'tool_call';
|
|
43218
|
+
index: number;
|
|
43219
|
+
}>;
|
|
42859
43220
|
}
|
|
42860
43221
|
/** One case's run result: pass/fail plus each grader's outcome. */
|
|
42861
43222
|
interface RunEvalCaseResult {
|
|
@@ -43759,6 +44120,31 @@ interface UpdateEvalSuiteInput {
|
|
|
43759
44120
|
/** Pin a run of this suite as the comparison baseline; null clears the pin. */
|
|
43760
44121
|
baselineBatchExecutionId?: string | null;
|
|
43761
44122
|
}
|
|
44123
|
+
/** Input to `client.evals.suites.addCaseFromExecution`. */
|
|
44124
|
+
interface CaptureCaseFromExecutionInput {
|
|
44125
|
+
/** Runtime execution id (from a log/trace row) of the agent run to fork from. */
|
|
44126
|
+
executionId: string;
|
|
44127
|
+
/** Case name. Defaults to "Saved from run <executionId>" server-side. */
|
|
44128
|
+
name?: string;
|
|
44129
|
+
/**
|
|
44130
|
+
* Where to fork the frozen history (index into the reconstructed message
|
|
44131
|
+
* array). Defaults to just before the run's last recorded tool-call action.
|
|
44132
|
+
*/
|
|
44133
|
+
forkMessageIndex?: number;
|
|
44134
|
+
/** What "good" looks like for the graded next step. */
|
|
44135
|
+
expected?: CaseExpected;
|
|
44136
|
+
notes?: string;
|
|
44137
|
+
}
|
|
44138
|
+
/** Result of `client.evals.suites.addCaseFromExecution`. */
|
|
44139
|
+
interface CaptureCaseResult {
|
|
44140
|
+
case: EvalSuiteCase;
|
|
44141
|
+
/**
|
|
44142
|
+
* True when every recorded tool output was captured in full. False when any
|
|
44143
|
+
* output was truncated at capture — the case still saves, but its tool
|
|
44144
|
+
* outputs are partial.
|
|
44145
|
+
*/
|
|
44146
|
+
replayable: boolean;
|
|
44147
|
+
}
|
|
43762
44148
|
/** Input to `client.evals.suites.updateCase`. */
|
|
43763
44149
|
interface UpdateEvalCaseInput {
|
|
43764
44150
|
name?: string;
|
|
@@ -43826,6 +44212,13 @@ declare class EvalSuitesNamespace {
|
|
|
43826
44212
|
addCases(suiteId: string, cases: EvalSuiteCaseInput[]): Promise<{
|
|
43827
44213
|
cases: EvalSuiteCase[];
|
|
43828
44214
|
}>;
|
|
44215
|
+
/**
|
|
44216
|
+
* Capture a test case from a real agent run ("fork here and test the next
|
|
44217
|
+
* step"): freezes the run's conversation history up to a fork point, attaches
|
|
44218
|
+
* every recorded tool result as an editable mock, and saves it with origin
|
|
44219
|
+
* `saved_from_run`. Capture only — reads the run, never re-executes it.
|
|
44220
|
+
*/
|
|
44221
|
+
addCaseFromExecution(suiteId: string, input: CaptureCaseFromExecutionInput): Promise<CaptureCaseResult>;
|
|
43829
44222
|
/** Edit, enable, or disable a test case. */
|
|
43830
44223
|
updateCase(suiteId: string, caseId: string, input: UpdateEvalCaseInput): Promise<EvalSuiteCase>;
|
|
43831
44224
|
/** Delete a test case. Past runs keep the case's persisted scores. */
|
|
@@ -43861,6 +44254,19 @@ interface ModelOverride$1 {
|
|
|
43861
44254
|
/** Optional max tokens override */
|
|
43862
44255
|
maxTokens?: number;
|
|
43863
44256
|
}
|
|
44257
|
+
/** Result of `POST /eval/graders/decompose` — a reviewed-before-saved proposal. */
|
|
44258
|
+
interface DecomposeCriteriaResult {
|
|
44259
|
+
/**
|
|
44260
|
+
* Proposed focused checks, ordered as the obligations appear in the
|
|
44261
|
+
* original criterion. Each accepted sub-check becomes a plain `{kind:'ai'}`
|
|
44262
|
+
* grader row (e.g. `judge(subCheck.criteria)`).
|
|
44263
|
+
*/
|
|
44264
|
+
subChecks: Array<{
|
|
44265
|
+
criteria: string;
|
|
44266
|
+
}>;
|
|
44267
|
+
/** Resolved model id that produced the proposal. */
|
|
44268
|
+
model: string;
|
|
44269
|
+
}
|
|
43864
44270
|
interface EvalRunConfig {
|
|
43865
44271
|
/** Flow ID to evaluate */
|
|
43866
44272
|
flowId?: string;
|
|
@@ -44049,6 +44455,23 @@ declare class EvalsNamespace {
|
|
|
44049
44455
|
* absorb-drift direction of the ensure protocol.
|
|
44050
44456
|
*/
|
|
44051
44457
|
pull(name: string): Promise<EvalPullResult>;
|
|
44458
|
+
/**
|
|
44459
|
+
* Split one plain-language AI-grader criterion carrying several obligations
|
|
44460
|
+
* into focused, independently judgeable sub-checks. Authoring assist only:
|
|
44461
|
+
* nothing is persisted — review the proposal, then save each accepted
|
|
44462
|
+
* sub-check as its own AI grader row. A single returned sub-check means the
|
|
44463
|
+
* criterion is already focused.
|
|
44464
|
+
*
|
|
44465
|
+
* @example
|
|
44466
|
+
* ```typescript
|
|
44467
|
+
* const { subChecks } = await Runtype.evals.decomposeCriteria(
|
|
44468
|
+
* 'Confirms the order number before issuing a refund, and never promises a delivery date.'
|
|
44469
|
+
* )
|
|
44470
|
+
* // In a defineEval suite, each accepted sub-check becomes its own judge row:
|
|
44471
|
+
* const graders = subChecks.map((s) => judge(s.criteria))
|
|
44472
|
+
* ```
|
|
44473
|
+
*/
|
|
44474
|
+
decomposeCriteria(criteria: string): Promise<DecomposeCriteriaResult>;
|
|
44052
44475
|
/**
|
|
44053
44476
|
* Run an eval suite synchronously and return the suite score + per-case grader
|
|
44054
44477
|
* outcomes — the executing counterpart of `ensure`, powering the `runtype
|
|
@@ -50811,4 +51234,4 @@ declare function getLikelySupportingCandidatePaths(bestCandidatePath: string | u
|
|
|
50811
51234
|
declare function getDefaultPlanPath(taskName: string): string;
|
|
50812
51235
|
declare function sanitizeTaskSlug(taskName: string): string;
|
|
50813
51236
|
|
|
50814
|
-
export { type AIGrader, type Agent, type AgentApprovalCompleteEvent, type AgentApprovalStartEvent, type AgentCompleteEvent, type AgentDefinition, type AgentDefinitionConfig, AgentDriftError, AgentEnsureConflictError, type AgentErrorEvent, type AgentEvent, type AgentEventType, type AgentExecuteRequest, type AgentExecuteResponse, type AgentIterationCompleteEvent, type AgentIterationStartEvent, type AgentMediaEvent, type AgentMessage, type AgentPausedEvent, type AgentPingEvent, type AgentPullResult, type AgentReflectionEvent, type AgentRuntimeToolDefinition, type AgentStartEvent, type AgentStreamCallbacks, type AgentStreamEvent, type AgentSubagentConfig, type AgentToolCompleteEvent, type AgentToolDeltaEvent, type AgentToolInputCompleteEvent, type AgentToolInputDeltaEvent, type AgentToolStartEvent, type AgentTurnCompleteEvent, type AgentTurnDeltaEvent, type AgentTurnStartEvent, type AgentVersionDetail, type AgentVersionListItem, type AgentVersionPublishResponse, AgentVersionsEndpoint, type AgentVersionsListResponse, AgentsEndpoint, AgentsNamespace, AnalyticsEndpoint, type ApiClient, type ApiKey, ApiKeysEndpoint, type ApiResponse, type App, type AppManifest, type AppVersion, type ApplyGeneratedProposalOptions, type ApplyGeneratedProposalResult, AppsEndpoint, type AssetReferenceContentPart, type AttachRuntimeToolsOptions, type BaseAgentEvent, BatchBuilder, type BatchClient, type BatchListParams, type BatchOptions, type BatchRequest, type BatchResult, type BatchScheduleConfig, type BatchStatus, BatchesNamespace, BillingEndpoint, type BillingSpendAnalyticsParams, type BindSkillInput, type BuiltInGraderId, type BuiltInTool, type BulkEditCondition, type BulkEditRequest, type BulkEditResponse, type BulkEditResult, type CaseExpected, ChatEndpoint, type CheckGrader, ClientBatchBuilder, type ClientConfig, type ClientConversation, ClientEvalBuilder, ClientFlowBuilder, type ClientToken, type ClientTokenConfig, type ClientTokenEnvironment, type ClientTokenVersionPin, ClientTokensEndpoint, type ClientToolDefinition, type ClientWidgetTheme, type ConditionalGetResult, type ConditionalStepConfig$1 as ConditionalStepConfig, type ContextErrorHandling, type ContextFallback, ContextTemplatesEndpoint, type Conversation, type ConversationListItem, type ConversationListParams, type ConversationMessage, type ConversationSource, ConversationsEndpoint, type ConversationsListResponse, type CreateApiKeyRequest, type CreateAppRequest, type CreateClientTokenRequest, type CreateClientTokenResponse, type CreateConversationRequest, type CreateEvalSuiteInput, type CreateFlowRequest, type CreateModelConfigRequest, type CreatePromptData, type CreatePromptRequest, type CreateProviderKeyRequest, type CreateRecordRequest, type CreateScheduleRequest, type CreateSecretRequest, type CreateToolRequest, type CustomMCPServer, type CustomMCPServerAuth, type CustomToolConfig, DEFAULT_RECOVERY_AFTER_EMPTY_SESSIONS, DEFAULT_STALL_STOP_AFTER, type DefineAgentInput, type DefineEvalCaseInput, type DefineEvalInput, type DefineFlowInput, type DefineProductInput, type DefineSkillInput, type DefineSurfaceInput, type DefineToolInput, type DeployCfSandboxRequest, type DeployCfSandboxResponse, type DeploySandboxRequest, type DeploySandboxResponse, type DiscoveredModel, type DispatchClient, DispatchEndpoint, type DispatchEnvironment, type DispatchEvent, type DispatchOptions$1 as DispatchOptions, type DispatchRequest, type EnsureAgentConverged, type EnsureAgentOptions, type EnsureAgentPlan, type EnsureAgentResult, type EnsureEvalResult, type EnsureFlowConverged, type EnsureFlowOptions, type EnsureFlowPlan, type EnsureFlowResult, type EnsureFpoOptions, type EnsureFpoResult, type EnsureProductConverged, type EnsureProductOptions, type EnsureProductPlan, type EnsureProductResult, type EnsureSkillConverged, type EnsureSkillOptions, type EnsureSkillPlan, type EnsureSkillResult, type EnsureSurfaceConverged, type EnsureSurfaceOptions, type EnsureSurfacePlan, type EnsureSurfaceResult, type EnsureToolConverged, type EnsureToolOptions, type EnsureToolPlan, type EnsureToolResult, type ErrorHandlingMode, EvalBuilder, type EvalCaseDefinition, type EvalCaseInput, type EvalClient, type EvalDefinition, EvalEndpoint, type EvalListParams, type EvalMessage, type EvalOptions, type EvalPullResult, type EvalRecord, type EvalRequest, type EvalResult, type EvalRunCaseScores, type EvalRunConfig, type EvalRunScores, EvalRunner, type EvalStatus, type EvalSuiteCase, type EvalSuiteCaseInput, type EvalSuiteDetail, type EvalSuiteLatestRun, type EvalSuiteListResult, type EvalSuiteRunQueued, type EvalSuiteRunResult, type EvalSuiteSummary, EvalSuitesNamespace, type EvalTarget, EvalsNamespace, type ExecuteToolRequest, type ExecuteToolResponse, type ExecutionStreamEvent, type ExternalAgentContext, type ExternalToolConfig, type FallbackFailEvent, type FallbackStartEvent, type FallbackSuccessEvent, type FallbackTrigger, type FallbackTriggerType, type FallbacksExhaustedEvent, type FallbacksInitiatedEvent, type FetchGitHubStepConfig$1 as FetchGitHubStepConfig, type FetchUrlStepConfig$1 as FetchUrlStepConfig, type FieldFormat, type FileContentPart, type Flow, type FlowAttachment, FlowBuilder, type FlowCompleteEvent, type FlowConfig$1 as FlowConfig, type FlowDefinition, type FlowDefinitionStep, FlowDriftError, FlowEnsureConflictError, type FlowErrorEvent, type FlowFallback, type FlowInlineEvalInput, type FlowListItem, type FlowPausedEvent, type FlowPullResult, FlowResult, type FlowStartEvent, type FlowStep, type FlowStepDefinition, type FlowStepType, FlowStepsEndpoint, type FlowStreamEvent, type FlowSummary, type FlowToolConfig, type FlowValidationClient, type FlowValidationIssue, type FlowValidationResult, type FlowVersionDetail, type FlowVersionListItem, type FlowVersionPublishResponse, FlowVersionsEndpoint, type FlowVersionsListResponse, FlowsEndpoint, FlowsNamespace, type FpoEntityOutcome, type FpoInput, type GenerateEmbeddingStepConfig$1 as GenerateEmbeddingStepConfig, type GeneratedRuntimeToolGateDecision, type GeneratedRuntimeToolGateOptions, type GetRecordStepConfig$1 as GetRecordStepConfig, type Gradeable, type GraderConfig, type GraderOutcome, type GraderSeverity, type ImageContentPart, type Integration, type IntegrationTool, IntegrationsEndpoint, type IntegrationsListResponse, type JSONSchema, type JsonArray, type JsonObject, type JsonPrimitive, type JsonValue, LEDGER_ARTIFACT_LINE_PREFIX, type ListConversationsResponse, type ListParams, type ListRecordsStepConfig$1 as ListRecordsStepConfig, type LocalToolConfig, type LocalToolDefinition, type LocalToolExecutionCompleteEvent, type LocalToolExecutionLoopSnapshotSlice, type LocalToolExecutionStartEvent, type LogEntry, type LogQueryParams, type LogQueryResponse, type LogQueryResult, type LogStatsParams, type LogStatsResponse, type LogStatsResult, LogsEndpoint, type Message$1 as Message, type MessageContent, type MessageFallback, type Metadata, type ModelConfig, ModelConfigsEndpoint, type ModelFallback, type ModelOverride, type ModelUsageDetail, type ModelUsageQueryParams, type ModelUsageResponse, type ModelUsageSummary, type ModelUsageTimeSeries, type PaginationResponse, type ProductDefinition, ProductDriftError, ProductEnsureConflictError, type ProductPullResult, ProductsNamespace, type Prompt$1 as Prompt, type PromptErrorHandling, type PromptFallback, type PromptListParams, type PromptStepConfig$1 as PromptStepConfig, PromptsEndpoint, PromptsNamespace, type ProviderApiKey, type ProviderKeyModel, ProviderKeysEndpoint, type PullFpoResult, RUNTYPE_CLIENT_KIND, type ReasoningConfig, type ReasoningContentPart, type ReasoningValue, type RecordConfig$1 as RecordConfig, type RecordCostAggregation, type RecordCostModelBreakdown, type RecordFilter, type RecordFilterCondition, type RecordFilterGroup, type RecordFilterOperator, type RecordListItem, type RecordListParams, type RecordStepResult, type RecordStepResultsParams, type RecordStepResultsResponse, type RecordWriteResponse, RecordsEndpoint, type RetrieveRecordStepConfig$1 as RetrieveRecordStepConfig, type RetryFallback, type RunEvalCaseResult, type RunEvalInput, type RunEvalResult, type RunTaskContextBudgetBreakdown, type RunTaskContextCompactionEvent, type RunTaskContextCompactionStrategy, type RunTaskContextNoticeEvent, type RunTaskContextSummaryEntry, type RunTaskContinuation, type RunTaskOffloadRecorder, type RunTaskOnContextCompaction, type RunTaskOnContextNotice, type RunTaskOnSession, type RunTaskOptions, type RunTaskResult, type RunTaskResumeState, type RunTaskSessionSummary, type RunTaskState, type RunTaskStateSlice, type RunTaskStatus, type RunTaskToolTraceSlice, type RuntimeCustomToolConfig, type RuntimeExternalToolConfig, type RuntimeFlowToolConfig, type RuntimeLocalToolConfig, type RuntimeSubagentToolConfig, type RuntimeTool, type RuntimeToolConfig, Runtype, type AgentSkillBinding as RuntypeAgentSkillBinding, RuntypeApiError, RuntypeClient, type ConditionalStepConfig as RuntypeConditionalStepConfig, type RuntypeConfig, type FetchGitHubStepConfig as RuntypeFetchGitHubStepConfig, type FetchUrlStepConfig as RuntypeFetchUrlStepConfig, RuntypeFlowBuilder, type FlowConfig as RuntypeFlowConfig, type GenerateEmbeddingStepConfig as RuntypeGenerateEmbeddingStepConfig, type GetRecordStepConfig as RuntypeGetRecordStepConfig, type ListRecordsStepConfig as RuntypeListRecordsStepConfig, type Message as RuntypeMessage, type ModelOverride$1 as RuntypeModelOverride, type Prompt as RuntypePrompt, type PromptStepConfig as RuntypePromptStepConfig, type RuntypeRecord, type RecordConfig as RuntypeRecordConfig, type RetrieveRecordStepConfig as RuntypeRetrieveRecordStepConfig, type SearchStepConfig as RuntypeSearchStepConfig, type SendEmailStepConfig as RuntypeSendEmailStepConfig, type SendEventStepConfig as RuntypeSendEventStepConfig, type SendStreamStepConfig as RuntypeSendStreamStepConfig, type SendTextStepConfig as RuntypeSendTextStepConfig, type SetVariableStepConfig as RuntypeSetVariableStepConfig, type Skill as RuntypeSkill, type SkillCapabilities as RuntypeSkillCapabilities, type SkillFrontmatter as RuntypeSkillFrontmatter, type SkillManifest as RuntypeSkillManifest, type SkillProposal as RuntypeSkillProposal, type SkillRuntypeExtensions as RuntypeSkillRuntypeExtensions, type SkillScanFinding as RuntypeSkillScanFinding, type SkillScanResult as RuntypeSkillScanResult, type SkillScanVerdict as RuntypeSkillScanVerdict, type SkillVersion as RuntypeSkillVersion, type TransformDataStepConfig as RuntypeTransformDataStepConfig, type UpsertFlowConfig as RuntypeUpsertFlowConfig, type UpsertRecordStepConfig as RuntypeUpsertRecordStepConfig, type VectorSearchStepConfig as RuntypeVectorSearchStepConfig, type WaitUntilStepConfig as RuntypeWaitUntilStepConfig, SDK_USER_AGENT, SDK_VERSION, STEP_FIELD_REGISTRY, STEP_TYPE_TO_METHOD, type Schedule, type ScheduleExecutionOptions, type ScheduleListParams, type ScheduleMessage, type ScheduleMessageSet, type ScheduleMessages, type ScheduleMutationResponse, type ScheduleRun, type ScheduleRunNowResponse, type ScheduleStatusResponse, type ScheduleTarget, type ScheduleTrigger, SchedulesEndpoint, type SearchStepConfig$1 as SearchStepConfig, type Secret, type SecretCheckResponse, type SecretDeleteResponse, type SecretSetupUrlRequest, type SecretSetupUrlResponse, SecretsEndpoint, type SendEmailStepConfig$1 as SendEmailStepConfig, type SendEventStepConfig$1 as SendEventStepConfig, type SendStreamStepConfig$1 as SendStreamStepConfig, type SendTextStepConfig$1 as SendTextStepConfig, type SetVariableStepConfig$1 as SetVariableStepConfig, type SkillDefinition, SkillDriftError, SkillEnsureConflictError, type SkillListPage, type SkillListPagination, type SkillListParams, type SkillManifestInput, type SkillMarkdownInput, type SkillOrigin, type SkillProposalStatus, SkillProposalsNamespace, type SkillPullResult, type SkillStatus, type SkillTrustLevel, type SkillVersionStatus, type SkillWithVersion, type SkillWriteInput, SkillsNamespace, type SlackInstallRequest, type StepCompleteEvent, type StepDeltaEvent, type StepFallback, type StepFieldMeta, type StepStartEvent, type StepWaitingLocalEvent, type StreamCallbacks, type StreamConsumeOptions, type StreamEvent, type StreamEventOf, type SubagentToolConfig, type Surface, type SurfaceDefinition, type SurfaceDefinitionEnvironment, type SurfaceDefinitionStatus, type SurfaceDefinitionType, SurfaceDriftError, SurfaceEnsureConflictError, type SurfaceListParams, type SurfacePullResult, SurfacesEndpoint, SurfacesNamespace, type TextContentPart, type Tool, type ToolApprovalGrant, ToolApprovalGrantsEndpoint, type ToolConfig, type ToolDefinition, type ToolDefinitionType, ToolDriftError, ToolEnsureConflictError, type ToolPullResult, type ToolWithValidation, type ToolsConfig, ToolsEndpoint, ToolsNamespace, type TransformDataStepConfig$1 as TransformDataStepConfig, UNIFIED_EVENTS_QUERY, type UpdateAppRequest, type UpdateClientTokenRequest, type UpdateConversationRequest, type UpdateEvalCaseInput, type UpdateEvalSuiteInput, type UpdatePromptData, type UpdateProviderKeyRequest, type UpdateScheduleRequest, type UpdateSecretRequest, type UpdateToolRequest, type UpdatedFlow, type UpsertFlowConfig$1 as UpsertFlowConfig, type UpsertOptions, type UpsertRecordStepConfig$1 as UpsertRecordStepConfig, type UserProfile, UsersEndpoint, type VectorSearchStepConfig$1 as VectorSearchStepConfig, type VersionType, type WaitUntilStepConfig$1 as WaitUntilStepConfig, type WorkflowCompileDeps, type WorkflowCompletionCriteriaConfig, type WorkflowConfig, type WorkflowConfigFactory, type WorkflowContext, type WorkflowDefinition, type WorkflowHookEntry, type WorkflowHookKind, type WorkflowHookRef, type WorkflowHookSignatures, type WorkflowMilestoneConfig, type WorkflowPhase, type WorkflowPolicyConfig, type WorkflowRecoveryConfig, type WorkflowSlot, type WorkflowStallPolicy, applyGeneratedRuntimeToolProposalToDispatchRequest, attachRuntimeToolsToDispatchRequest, buildEmptySessionNudge, buildGeneratedRuntimeToolGateOutput, buildLedgerOffloadReference, buildPolicyGuidance, buildSendViewOffloadMarker, calledTool, compileWorkflowConfig, completed, computeAgentContentHash, computeEvalContentHash, computeFlowContentHash, computeFpoContentHash, computeProductContentHash, computeSkillContentHash, computeSurfaceContentHash, computeToolContentHash, contains, cost, createAgentEventTranslator, createClient, createExternalTool, createFlowEventTranslator, defaultWorkflow, defaultWorkflowConfig, defineAgent, defineEval, defineFlow, defineFpo, definePlaybook, defineProduct, defineSkill, defineSurface, defineTool, deployWorkflow, ensureDefaultWorkflowHooks, ensureEval, ensureFpo, evaluateGeneratedRuntimeToolProposal, extractDeclaredToolResultChars, gameWorkflow, getDefaultPlanPath, getLikelySupportingCandidatePaths, interpolateWorkflowTemplate, isDiscoveryToolName, isMarathonArtifactPath, isPreservationSensitiveTask, isUnifiedEventType, isWorkflowHookRef, jsonField, judge, judges, latency, length, listWorkflowHooks, matchesExpected, maxToolCalls, noError, normalizeAgentDefinition, normalizeCandidatePath, normalizeFpoDefinition, normalizeProductDefinition, normalizeSkillDefinition, normalizeSurfaceDefinition, normalizeToolDefinition, notCalledTool, notContains, parseFinalBuffer, parseLedgerArtifactRelativePath, parseOffloadedOutputId, parseSSEChunk, processStream, pullEval, pullFpo, ranStep, regex, registerWorkflowHook, resolveStallStopAfter, resolveWorkflowHook, runEvalSuite, sanitizeTaskSlug, shouldInjectEmptySessionNudge, shouldRequestModelEscalation, stepOrder, streamEvents, toolOrder, unregisterWorkflowHook, usedNoTools, validJson, withUnifiedEvents };
|
|
51237
|
+
export { type AIGrader, type Agent, type AgentApprovalCompleteEvent, type AgentApprovalStartEvent, type AgentCompleteEvent, type AgentDefinition, type AgentDefinitionConfig, AgentDriftError, AgentEnsureConflictError, type AgentErrorEvent, type AgentEvent, type AgentEventType, type AgentExecuteRequest, type AgentExecuteResponse, type AgentIterationCompleteEvent, type AgentIterationStartEvent, type AgentMediaEvent, type AgentMessage, type AgentPausedEvent, type AgentPingEvent, type AgentPullResult, type AgentReflectionEvent, type AgentRuntimeToolDefinition, type AgentStartEvent, type AgentStreamCallbacks, type AgentStreamEvent, type AgentSubagentConfig, type AgentToolCompleteEvent, type AgentToolDeltaEvent, type AgentToolInputCompleteEvent, type AgentToolInputDeltaEvent, type AgentToolStartEvent, type AgentTurnCompleteEvent, type AgentTurnDeltaEvent, type AgentTurnStartEvent, type AgentVersionDetail, type AgentVersionListItem, type AgentVersionPublishResponse, AgentVersionsEndpoint, type AgentVersionsListResponse, AgentsEndpoint, AgentsNamespace, AnalyticsEndpoint, type ApiClient, type ApiKey, ApiKeysEndpoint, type ApiResponse, type App, type AppManifest, type AppVersion, type ApplyGeneratedProposalOptions, type ApplyGeneratedProposalResult, AppsEndpoint, type AssetReferenceContentPart, type AttachRuntimeToolsOptions, type BaseAgentEvent, BatchBuilder, type BatchClient, type BatchListParams, type BatchOptions, type BatchRequest, type BatchResult, type BatchScheduleConfig, type BatchStatus, BatchesNamespace, BillingEndpoint, type BillingSpendAnalyticsParams, type BindSkillInput, type BuiltInGraderId, type BuiltInTool, type BulkEditCondition, type BulkEditRequest, type BulkEditResponse, type BulkEditResult, type CaseExpected, ChatEndpoint, type CheckGrader, ClientBatchBuilder, type ClientConfig, type ClientConversation, ClientEvalBuilder, ClientFlowBuilder, type ClientToken, type ClientTokenConfig, type ClientTokenEnvironment, type ClientTokenVersionPin, ClientTokensEndpoint, type ClientToolDefinition, type ClientWidgetTheme, type ConditionalGetResult, type ConditionalStepConfig$1 as ConditionalStepConfig, type ContextErrorHandling, type ContextFallback, ContextTemplatesEndpoint, type Conversation, type ConversationListItem, type ConversationListParams, type ConversationMessage, type ConversationSource, ConversationsEndpoint, type ConversationsListResponse, type CreateApiKeyRequest, type CreateAppRequest, type CreateClientTokenRequest, type CreateClientTokenResponse, type CreateConversationRequest, type CreateEvalSuiteInput, type CreateFlowRequest, type CreateModelConfigRequest, type CreatePromptData, type CreatePromptRequest, type CreateProviderKeyRequest, type CreateRecordRequest, type CreateScheduleRequest, type CreateSecretRequest, type CreateToolRequest, type CustomMCPServer, type CustomMCPServerAuth, type CustomToolConfig, DEFAULT_RECOVERY_AFTER_EMPTY_SESSIONS, DEFAULT_STALL_STOP_AFTER, type DecomposeCriteriaResult, type DefineAgentInput, type DefineEvalCaseInput, type DefineEvalInput, type DefineFlowInput, type DefineProductInput, type DefineSkillInput, type DefineSurfaceInput, type DefineToolInput, type DeployCfSandboxRequest, type DeployCfSandboxResponse, type DeploySandboxRequest, type DeploySandboxResponse, type DiscoveredModel, type DispatchClient, DispatchEndpoint, type DispatchEnvironment, type DispatchEvent, type DispatchOptions$1 as DispatchOptions, type DispatchRequest, type EnsureAgentConverged, type EnsureAgentOptions, type EnsureAgentPlan, type EnsureAgentResult, type EnsureEvalResult, type EnsureFlowConverged, type EnsureFlowOptions, type EnsureFlowPlan, type EnsureFlowResult, type EnsureFpoOptions, type EnsureFpoResult, type EnsureProductConverged, type EnsureProductOptions, type EnsureProductPlan, type EnsureProductResult, type EnsureSkillConverged, type EnsureSkillOptions, type EnsureSkillPlan, type EnsureSkillResult, type EnsureSurfaceConverged, type EnsureSurfaceOptions, type EnsureSurfacePlan, type EnsureSurfaceResult, type EnsureToolConverged, type EnsureToolOptions, type EnsureToolPlan, type EnsureToolResult, type ErrorHandlingMode, EvalBuilder, type EvalCaseDefinition, type EvalCaseInput, type EvalClient, type EvalDefinition, EvalEndpoint, type EvalListParams, type EvalMessage, type EvalOptions, type EvalPullResult, type EvalRecord, type EvalRequest, type EvalResult, type EvalRunCaseScores, type EvalRunConfig, type EvalRunScores, EvalRunner, type EvalStatus, type EvalSuiteCase, type EvalSuiteCaseInput, type EvalSuiteDetail, type EvalSuiteLatestRun, type EvalSuiteListResult, type EvalSuiteRunQueued, type EvalSuiteRunResult, type EvalSuiteSummary, EvalSuitesNamespace, type EvalTarget, EvalsNamespace, type ExecuteToolRequest, type ExecuteToolResponse, type ExecutionStreamEvent, type ExternalAgentContext, type ExternalToolConfig, type FallbackFailEvent, type FallbackStartEvent, type FallbackSuccessEvent, type FallbackTrigger, type FallbackTriggerType, type FallbacksExhaustedEvent, type FallbacksInitiatedEvent, type FetchGitHubStepConfig$1 as FetchGitHubStepConfig, type FetchUrlStepConfig$1 as FetchUrlStepConfig, type FieldFormat, type FileContentPart, type Flow, type FlowAttachment, FlowBuilder, type FlowCompleteEvent, type FlowConfig$1 as FlowConfig, type FlowDefinition, type FlowDefinitionStep, FlowDriftError, FlowEnsureConflictError, type FlowErrorEvent, type FlowFallback, type FlowInlineEvalInput, type FlowListItem, type FlowPausedEvent, type FlowPullResult, FlowResult, type FlowStartEvent, type FlowStep, type FlowStepDefinition, type FlowStepType, FlowStepsEndpoint, type FlowStreamEvent, type FlowSummary, type FlowToolConfig, type FlowValidationClient, type FlowValidationIssue, type FlowValidationResult, type FlowVersionDetail, type FlowVersionListItem, type FlowVersionPublishResponse, FlowVersionsEndpoint, type FlowVersionsListResponse, FlowsEndpoint, FlowsNamespace, type FpoEntityOutcome, type FpoInput, type GenerateEmbeddingStepConfig$1 as GenerateEmbeddingStepConfig, type GeneratedRuntimeToolGateDecision, type GeneratedRuntimeToolGateOptions, type GetRecordStepConfig$1 as GetRecordStepConfig, type Gradeable, type GraderConfig, type GraderOutcome, type GraderSeverity, type ImageContentPart, type Integration, type IntegrationTool, IntegrationsEndpoint, type IntegrationsListResponse, type JSONSchema, type JsonArray, type JsonObject, type JsonPrimitive, type JsonValue, LEDGER_ARTIFACT_LINE_PREFIX, type ListConversationsResponse, type ListParams, type ListRecordsStepConfig$1 as ListRecordsStepConfig, type LocalToolConfig, type LocalToolDefinition, type LocalToolExecutionCompleteEvent, type LocalToolExecutionLoopSnapshotSlice, type LocalToolExecutionStartEvent, type LogEntry, type LogQueryParams, type LogQueryResponse, type LogQueryResult, type LogStatsParams, type LogStatsResponse, type LogStatsResult, LogsEndpoint, type Message$1 as Message, type MessageContent, type MessageFallback, type Metadata, type ModelConfig, ModelConfigsEndpoint, type ModelFallback, type ModelOverride, type ModelUsageDetail, type ModelUsageQueryParams, type ModelUsageResponse, type ModelUsageSummary, type ModelUsageTimeSeries, type PaginationResponse, type ProductDefinition, ProductDriftError, ProductEnsureConflictError, type ProductPullResult, ProductsNamespace, type Prompt$1 as Prompt, type PromptErrorHandling, type PromptFallback, type PromptListParams, type PromptStepConfig$1 as PromptStepConfig, PromptsEndpoint, PromptsNamespace, type ProviderApiKey, type ProviderKeyModel, ProviderKeysEndpoint, type PullFpoResult, RUNTYPE_CLIENT_KIND, type ReasoningConfig, type ReasoningContentPart, type ReasoningValue, type RecordConfig$1 as RecordConfig, type RecordCostAggregation, type RecordCostModelBreakdown, type RecordFilter, type RecordFilterCondition, type RecordFilterGroup, type RecordFilterOperator, type RecordListItem, type RecordListParams, type RecordStepResult, type RecordStepResultsParams, type RecordStepResultsResponse, type RecordWriteResponse, RecordsEndpoint, type RetrieveRecordStepConfig$1 as RetrieveRecordStepConfig, type RetryFallback, type RunEvalCaseResult, type RunEvalInput, type RunEvalResult, type RunTaskContextBudgetBreakdown, type RunTaskContextCompactionEvent, type RunTaskContextCompactionStrategy, type RunTaskContextNoticeEvent, type RunTaskContextSummaryEntry, type RunTaskContinuation, type RunTaskOffloadRecorder, type RunTaskOnContextCompaction, type RunTaskOnContextNotice, type RunTaskOnSession, type RunTaskOptions, type RunTaskResult, type RunTaskResumeState, type RunTaskSessionSummary, type RunTaskState, type RunTaskStateSlice, type RunTaskStatus, type RunTaskToolTraceSlice, type RuntimeCustomToolConfig, type RuntimeExternalToolConfig, type RuntimeFlowToolConfig, type RuntimeLocalToolConfig, type RuntimeSubagentToolConfig, type RuntimeTool, type RuntimeToolConfig, Runtype, type AgentSkillBinding as RuntypeAgentSkillBinding, RuntypeApiError, RuntypeClient, type ConditionalStepConfig as RuntypeConditionalStepConfig, type RuntypeConfig, type FetchGitHubStepConfig as RuntypeFetchGitHubStepConfig, type FetchUrlStepConfig as RuntypeFetchUrlStepConfig, RuntypeFlowBuilder, type FlowConfig as RuntypeFlowConfig, type GenerateEmbeddingStepConfig as RuntypeGenerateEmbeddingStepConfig, type GetRecordStepConfig as RuntypeGetRecordStepConfig, type ListRecordsStepConfig as RuntypeListRecordsStepConfig, type Message as RuntypeMessage, type ModelOverride$1 as RuntypeModelOverride, type Prompt as RuntypePrompt, type PromptStepConfig as RuntypePromptStepConfig, type RuntypeRecord, type RecordConfig as RuntypeRecordConfig, type RetrieveRecordStepConfig as RuntypeRetrieveRecordStepConfig, type SearchStepConfig as RuntypeSearchStepConfig, type SendEmailStepConfig as RuntypeSendEmailStepConfig, type SendEventStepConfig as RuntypeSendEventStepConfig, type SendStreamStepConfig as RuntypeSendStreamStepConfig, type SendTextStepConfig as RuntypeSendTextStepConfig, type SetVariableStepConfig as RuntypeSetVariableStepConfig, type Skill as RuntypeSkill, type SkillCapabilities as RuntypeSkillCapabilities, type SkillFrontmatter as RuntypeSkillFrontmatter, type SkillManifest as RuntypeSkillManifest, type SkillProposal as RuntypeSkillProposal, type SkillRuntypeExtensions as RuntypeSkillRuntypeExtensions, type SkillScanFinding as RuntypeSkillScanFinding, type SkillScanResult as RuntypeSkillScanResult, type SkillScanVerdict as RuntypeSkillScanVerdict, type SkillVersion as RuntypeSkillVersion, type TransformDataStepConfig as RuntypeTransformDataStepConfig, type UpsertFlowConfig as RuntypeUpsertFlowConfig, type UpsertRecordStepConfig as RuntypeUpsertRecordStepConfig, type VectorSearchStepConfig as RuntypeVectorSearchStepConfig, type WaitUntilStepConfig as RuntypeWaitUntilStepConfig, SDK_USER_AGENT, SDK_VERSION, STEP_FIELD_REGISTRY, STEP_TYPE_TO_METHOD, type Schedule, type ScheduleExecutionOptions, type ScheduleListParams, type ScheduleMessage, type ScheduleMessageSet, type ScheduleMessages, type ScheduleMutationResponse, type ScheduleRun, type ScheduleRunNowResponse, type ScheduleStatusResponse, type ScheduleTarget, type ScheduleTrigger, SchedulesEndpoint, type SearchStepConfig$1 as SearchStepConfig, type Secret, type SecretCheckResponse, type SecretDeleteResponse, type SecretSetupUrlRequest, type SecretSetupUrlResponse, SecretsEndpoint, type SendEmailStepConfig$1 as SendEmailStepConfig, type SendEventStepConfig$1 as SendEventStepConfig, type SendStreamStepConfig$1 as SendStreamStepConfig, type SendTextStepConfig$1 as SendTextStepConfig, type SetVariableStepConfig$1 as SetVariableStepConfig, type SkillDefinition, SkillDriftError, SkillEnsureConflictError, type SkillListPage, type SkillListPagination, type SkillListParams, type SkillManifestInput, type SkillMarkdownInput, type SkillOrigin, type SkillProposalStatus, SkillProposalsNamespace, type SkillPullResult, type SkillStatus, type SkillTrustLevel, type SkillVersionStatus, type SkillWithVersion, type SkillWriteInput, SkillsNamespace, type SlackInstallRequest, type StepCompleteEvent, type StepDeltaEvent, type StepFallback, type StepFieldMeta, type StepStartEvent, type StepWaitingLocalEvent, type StreamCallbacks, type StreamConsumeOptions, type StreamEvent, type StreamEventOf, type SubagentToolConfig, type Surface, type SurfaceDefinition, type SurfaceDefinitionEnvironment, type SurfaceDefinitionStatus, type SurfaceDefinitionType, SurfaceDriftError, SurfaceEnsureConflictError, type SurfaceListParams, type SurfacePullResult, SurfacesEndpoint, SurfacesNamespace, type TextContentPart, type Tool, type ToolApprovalGrant, ToolApprovalGrantsEndpoint, type ToolConfig, type ToolDefinition, type ToolDefinitionType, ToolDriftError, ToolEnsureConflictError, type ToolPullResult, type ToolWithValidation, type ToolsConfig, ToolsEndpoint, ToolsNamespace, type TransformDataStepConfig$1 as TransformDataStepConfig, UNIFIED_EVENTS_QUERY, type UpdateAppRequest, type UpdateClientTokenRequest, type UpdateConversationRequest, type UpdateEvalCaseInput, type UpdateEvalSuiteInput, type UpdatePromptData, type UpdateProviderKeyRequest, type UpdateScheduleRequest, type UpdateSecretRequest, type UpdateToolRequest, type UpdatedFlow, type UpsertFlowConfig$1 as UpsertFlowConfig, type UpsertOptions, type UpsertRecordStepConfig$1 as UpsertRecordStepConfig, type UserProfile, UsersEndpoint, type VectorSearchStepConfig$1 as VectorSearchStepConfig, type VersionType, type WaitUntilStepConfig$1 as WaitUntilStepConfig, type WorkflowCompileDeps, type WorkflowCompletionCriteriaConfig, type WorkflowConfig, type WorkflowConfigFactory, type WorkflowContext, type WorkflowDefinition, type WorkflowHookEntry, type WorkflowHookKind, type WorkflowHookRef, type WorkflowHookSignatures, type WorkflowMilestoneConfig, type WorkflowPhase, type WorkflowPolicyConfig, type WorkflowRecoveryConfig, type WorkflowSlot, type WorkflowStallPolicy, applyGeneratedRuntimeToolProposalToDispatchRequest, attachRuntimeToolsToDispatchRequest, buildEmptySessionNudge, buildGeneratedRuntimeToolGateOutput, buildLedgerOffloadReference, buildPolicyGuidance, buildSendViewOffloadMarker, calledTool, compileWorkflowConfig, completed, computeAgentContentHash, computeEvalContentHash, computeFlowContentHash, computeFpoContentHash, computeProductContentHash, computeSkillContentHash, computeSurfaceContentHash, computeToolContentHash, contains, cost, createAgentEventTranslator, createClient, createExternalTool, createFlowEventTranslator, defaultWorkflow, defaultWorkflowConfig, defineAgent, defineEval, defineFlow, defineFpo, definePlaybook, defineProduct, defineSkill, defineSurface, defineTool, deployWorkflow, ensureDefaultWorkflowHooks, ensureEval, ensureFpo, evaluateGeneratedRuntimeToolProposal, extractDeclaredToolResultChars, gameWorkflow, getDefaultPlanPath, getLikelySupportingCandidatePaths, interpolateWorkflowTemplate, isDiscoveryToolName, isMarathonArtifactPath, isPreservationSensitiveTask, isUnifiedEventType, isWorkflowHookRef, jsonField, judge, judges, latency, length, listWorkflowHooks, matchesExpected, maxToolCalls, noError, normalizeAgentDefinition, normalizeCandidatePath, normalizeFpoDefinition, normalizeProductDefinition, normalizeSkillDefinition, normalizeSurfaceDefinition, normalizeToolDefinition, notCalledTool, notContains, parseFinalBuffer, parseLedgerArtifactRelativePath, parseOffloadedOutputId, parseSSEChunk, processStream, pullEval, pullFpo, ranStep, regex, registerWorkflowHook, resolveStallStopAfter, resolveWorkflowHook, runEvalSuite, sanitizeTaskSlug, shouldInjectEmptySessionNudge, shouldRequestModelEscalation, stepOrder, streamEvents, toolOrder, unregisterWorkflowHook, usedNoTools, validJson, withUnifiedEvents };
|