@vitest-evals/core 0.16.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +303 -3
- package/dist/index.d.ts +303 -3
- package/dist/index.js +23 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +23 -3
- package/dist/index.mjs.map +1 -1
- package/dist/node.d.mts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +39 -30
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +39 -30
- package/dist/node.mjs.map +1 -1
- package/dist/{workspace-DSZKph1P.d.mts → workspace-BrBLUPEb.d.mts} +200 -2
- package/dist/{workspace-DSZKph1P.d.ts → workspace-BrBLUPEb.d.ts} +200 -2
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { J as JsonValue } from './workspace-
|
|
2
|
-
export { C as CollectReportWorkspaceOptions, a as JsonObjectSchema, d as JsonPrimitive, b as JsonPrimitiveSchema, c as JsonValueSchema, R as REPORT_WORKSPACE_SCHEMA_VERSION, t as ReportCase, q as ReportCaseSchema, u as ReportRun, r as ReportRunSchema, v as ReportWorkspace, w as ReportWorkspaceInput, s as ReportWorkspaceSchema, i as VitestJsonAssertion, V as VitestJsonAssertionSchema, j as VitestJsonFile, e as VitestJsonFileSchema, k as VitestJsonLocation, f as VitestJsonLocationSchema, l as VitestJsonReport, g as VitestJsonReportSchema, m as VitestJsonStatus, h as VitestJsonStatusSchema, n as collectReportWorkspace, o as parseReportWorkspace, p as parseVitestJsonReport } from './workspace-
|
|
1
|
+
import { J as JsonValue } from './workspace-BrBLUPEb.mjs';
|
|
2
|
+
export { C as CollectReportWorkspaceOptions, a as JsonObjectSchema, d as JsonPrimitive, b as JsonPrimitiveSchema, c as JsonValueSchema, R as REPORT_WORKSPACE_SCHEMA_VERSION, t as ReportCase, q as ReportCaseSchema, u as ReportRun, r as ReportRunSchema, v as ReportWorkspace, w as ReportWorkspaceInput, s as ReportWorkspaceSchema, i as VitestJsonAssertion, V as VitestJsonAssertionSchema, j as VitestJsonFile, e as VitestJsonFileSchema, k as VitestJsonLocation, f as VitestJsonLocationSchema, l as VitestJsonReport, g as VitestJsonReportSchema, m as VitestJsonStatus, h as VitestJsonStatusSchema, n as collectReportWorkspace, o as parseReportWorkspace, p as parseVitestJsonReport } from './workspace-BrBLUPEb.mjs';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
|
|
5
5
|
/** Well-known OpenTelemetry GenAI operation names. */
|
|
@@ -299,6 +299,7 @@ declare const UsageSummarySchema: z.ZodObject<{
|
|
|
299
299
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
300
300
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
301
301
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
302
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
302
303
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
303
304
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
304
305
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
@@ -317,11 +318,13 @@ type UsageSummary = {
|
|
|
317
318
|
reasoningTokens?: number;
|
|
318
319
|
/** Total token count reported by the provider or adapter. */
|
|
319
320
|
totalTokens?: number;
|
|
321
|
+
/** USD-denominated cost attributed to this run. Omit when unknown; zero means known free. */
|
|
322
|
+
costUsd?: number;
|
|
320
323
|
/** Count of tool calls observed during the run. */
|
|
321
324
|
toolCalls?: number;
|
|
322
325
|
/** Retry count observed during the run. */
|
|
323
326
|
retries?: number;
|
|
324
|
-
/** Provider-specific JSON-safe usage details.
|
|
327
|
+
/** Provider-specific JSON-safe usage details. */
|
|
325
328
|
metadata?: Record<string, JsonValue>;
|
|
326
329
|
};
|
|
327
330
|
/** Timing values normalized by vitest-evals harnesses. */
|
|
@@ -607,6 +610,7 @@ declare const HarnessRunSchema: z.ZodObject<{
|
|
|
607
610
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
608
611
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
609
612
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
613
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
610
614
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
611
615
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
612
616
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
@@ -877,6 +881,7 @@ declare const HarnessMetaSchema: z.ZodObject<{
|
|
|
877
881
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
878
882
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
879
883
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
884
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
880
885
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
881
886
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
882
887
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
@@ -935,6 +940,104 @@ type HarnessMeta = z.infer<typeof HarnessMetaSchema>;
|
|
|
935
940
|
declare const EvalScoreSchema: z.ZodObject<{
|
|
936
941
|
name: z.ZodOptional<z.ZodString>;
|
|
937
942
|
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
943
|
+
judgeRuns: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
944
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
945
|
+
session: z.ZodObject<{
|
|
946
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
947
|
+
type: z.ZodLiteral<"message">;
|
|
948
|
+
role: z.ZodEnum<{
|
|
949
|
+
system: "system";
|
|
950
|
+
user: "user";
|
|
951
|
+
assistant: "assistant";
|
|
952
|
+
}>;
|
|
953
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
954
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
955
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
956
|
+
type: z.ZodLiteral<"tool_call">;
|
|
957
|
+
id: z.ZodString;
|
|
958
|
+
name: z.ZodString;
|
|
959
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
960
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
961
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
962
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
963
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
964
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
965
|
+
type: z.ZodLiteral<"tool_result">;
|
|
966
|
+
toolCallId: z.ZodString;
|
|
967
|
+
name: z.ZodOptional<z.ZodString>;
|
|
968
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
969
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
970
|
+
message: z.ZodString;
|
|
971
|
+
type: z.ZodOptional<z.ZodString>;
|
|
972
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
973
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
974
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
975
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
976
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
977
|
+
}, z.core.$strict>], "type">>;
|
|
978
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
979
|
+
model: z.ZodOptional<z.ZodString>;
|
|
980
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
981
|
+
}, z.core.$strict>;
|
|
982
|
+
usage: z.ZodObject<{
|
|
983
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
984
|
+
model: z.ZodOptional<z.ZodString>;
|
|
985
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
986
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
987
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
988
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
989
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
990
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
991
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
992
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
993
|
+
}, z.core.$strict>;
|
|
994
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
995
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
996
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
997
|
+
}, z.core.$strict>>;
|
|
998
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
999
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1000
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1001
|
+
name: z.ZodOptional<z.ZodString>;
|
|
1002
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1003
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1004
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1005
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1006
|
+
spans: z.ZodArray<z.ZodObject<{
|
|
1007
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1008
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
1009
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
1010
|
+
name: z.ZodString;
|
|
1011
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
1012
|
+
custom: "custom";
|
|
1013
|
+
tool: "tool";
|
|
1014
|
+
model: "model";
|
|
1015
|
+
run: "run";
|
|
1016
|
+
agent: "agent";
|
|
1017
|
+
guardrail: "guardrail";
|
|
1018
|
+
handoff: "handoff";
|
|
1019
|
+
}>>;
|
|
1020
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1021
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1022
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1023
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
1024
|
+
error: "error";
|
|
1025
|
+
ok: "ok";
|
|
1026
|
+
}>>;
|
|
1027
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
1028
|
+
message: z.ZodString;
|
|
1029
|
+
type: z.ZodOptional<z.ZodString>;
|
|
1030
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
1031
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1032
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1033
|
+
name: z.ZodString;
|
|
1034
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
1035
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1036
|
+
}, z.core.$strict>>>;
|
|
1037
|
+
}, z.core.$strict>>;
|
|
1038
|
+
}, z.core.$strict>>>;
|
|
1039
|
+
errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1040
|
+
}, z.core.$strict>>>;
|
|
938
1041
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
939
1042
|
}, z.core.$strict>;
|
|
940
1043
|
/** Score record stored by vitest-evals on Vitest task metadata. */
|
|
@@ -944,6 +1047,104 @@ declare const EvalMetaSchema: z.ZodObject<{
|
|
|
944
1047
|
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
945
1048
|
name: z.ZodOptional<z.ZodString>;
|
|
946
1049
|
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
1050
|
+
judgeRuns: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1051
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1052
|
+
session: z.ZodObject<{
|
|
1053
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1054
|
+
type: z.ZodLiteral<"message">;
|
|
1055
|
+
role: z.ZodEnum<{
|
|
1056
|
+
system: "system";
|
|
1057
|
+
user: "user";
|
|
1058
|
+
assistant: "assistant";
|
|
1059
|
+
}>;
|
|
1060
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1061
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1062
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1063
|
+
type: z.ZodLiteral<"tool_call">;
|
|
1064
|
+
id: z.ZodString;
|
|
1065
|
+
name: z.ZodString;
|
|
1066
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1067
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1068
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1069
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1070
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1071
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1072
|
+
type: z.ZodLiteral<"tool_result">;
|
|
1073
|
+
toolCallId: z.ZodString;
|
|
1074
|
+
name: z.ZodOptional<z.ZodString>;
|
|
1075
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1076
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
1077
|
+
message: z.ZodString;
|
|
1078
|
+
type: z.ZodOptional<z.ZodString>;
|
|
1079
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
1080
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1081
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1082
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1083
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1084
|
+
}, z.core.$strict>], "type">>;
|
|
1085
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
1086
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1087
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1088
|
+
}, z.core.$strict>;
|
|
1089
|
+
usage: z.ZodObject<{
|
|
1090
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
1091
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1092
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
1093
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
1094
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
1095
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
1096
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
1097
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
1098
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
1099
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1100
|
+
}, z.core.$strict>;
|
|
1101
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
1102
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
1103
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1104
|
+
}, z.core.$strict>>;
|
|
1105
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1106
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1107
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1108
|
+
name: z.ZodOptional<z.ZodString>;
|
|
1109
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1110
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1111
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1112
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1113
|
+
spans: z.ZodArray<z.ZodObject<{
|
|
1114
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1115
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
1116
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
1117
|
+
name: z.ZodString;
|
|
1118
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
1119
|
+
custom: "custom";
|
|
1120
|
+
tool: "tool";
|
|
1121
|
+
model: "model";
|
|
1122
|
+
run: "run";
|
|
1123
|
+
agent: "agent";
|
|
1124
|
+
guardrail: "guardrail";
|
|
1125
|
+
handoff: "handoff";
|
|
1126
|
+
}>>;
|
|
1127
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1128
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1129
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1130
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
1131
|
+
error: "error";
|
|
1132
|
+
ok: "ok";
|
|
1133
|
+
}>>;
|
|
1134
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
1135
|
+
message: z.ZodString;
|
|
1136
|
+
type: z.ZodOptional<z.ZodString>;
|
|
1137
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
1138
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1139
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1140
|
+
name: z.ZodString;
|
|
1141
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
1142
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1143
|
+
}, z.core.$strict>>>;
|
|
1144
|
+
}, z.core.$strict>>;
|
|
1145
|
+
}, z.core.$strict>>>;
|
|
1146
|
+
errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1147
|
+
}, z.core.$strict>>>;
|
|
947
1148
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
948
1149
|
}, z.core.$strict>>>;
|
|
949
1150
|
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
@@ -976,6 +1177,104 @@ declare const EvalTaskMetaSchema: z.ZodObject<{
|
|
|
976
1177
|
scores: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
977
1178
|
name: z.ZodOptional<z.ZodString>;
|
|
978
1179
|
score: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
1180
|
+
judgeRuns: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1181
|
+
output: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1182
|
+
session: z.ZodObject<{
|
|
1183
|
+
events: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1184
|
+
type: z.ZodLiteral<"message">;
|
|
1185
|
+
role: z.ZodEnum<{
|
|
1186
|
+
system: "system";
|
|
1187
|
+
user: "user";
|
|
1188
|
+
assistant: "assistant";
|
|
1189
|
+
}>;
|
|
1190
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1191
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1192
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1193
|
+
type: z.ZodLiteral<"tool_call">;
|
|
1194
|
+
id: z.ZodString;
|
|
1195
|
+
name: z.ZodString;
|
|
1196
|
+
arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1197
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1198
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1199
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1200
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1201
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1202
|
+
type: z.ZodLiteral<"tool_result">;
|
|
1203
|
+
toolCallId: z.ZodString;
|
|
1204
|
+
name: z.ZodOptional<z.ZodString>;
|
|
1205
|
+
content: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
1206
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
1207
|
+
message: z.ZodString;
|
|
1208
|
+
type: z.ZodOptional<z.ZodString>;
|
|
1209
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
1210
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1211
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1212
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1213
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1214
|
+
}, z.core.$strict>], "type">>;
|
|
1215
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
1216
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1217
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1218
|
+
}, z.core.$strict>;
|
|
1219
|
+
usage: z.ZodObject<{
|
|
1220
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
1221
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1222
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
1223
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
1224
|
+
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
1225
|
+
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
1226
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
1227
|
+
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
1228
|
+
retries: z.ZodOptional<z.ZodNumber>;
|
|
1229
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1230
|
+
}, z.core.$strict>;
|
|
1231
|
+
timings: z.ZodOptional<z.ZodObject<{
|
|
1232
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
1233
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1234
|
+
}, z.core.$strict>>;
|
|
1235
|
+
artifacts: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1236
|
+
traces: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1237
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1238
|
+
name: z.ZodOptional<z.ZodString>;
|
|
1239
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1240
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1241
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1242
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1243
|
+
spans: z.ZodArray<z.ZodObject<{
|
|
1244
|
+
id: z.ZodOptional<z.ZodString>;
|
|
1245
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
1246
|
+
parentId: z.ZodOptional<z.ZodString>;
|
|
1247
|
+
name: z.ZodString;
|
|
1248
|
+
kind: z.ZodOptional<z.ZodEnum<{
|
|
1249
|
+
custom: "custom";
|
|
1250
|
+
tool: "tool";
|
|
1251
|
+
model: "model";
|
|
1252
|
+
run: "run";
|
|
1253
|
+
agent: "agent";
|
|
1254
|
+
guardrail: "guardrail";
|
|
1255
|
+
handoff: "handoff";
|
|
1256
|
+
}>>;
|
|
1257
|
+
startedAt: z.ZodOptional<z.ZodString>;
|
|
1258
|
+
finishedAt: z.ZodOptional<z.ZodString>;
|
|
1259
|
+
durationMs: z.ZodOptional<z.ZodNumber>;
|
|
1260
|
+
status: z.ZodOptional<z.ZodEnum<{
|
|
1261
|
+
error: "error";
|
|
1262
|
+
ok: "ok";
|
|
1263
|
+
}>>;
|
|
1264
|
+
error: z.ZodOptional<z.ZodObject<{
|
|
1265
|
+
message: z.ZodString;
|
|
1266
|
+
type: z.ZodOptional<z.ZodString>;
|
|
1267
|
+
}, z.core.$catchall<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>>;
|
|
1268
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1269
|
+
events: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1270
|
+
name: z.ZodString;
|
|
1271
|
+
timestamp: z.ZodOptional<z.ZodString>;
|
|
1272
|
+
attributes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1273
|
+
}, z.core.$strict>>>;
|
|
1274
|
+
}, z.core.$strict>>;
|
|
1275
|
+
}, z.core.$strict>>>;
|
|
1276
|
+
errors: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
1277
|
+
}, z.core.$strict>>>;
|
|
979
1278
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|
|
980
1279
|
}, z.core.$strict>>>;
|
|
981
1280
|
avgScore: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodOptional<z.ZodNullable<z.ZodNumber>>>;
|
|
@@ -1048,6 +1347,7 @@ declare const EvalTaskMetaSchema: z.ZodObject<{
|
|
|
1048
1347
|
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
1049
1348
|
reasoningTokens: z.ZodOptional<z.ZodNumber>;
|
|
1050
1349
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
1350
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
1051
1351
|
toolCalls: z.ZodOptional<z.ZodNumber>;
|
|
1052
1352
|
retries: z.ZodOptional<z.ZodNumber>;
|
|
1053
1353
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>>;
|