niceeval 0.19.0 → 0.19.1-canary.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +1 -1
- package/dist/assertions/record/codec.cjs +19 -13
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/codec.d.cts +2 -0
- package/dist/assertions/record/codec.d.mts +2 -0
- package/dist/assertions/record/codec.d.ts +2 -0
- package/dist/assertions/record/codec.js +16 -14
- package/dist/inspection/assertion-projection.cjs +4 -1
- package/dist/inspection/assertion-projection.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +18 -14
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/cli/contribution.d.cts +7 -0
- package/dist/inspection/cli/contribution.d.mts +7 -0
- package/dist/inspection/cli/contribution.d.ts +7 -0
- package/dist/inspection/overview.cjs +28 -14
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +151 -18
- package/dist/inspection/overview.d.mts +151 -18
- package/dist/inspection/overview.d.ts +151 -18
- package/dist/inspection/overview.js +4 -2
- package/dist/inspection/project-result.d.cts +148 -148
- package/dist/inspection/project-result.d.mts +148 -148
- package/dist/inspection/project-result.d.ts +148 -148
- package/dist/inspection/protocol-values.cjs +1 -1
- package/dist/inspection/protocol-values.cjs.map +1 -1
- package/dist/inspection/protocol-values.d.cts +1 -1
- package/dist/inspection/protocol-values.d.mts +1 -1
- package/dist/inspection/protocol-values.d.ts +1 -1
- package/dist/inspection/protocol.d.cts +611 -479
- package/dist/inspection/protocol.d.mts +611 -479
- package/dist/inspection/protocol.d.ts +611 -479
- package/dist/inspection/public.cjs +3 -1
- package/dist/inspection/public.cjs.map +1 -1
- package/dist/inspection/public.d.cts +1 -0
- package/dist/inspection/public.d.mts +1 -0
- package/dist/inspection/public.d.ts +1 -0
- package/dist/inspection/public.js +18 -16
- package/dist/inspection/public.mjs +54 -52
- package/dist/inspection/results.cjs +11 -15
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +769 -324
- package/dist/inspection/results.d.mts +769 -324
- package/dist/inspection/results.d.ts +769 -324
- package/dist/inspection/results.js +4 -0
- package/dist/inspection/select.cjs +12 -72
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/verdict.cjs +61 -0
- package/dist/inspection/verdict.cjs.map +1 -0
- package/dist/inspection/verdict.d.cts +3 -0
- package/dist/inspection/verdict.d.mts +3 -0
- package/dist/inspection/verdict.d.ts +3 -0
- package/dist/inspection/verdict.js +6 -0
- package/dist/inspection/verdict.js.map +1 -0
- package/dist/runner/discover.cjs +5 -4
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -2
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +2 -0
- package/dist/sandbox/index.d.mts +2 -0
- package/dist/sandbox/index.d.ts +2 -0
- package/dist/sandbox/index.js +34 -32
- package/dist/sandbox/index.mjs +34 -32
- package/dist/sandbox/opensandbox/managed-process.cjs +224 -0
- package/dist/sandbox/opensandbox/managed-process.cjs.map +1 -0
- package/dist/sandbox/opensandbox/managed-process.d.cts +9 -0
- package/dist/sandbox/opensandbox/managed-process.d.mts +9 -0
- package/dist/sandbox/opensandbox/managed-process.d.ts +9 -0
- package/dist/sandbox/opensandbox/managed-process.js +6 -0
- package/dist/sandbox/opensandbox/managed-process.js.map +1 -0
- package/dist/sandbox/opensandbox/materialize.cjs +25 -0
- package/dist/sandbox/opensandbox/materialize.cjs.map +1 -0
- package/dist/sandbox/opensandbox/materialize.d.cts +4 -0
- package/dist/sandbox/opensandbox/materialize.d.mts +4 -0
- package/dist/sandbox/opensandbox/materialize.d.ts +4 -0
- package/dist/sandbox/opensandbox/materialize.js +6 -0
- package/dist/sandbox/opensandbox/materialize.js.map +1 -0
- package/dist/sandbox/opensandbox/process-bridge.cjs +89 -0
- package/dist/sandbox/opensandbox/process-bridge.cjs.map +1 -0
- package/dist/sandbox/opensandbox/process-bridge.d.cts +2 -0
- package/dist/sandbox/opensandbox/process-bridge.d.mts +2 -0
- package/dist/sandbox/opensandbox/process-bridge.d.ts +2 -0
- package/dist/sandbox/opensandbox/process-bridge.js +6 -0
- package/dist/sandbox/opensandbox/process-bridge.js.map +1 -0
- package/dist/sandbox/opensandbox/sandbox.cjs +253 -0
- package/dist/sandbox/opensandbox/sandbox.cjs.map +1 -0
- package/dist/sandbox/opensandbox/sandbox.d.cts +51 -0
- package/dist/sandbox/opensandbox/sandbox.d.mts +51 -0
- package/dist/sandbox/opensandbox/sandbox.d.ts +51 -0
- package/dist/sandbox/opensandbox/sandbox.js +6 -0
- package/dist/sandbox/opensandbox/sandbox.js.map +1 -0
- package/dist/sandbox/opensandbox.cjs +88 -0
- package/dist/sandbox/opensandbox.cjs.map +1 -0
- package/dist/sandbox/opensandbox.d.cts +17 -0
- package/dist/sandbox/opensandbox.d.mts +17 -0
- package/dist/sandbox/opensandbox.d.ts +17 -0
- package/dist/sandbox/opensandbox.js +6 -0
- package/dist/sandbox/opensandbox.js.map +1 -0
- package/dist/sandbox/runtime.cjs +1 -0
- package/dist/sandbox/runtime.cjs.map +1 -1
- package/dist/sandbox/runtime.d.cts +2 -0
- package/dist/sandbox/runtime.d.mts +2 -0
- package/dist/sandbox/runtime.d.ts +2 -0
- package/dist/sandbox/runtime.js +2 -0
- package/dist/sandbox/types.cjs.map +1 -1
- package/dist/sandbox/types.d.cts +1 -1
- package/dist/sandbox/types.d.mts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/dist/show/model.cjs +6 -5
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +1 -1
- package/dist/show/model.d.mts +1 -1
- package/dist/show/model.d.ts +1 -1
- package/dist/show/render.cjs +2 -2
- package/dist/show/render.cjs.map +1 -1
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/{index-CxrOUvHC.js → index-f9ZKc4km.js} +16 -16
- package/dist/view/app-dist/assets/{quality-cost-scatter-B6Aiss7x.js → quality-cost-scatter-C6lsrigr.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/function-runtime.mjs +126 -118
- package/dist/view/http-protocol.d.cts +1 -1
- package/dist/view/http-protocol.d.mts +1 -1
- package/dist/view/http-protocol.d.ts +1 -1
- package/dist/view/inspection-host.d.cts +625 -493
- package/dist/view/inspection-host.d.mts +625 -493
- package/dist/view/inspection-host.d.ts +625 -493
- package/docs-site/zh/explanation/evals.mdx +4 -0
- package/docs-site/zh/reference/cli.mdx +45 -4
- package/docs-site/zh/tutorials/compare-runs.mdx +1 -1
- package/docs-site/zh/tutorials/niceeval-taste.mdx +34 -1
- package/docs-site/zh/tutorials/sandbox-providers.mdx +35 -4
- package/docs-site/zh/tutorials/upgrade-0-19.mdx +1 -1
- package/docs-site/zh/tutorials/viewing-results.mdx +36 -3
- package/package.json +6 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
declare const SuccessMetadataSchema: Schema.Struct<{
|
|
3
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
3
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
4
4
|
readonly outcome: Schema.Literal<"success">;
|
|
5
5
|
readonly behaviorVersion: Schema.Literal<"2.0.0">;
|
|
6
6
|
readonly source: Schema.Struct<{
|
|
@@ -22,7 +22,7 @@ declare const SuccessMetadataSchema: Schema.Struct<{
|
|
|
22
22
|
}>;
|
|
23
23
|
}>;
|
|
24
24
|
declare const RunsListSuccessMetadataSchema: Schema.Struct<{
|
|
25
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
25
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
26
26
|
readonly outcome: Schema.Literal<"success">;
|
|
27
27
|
readonly behaviorVersion: Schema.Literal<"2.0.0">;
|
|
28
28
|
readonly source: Schema.Struct<{
|
|
@@ -46,7 +46,7 @@ declare const RunsListSuccessMetadataSchema: Schema.Struct<{
|
|
|
46
46
|
}>;
|
|
47
47
|
}>;
|
|
48
48
|
declare const CompareSuccessMetadataSchema: Schema.Struct<{
|
|
49
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
49
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
50
50
|
readonly source: Schema.Struct<{
|
|
51
51
|
readonly kind: Schema.Literals<readonly ["project-record", "external-record"]>;
|
|
52
52
|
readonly publicationCutoffIdentity: Schema.String;
|
|
@@ -1119,9 +1119,6 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
1119
1119
|
readonly groupPath: Schema.$Array<Schema.String>;
|
|
1120
1120
|
readonly members: Schema.$Array<Schema.Struct<{
|
|
1121
1121
|
readonly runId: Schema.String;
|
|
1122
|
-
readonly plannedAttemptId: Schema.String;
|
|
1123
|
-
readonly evalId: Schema.String;
|
|
1124
|
-
readonly attemptOrdinal: Schema.Number;
|
|
1125
1122
|
readonly labels: Schema.$Record<Schema.String, Schema.Union<readonly [Schema.Struct<{
|
|
1126
1123
|
readonly state: Schema.Literal<"available">;
|
|
1127
1124
|
readonly value: Schema.String;
|
|
@@ -1130,155 +1127,158 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
1130
1127
|
}>, Schema.Struct<{
|
|
1131
1128
|
readonly state: Schema.Literal<"unavailable">;
|
|
1132
1129
|
}>]>>;
|
|
1133
|
-
readonly
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
readonly
|
|
1137
|
-
readonly
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
readonly
|
|
1142
|
-
readonly
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1162
|
-
readonly identity: Schema.Struct<{
|
|
1163
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
1164
|
-
readonly locator: Schema.String;
|
|
1165
|
-
}>;
|
|
1166
|
-
}>>;
|
|
1167
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1168
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1169
|
-
readonly min: Schema.Number;
|
|
1170
|
-
readonly max: Schema.Number;
|
|
1171
|
-
}>>;
|
|
1172
|
-
}>;
|
|
1173
|
-
readonly costUSD: Schema.Struct<{
|
|
1174
|
-
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
1175
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1176
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
1177
|
-
readonly samples: Schema.Number;
|
|
1178
|
-
readonly total: Schema.Number;
|
|
1179
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1180
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1181
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1182
|
-
readonly locator: Schema.String;
|
|
1183
|
-
}>, Schema.Struct<{
|
|
1184
|
-
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
1185
|
-
readonly runId: Schema.String;
|
|
1186
|
-
readonly plannedAttemptId: Schema.String;
|
|
1187
|
-
readonly evalId: Schema.String;
|
|
1188
|
-
readonly attemptOrdinal: Schema.Number;
|
|
1189
|
-
}>, Schema.Struct<{
|
|
1190
|
-
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
1191
|
-
readonly locator: Schema.String;
|
|
1192
|
-
readonly entryId: Schema.String;
|
|
1193
|
-
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
1194
|
-
}>, Schema.Struct<{
|
|
1195
|
-
readonly code: Schema.Literal<"score-partial">;
|
|
1196
|
-
readonly locator: Schema.NullOr<Schema.String>;
|
|
1197
|
-
}>]>>;
|
|
1198
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1199
|
-
readonly identity: Schema.Struct<{
|
|
1200
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
1201
|
-
readonly locator: Schema.String;
|
|
1202
|
-
}>;
|
|
1203
|
-
}>>;
|
|
1204
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1205
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1206
|
-
readonly min: Schema.Number;
|
|
1207
|
-
readonly max: Schema.Number;
|
|
1208
|
-
}>>;
|
|
1209
|
-
}>;
|
|
1210
|
-
readonly durationMs: Schema.Struct<{
|
|
1211
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1212
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
1213
|
-
readonly samples: Schema.Number;
|
|
1214
|
-
readonly total: Schema.Number;
|
|
1215
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1216
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1217
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1130
|
+
readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
|
|
1131
|
+
readonly score: Schema.Struct<{
|
|
1132
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1133
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
1134
|
+
readonly samples: Schema.Number;
|
|
1135
|
+
readonly total: Schema.Number;
|
|
1136
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1137
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1138
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1139
|
+
readonly locator: Schema.String;
|
|
1140
|
+
}>, Schema.Struct<{
|
|
1141
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
1142
|
+
readonly runId: Schema.String;
|
|
1143
|
+
readonly plannedAttemptId: Schema.String;
|
|
1144
|
+
readonly evalId: Schema.String;
|
|
1145
|
+
readonly attemptOrdinal: Schema.Number;
|
|
1146
|
+
}>, Schema.Struct<{
|
|
1147
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
1148
|
+
readonly locator: Schema.String;
|
|
1149
|
+
readonly entryId: Schema.String;
|
|
1150
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
1151
|
+
}>, Schema.Struct<{
|
|
1152
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
1153
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
1154
|
+
}>]>>;
|
|
1155
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1156
|
+
readonly identity: Schema.Struct<{
|
|
1157
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
1218
1158
|
readonly locator: Schema.String;
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1159
|
+
}>;
|
|
1160
|
+
}>>;
|
|
1161
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1162
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1163
|
+
readonly min: Schema.Number;
|
|
1164
|
+
readonly max: Schema.Number;
|
|
1165
|
+
}>>;
|
|
1166
|
+
}>;
|
|
1167
|
+
readonly costUSD: Schema.Struct<{
|
|
1168
|
+
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
1169
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1170
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
1171
|
+
readonly samples: Schema.Number;
|
|
1172
|
+
readonly total: Schema.Number;
|
|
1173
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1174
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1175
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1176
|
+
readonly locator: Schema.String;
|
|
1177
|
+
}>, Schema.Struct<{
|
|
1178
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
1179
|
+
readonly runId: Schema.String;
|
|
1180
|
+
readonly plannedAttemptId: Schema.String;
|
|
1181
|
+
readonly evalId: Schema.String;
|
|
1182
|
+
readonly attemptOrdinal: Schema.Number;
|
|
1183
|
+
}>, Schema.Struct<{
|
|
1184
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
1185
|
+
readonly locator: Schema.String;
|
|
1186
|
+
readonly entryId: Schema.String;
|
|
1187
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
1188
|
+
}>, Schema.Struct<{
|
|
1189
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
1190
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
1191
|
+
}>]>>;
|
|
1192
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1193
|
+
readonly identity: Schema.Struct<{
|
|
1194
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
1227
1195
|
readonly locator: Schema.String;
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
}
|
|
1246
|
-
|
|
1247
|
-
readonly
|
|
1248
|
-
readonly
|
|
1249
|
-
readonly
|
|
1250
|
-
readonly
|
|
1251
|
-
|
|
1252
|
-
readonly
|
|
1253
|
-
|
|
1196
|
+
}>;
|
|
1197
|
+
}>>;
|
|
1198
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1199
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1200
|
+
readonly min: Schema.Number;
|
|
1201
|
+
readonly max: Schema.Number;
|
|
1202
|
+
}>>;
|
|
1203
|
+
}>;
|
|
1204
|
+
readonly durationMs: Schema.Struct<{
|
|
1205
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1206
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
1207
|
+
readonly samples: Schema.Number;
|
|
1208
|
+
readonly total: Schema.Number;
|
|
1209
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1210
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1211
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1212
|
+
readonly locator: Schema.String;
|
|
1213
|
+
}>, Schema.Struct<{
|
|
1214
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
1215
|
+
readonly runId: Schema.String;
|
|
1216
|
+
readonly plannedAttemptId: Schema.String;
|
|
1217
|
+
readonly evalId: Schema.String;
|
|
1218
|
+
readonly attemptOrdinal: Schema.Number;
|
|
1219
|
+
}>, Schema.Struct<{
|
|
1220
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
1221
|
+
readonly locator: Schema.String;
|
|
1222
|
+
readonly entryId: Schema.String;
|
|
1223
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
1224
|
+
}>, Schema.Struct<{
|
|
1225
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
1226
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
1227
|
+
}>]>>;
|
|
1228
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1229
|
+
readonly identity: Schema.Struct<{
|
|
1230
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
1254
1231
|
readonly locator: Schema.String;
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1232
|
+
}>;
|
|
1233
|
+
}>>;
|
|
1234
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1235
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1236
|
+
readonly min: Schema.Number;
|
|
1237
|
+
readonly max: Schema.Number;
|
|
1238
|
+
}>>;
|
|
1239
|
+
}>;
|
|
1240
|
+
readonly tokens: Schema.Struct<{
|
|
1241
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
1242
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
1243
|
+
readonly samples: Schema.Number;
|
|
1244
|
+
readonly total: Schema.Number;
|
|
1245
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
1246
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
1247
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
1248
|
+
readonly locator: Schema.String;
|
|
1249
|
+
}>, Schema.Struct<{
|
|
1250
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
1251
|
+
readonly runId: Schema.String;
|
|
1252
|
+
readonly plannedAttemptId: Schema.String;
|
|
1253
|
+
readonly evalId: Schema.String;
|
|
1254
|
+
readonly attemptOrdinal: Schema.Number;
|
|
1255
|
+
}>, Schema.Struct<{
|
|
1256
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
1257
|
+
readonly locator: Schema.String;
|
|
1258
|
+
readonly entryId: Schema.String;
|
|
1259
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
1260
|
+
}>, Schema.Struct<{
|
|
1261
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
1262
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
1263
|
+
}>]>>;
|
|
1264
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
1265
|
+
readonly identity: Schema.Struct<{
|
|
1266
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
1263
1267
|
readonly locator: Schema.String;
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
readonly min: Schema.Number;
|
|
1279
|
-
readonly max: Schema.Number;
|
|
1280
|
-
}>>;
|
|
1281
|
-
}>;
|
|
1268
|
+
}>;
|
|
1269
|
+
}>>;
|
|
1270
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
1271
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
1272
|
+
readonly min: Schema.Number;
|
|
1273
|
+
readonly max: Schema.Number;
|
|
1274
|
+
}>>;
|
|
1275
|
+
}>;
|
|
1276
|
+
readonly plannedAttemptId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
|
|
1277
|
+
readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
|
|
1278
|
+
readonly attemptOrdinal: Schema.Number;
|
|
1279
|
+
readonly publication: Schema.Union<readonly [Schema.Struct<{
|
|
1280
|
+
readonly state: Schema.Literal<"pending">;
|
|
1281
|
+
}>, Schema.Struct<{
|
|
1282
1282
|
readonly originPlannedAttemptId: Schema.String;
|
|
1283
1283
|
readonly state: Schema.Literal<"published">;
|
|
1284
1284
|
readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
|
|
@@ -2360,9 +2360,6 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
2360
2360
|
readonly groupPath: Schema.$Array<Schema.String>;
|
|
2361
2361
|
readonly members: Schema.$Array<Schema.Struct<{
|
|
2362
2362
|
readonly runId: Schema.String;
|
|
2363
|
-
readonly plannedAttemptId: Schema.String;
|
|
2364
|
-
readonly evalId: Schema.String;
|
|
2365
|
-
readonly attemptOrdinal: Schema.Number;
|
|
2366
2363
|
readonly labels: Schema.$Record<Schema.String, Schema.Union<readonly [Schema.Struct<{
|
|
2367
2364
|
readonly state: Schema.Literal<"available">;
|
|
2368
2365
|
readonly value: Schema.String;
|
|
@@ -2371,155 +2368,158 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
2371
2368
|
}>, Schema.Struct<{
|
|
2372
2369
|
readonly state: Schema.Literal<"unavailable">;
|
|
2373
2370
|
}>]>>;
|
|
2374
|
-
readonly
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
readonly
|
|
2378
|
-
readonly
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
readonly
|
|
2383
|
-
readonly
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2403
|
-
readonly identity: Schema.Struct<{
|
|
2404
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
2405
|
-
readonly locator: Schema.String;
|
|
2406
|
-
}>;
|
|
2407
|
-
}>>;
|
|
2408
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2409
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2410
|
-
readonly min: Schema.Number;
|
|
2411
|
-
readonly max: Schema.Number;
|
|
2412
|
-
}>>;
|
|
2413
|
-
}>;
|
|
2414
|
-
readonly costUSD: Schema.Struct<{
|
|
2415
|
-
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
2416
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2417
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
2418
|
-
readonly samples: Schema.Number;
|
|
2419
|
-
readonly total: Schema.Number;
|
|
2420
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2421
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2422
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2423
|
-
readonly locator: Schema.String;
|
|
2424
|
-
}>, Schema.Struct<{
|
|
2425
|
-
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
2426
|
-
readonly runId: Schema.String;
|
|
2427
|
-
readonly plannedAttemptId: Schema.String;
|
|
2428
|
-
readonly evalId: Schema.String;
|
|
2429
|
-
readonly attemptOrdinal: Schema.Number;
|
|
2430
|
-
}>, Schema.Struct<{
|
|
2431
|
-
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
2432
|
-
readonly locator: Schema.String;
|
|
2433
|
-
readonly entryId: Schema.String;
|
|
2434
|
-
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
2435
|
-
}>, Schema.Struct<{
|
|
2436
|
-
readonly code: Schema.Literal<"score-partial">;
|
|
2437
|
-
readonly locator: Schema.NullOr<Schema.String>;
|
|
2438
|
-
}>]>>;
|
|
2439
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2440
|
-
readonly identity: Schema.Struct<{
|
|
2441
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
2442
|
-
readonly locator: Schema.String;
|
|
2443
|
-
}>;
|
|
2444
|
-
}>>;
|
|
2445
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2446
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2447
|
-
readonly min: Schema.Number;
|
|
2448
|
-
readonly max: Schema.Number;
|
|
2449
|
-
}>>;
|
|
2450
|
-
}>;
|
|
2451
|
-
readonly durationMs: Schema.Struct<{
|
|
2452
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2453
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
2454
|
-
readonly samples: Schema.Number;
|
|
2455
|
-
readonly total: Schema.Number;
|
|
2456
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2457
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2458
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2371
|
+
readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
|
|
2372
|
+
readonly score: Schema.Struct<{
|
|
2373
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2374
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
2375
|
+
readonly samples: Schema.Number;
|
|
2376
|
+
readonly total: Schema.Number;
|
|
2377
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2378
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2379
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2380
|
+
readonly locator: Schema.String;
|
|
2381
|
+
}>, Schema.Struct<{
|
|
2382
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
2383
|
+
readonly runId: Schema.String;
|
|
2384
|
+
readonly plannedAttemptId: Schema.String;
|
|
2385
|
+
readonly evalId: Schema.String;
|
|
2386
|
+
readonly attemptOrdinal: Schema.Number;
|
|
2387
|
+
}>, Schema.Struct<{
|
|
2388
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
2389
|
+
readonly locator: Schema.String;
|
|
2390
|
+
readonly entryId: Schema.String;
|
|
2391
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
2392
|
+
}>, Schema.Struct<{
|
|
2393
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
2394
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
2395
|
+
}>]>>;
|
|
2396
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2397
|
+
readonly identity: Schema.Struct<{
|
|
2398
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
2459
2399
|
readonly locator: Schema.String;
|
|
2460
|
-
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2400
|
+
}>;
|
|
2401
|
+
}>>;
|
|
2402
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2403
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2404
|
+
readonly min: Schema.Number;
|
|
2405
|
+
readonly max: Schema.Number;
|
|
2406
|
+
}>>;
|
|
2407
|
+
}>;
|
|
2408
|
+
readonly costUSD: Schema.Struct<{
|
|
2409
|
+
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
2410
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2411
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
2412
|
+
readonly samples: Schema.Number;
|
|
2413
|
+
readonly total: Schema.Number;
|
|
2414
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2415
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2416
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2417
|
+
readonly locator: Schema.String;
|
|
2418
|
+
}>, Schema.Struct<{
|
|
2419
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
2420
|
+
readonly runId: Schema.String;
|
|
2421
|
+
readonly plannedAttemptId: Schema.String;
|
|
2422
|
+
readonly evalId: Schema.String;
|
|
2423
|
+
readonly attemptOrdinal: Schema.Number;
|
|
2424
|
+
}>, Schema.Struct<{
|
|
2425
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
2426
|
+
readonly locator: Schema.String;
|
|
2427
|
+
readonly entryId: Schema.String;
|
|
2428
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
2429
|
+
}>, Schema.Struct<{
|
|
2430
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
2431
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
2432
|
+
}>]>>;
|
|
2433
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2434
|
+
readonly identity: Schema.Struct<{
|
|
2435
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
2468
2436
|
readonly locator: Schema.String;
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
}
|
|
2487
|
-
|
|
2488
|
-
readonly
|
|
2489
|
-
readonly
|
|
2490
|
-
readonly
|
|
2491
|
-
readonly
|
|
2492
|
-
|
|
2493
|
-
readonly
|
|
2494
|
-
|
|
2437
|
+
}>;
|
|
2438
|
+
}>>;
|
|
2439
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2440
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2441
|
+
readonly min: Schema.Number;
|
|
2442
|
+
readonly max: Schema.Number;
|
|
2443
|
+
}>>;
|
|
2444
|
+
}>;
|
|
2445
|
+
readonly durationMs: Schema.Struct<{
|
|
2446
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2447
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
2448
|
+
readonly samples: Schema.Number;
|
|
2449
|
+
readonly total: Schema.Number;
|
|
2450
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2451
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2452
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2453
|
+
readonly locator: Schema.String;
|
|
2454
|
+
}>, Schema.Struct<{
|
|
2455
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
2456
|
+
readonly runId: Schema.String;
|
|
2457
|
+
readonly plannedAttemptId: Schema.String;
|
|
2458
|
+
readonly evalId: Schema.String;
|
|
2459
|
+
readonly attemptOrdinal: Schema.Number;
|
|
2460
|
+
}>, Schema.Struct<{
|
|
2461
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
2462
|
+
readonly locator: Schema.String;
|
|
2463
|
+
readonly entryId: Schema.String;
|
|
2464
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
2465
|
+
}>, Schema.Struct<{
|
|
2466
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
2467
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
2468
|
+
}>]>>;
|
|
2469
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2470
|
+
readonly identity: Schema.Struct<{
|
|
2471
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
2495
2472
|
readonly locator: Schema.String;
|
|
2496
|
-
}
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2473
|
+
}>;
|
|
2474
|
+
}>>;
|
|
2475
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2476
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2477
|
+
readonly min: Schema.Number;
|
|
2478
|
+
readonly max: Schema.Number;
|
|
2479
|
+
}>>;
|
|
2480
|
+
}>;
|
|
2481
|
+
readonly tokens: Schema.Struct<{
|
|
2482
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
2483
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
2484
|
+
readonly samples: Schema.Number;
|
|
2485
|
+
readonly total: Schema.Number;
|
|
2486
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
2487
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
2488
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
2489
|
+
readonly locator: Schema.String;
|
|
2490
|
+
}>, Schema.Struct<{
|
|
2491
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
2492
|
+
readonly runId: Schema.String;
|
|
2493
|
+
readonly plannedAttemptId: Schema.String;
|
|
2494
|
+
readonly evalId: Schema.String;
|
|
2495
|
+
readonly attemptOrdinal: Schema.Number;
|
|
2496
|
+
}>, Schema.Struct<{
|
|
2497
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
2498
|
+
readonly locator: Schema.String;
|
|
2499
|
+
readonly entryId: Schema.String;
|
|
2500
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
2501
|
+
}>, Schema.Struct<{
|
|
2502
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
2503
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
2504
|
+
}>]>>;
|
|
2505
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
2506
|
+
readonly identity: Schema.Struct<{
|
|
2507
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
2504
2508
|
readonly locator: Schema.String;
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
readonly min: Schema.Number;
|
|
2520
|
-
readonly max: Schema.Number;
|
|
2521
|
-
}>>;
|
|
2522
|
-
}>;
|
|
2509
|
+
}>;
|
|
2510
|
+
}>>;
|
|
2511
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
2512
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
2513
|
+
readonly min: Schema.Number;
|
|
2514
|
+
readonly max: Schema.Number;
|
|
2515
|
+
}>>;
|
|
2516
|
+
}>;
|
|
2517
|
+
readonly plannedAttemptId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
|
|
2518
|
+
readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
|
|
2519
|
+
readonly attemptOrdinal: Schema.Number;
|
|
2520
|
+
readonly publication: Schema.Union<readonly [Schema.Struct<{
|
|
2521
|
+
readonly state: Schema.Literal<"pending">;
|
|
2522
|
+
}>, Schema.Struct<{
|
|
2523
2523
|
readonly originPlannedAttemptId: Schema.String;
|
|
2524
2524
|
readonly state: Schema.Literal<"published">;
|
|
2525
2525
|
readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
|
|
@@ -3538,9 +3538,6 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
3538
3538
|
readonly groupPath: Schema.$Array<Schema.String>;
|
|
3539
3539
|
readonly members: Schema.$Array<Schema.Struct<{
|
|
3540
3540
|
readonly runId: Schema.String;
|
|
3541
|
-
readonly plannedAttemptId: Schema.String;
|
|
3542
|
-
readonly evalId: Schema.String;
|
|
3543
|
-
readonly attemptOrdinal: Schema.Number;
|
|
3544
3541
|
readonly labels: Schema.$Record<Schema.String, Schema.Union<readonly [Schema.Struct<{
|
|
3545
3542
|
readonly state: Schema.Literal<"available">;
|
|
3546
3543
|
readonly value: Schema.String;
|
|
@@ -3549,155 +3546,158 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
3549
3546
|
}>, Schema.Struct<{
|
|
3550
3547
|
readonly state: Schema.Literal<"unavailable">;
|
|
3551
3548
|
}>]>>;
|
|
3552
|
-
readonly
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
readonly
|
|
3556
|
-
readonly
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
readonly
|
|
3561
|
-
readonly
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3581
|
-
readonly identity: Schema.Struct<{
|
|
3582
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
3583
|
-
readonly locator: Schema.String;
|
|
3584
|
-
}>;
|
|
3585
|
-
}>>;
|
|
3586
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3587
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3588
|
-
readonly min: Schema.Number;
|
|
3589
|
-
readonly max: Schema.Number;
|
|
3590
|
-
}>>;
|
|
3591
|
-
}>;
|
|
3592
|
-
readonly costUSD: Schema.Struct<{
|
|
3593
|
-
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
3594
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3595
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
3596
|
-
readonly samples: Schema.Number;
|
|
3597
|
-
readonly total: Schema.Number;
|
|
3598
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3599
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3600
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3601
|
-
readonly locator: Schema.String;
|
|
3602
|
-
}>, Schema.Struct<{
|
|
3603
|
-
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
3604
|
-
readonly runId: Schema.String;
|
|
3605
|
-
readonly plannedAttemptId: Schema.String;
|
|
3606
|
-
readonly evalId: Schema.String;
|
|
3607
|
-
readonly attemptOrdinal: Schema.Number;
|
|
3608
|
-
}>, Schema.Struct<{
|
|
3609
|
-
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
3610
|
-
readonly locator: Schema.String;
|
|
3611
|
-
readonly entryId: Schema.String;
|
|
3612
|
-
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
3613
|
-
}>, Schema.Struct<{
|
|
3614
|
-
readonly code: Schema.Literal<"score-partial">;
|
|
3615
|
-
readonly locator: Schema.NullOr<Schema.String>;
|
|
3616
|
-
}>]>>;
|
|
3617
|
-
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3618
|
-
readonly identity: Schema.Struct<{
|
|
3619
|
-
readonly kind: Schema.Literal<"attempt">;
|
|
3620
|
-
readonly locator: Schema.String;
|
|
3621
|
-
}>;
|
|
3622
|
-
}>>;
|
|
3623
|
-
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3624
|
-
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3625
|
-
readonly min: Schema.Number;
|
|
3626
|
-
readonly max: Schema.Number;
|
|
3627
|
-
}>>;
|
|
3628
|
-
}>;
|
|
3629
|
-
readonly durationMs: Schema.Struct<{
|
|
3630
|
-
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3631
|
-
readonly value: Schema.NullOr<Schema.Number>;
|
|
3632
|
-
readonly samples: Schema.Number;
|
|
3633
|
-
readonly total: Schema.Number;
|
|
3634
|
-
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3635
|
-
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3636
|
-
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3549
|
+
readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
|
|
3550
|
+
readonly score: Schema.Struct<{
|
|
3551
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3552
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
3553
|
+
readonly samples: Schema.Number;
|
|
3554
|
+
readonly total: Schema.Number;
|
|
3555
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3556
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3557
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3558
|
+
readonly locator: Schema.String;
|
|
3559
|
+
}>, Schema.Struct<{
|
|
3560
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
3561
|
+
readonly runId: Schema.String;
|
|
3562
|
+
readonly plannedAttemptId: Schema.String;
|
|
3563
|
+
readonly evalId: Schema.String;
|
|
3564
|
+
readonly attemptOrdinal: Schema.Number;
|
|
3565
|
+
}>, Schema.Struct<{
|
|
3566
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
3567
|
+
readonly locator: Schema.String;
|
|
3568
|
+
readonly entryId: Schema.String;
|
|
3569
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
3570
|
+
}>, Schema.Struct<{
|
|
3571
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
3572
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
3573
|
+
}>]>>;
|
|
3574
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3575
|
+
readonly identity: Schema.Struct<{
|
|
3576
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
3637
3577
|
readonly locator: Schema.String;
|
|
3638
|
-
}
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3578
|
+
}>;
|
|
3579
|
+
}>>;
|
|
3580
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3581
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3582
|
+
readonly min: Schema.Number;
|
|
3583
|
+
readonly max: Schema.Number;
|
|
3584
|
+
}>>;
|
|
3585
|
+
}>;
|
|
3586
|
+
readonly costUSD: Schema.Struct<{
|
|
3587
|
+
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
3588
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3589
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
3590
|
+
readonly samples: Schema.Number;
|
|
3591
|
+
readonly total: Schema.Number;
|
|
3592
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3593
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3594
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3595
|
+
readonly locator: Schema.String;
|
|
3596
|
+
}>, Schema.Struct<{
|
|
3597
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
3598
|
+
readonly runId: Schema.String;
|
|
3599
|
+
readonly plannedAttemptId: Schema.String;
|
|
3600
|
+
readonly evalId: Schema.String;
|
|
3601
|
+
readonly attemptOrdinal: Schema.Number;
|
|
3602
|
+
}>, Schema.Struct<{
|
|
3603
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
3604
|
+
readonly locator: Schema.String;
|
|
3605
|
+
readonly entryId: Schema.String;
|
|
3606
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
3607
|
+
}>, Schema.Struct<{
|
|
3608
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
3609
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
3610
|
+
}>]>>;
|
|
3611
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3612
|
+
readonly identity: Schema.Struct<{
|
|
3613
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
3646
3614
|
readonly locator: Schema.String;
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
}
|
|
3665
|
-
|
|
3666
|
-
readonly
|
|
3667
|
-
readonly
|
|
3668
|
-
readonly
|
|
3669
|
-
readonly
|
|
3670
|
-
|
|
3671
|
-
readonly
|
|
3672
|
-
|
|
3615
|
+
}>;
|
|
3616
|
+
}>>;
|
|
3617
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3618
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3619
|
+
readonly min: Schema.Number;
|
|
3620
|
+
readonly max: Schema.Number;
|
|
3621
|
+
}>>;
|
|
3622
|
+
}>;
|
|
3623
|
+
readonly durationMs: Schema.Struct<{
|
|
3624
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3625
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
3626
|
+
readonly samples: Schema.Number;
|
|
3627
|
+
readonly total: Schema.Number;
|
|
3628
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3629
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3630
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3631
|
+
readonly locator: Schema.String;
|
|
3632
|
+
}>, Schema.Struct<{
|
|
3633
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
3634
|
+
readonly runId: Schema.String;
|
|
3635
|
+
readonly plannedAttemptId: Schema.String;
|
|
3636
|
+
readonly evalId: Schema.String;
|
|
3637
|
+
readonly attemptOrdinal: Schema.Number;
|
|
3638
|
+
}>, Schema.Struct<{
|
|
3639
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
3640
|
+
readonly locator: Schema.String;
|
|
3641
|
+
readonly entryId: Schema.String;
|
|
3642
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
3643
|
+
}>, Schema.Struct<{
|
|
3644
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
3645
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
3646
|
+
}>]>>;
|
|
3647
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3648
|
+
readonly identity: Schema.Struct<{
|
|
3649
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
3673
3650
|
readonly locator: Schema.String;
|
|
3674
|
-
}
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3651
|
+
}>;
|
|
3652
|
+
}>>;
|
|
3653
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3654
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3655
|
+
readonly min: Schema.Number;
|
|
3656
|
+
readonly max: Schema.Number;
|
|
3657
|
+
}>>;
|
|
3658
|
+
}>;
|
|
3659
|
+
readonly tokens: Schema.Struct<{
|
|
3660
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
3661
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
3662
|
+
readonly samples: Schema.Number;
|
|
3663
|
+
readonly total: Schema.Number;
|
|
3664
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
3665
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
3666
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
3667
|
+
readonly locator: Schema.String;
|
|
3668
|
+
}>, Schema.Struct<{
|
|
3669
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
3670
|
+
readonly runId: Schema.String;
|
|
3671
|
+
readonly plannedAttemptId: Schema.String;
|
|
3672
|
+
readonly evalId: Schema.String;
|
|
3673
|
+
readonly attemptOrdinal: Schema.Number;
|
|
3674
|
+
}>, Schema.Struct<{
|
|
3675
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
3676
|
+
readonly locator: Schema.String;
|
|
3677
|
+
readonly entryId: Schema.String;
|
|
3678
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
3679
|
+
}>, Schema.Struct<{
|
|
3680
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
3681
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
3682
|
+
}>]>>;
|
|
3683
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
3684
|
+
readonly identity: Schema.Struct<{
|
|
3685
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
3682
3686
|
readonly locator: Schema.String;
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
readonly min: Schema.Number;
|
|
3698
|
-
readonly max: Schema.Number;
|
|
3699
|
-
}>>;
|
|
3700
|
-
}>;
|
|
3687
|
+
}>;
|
|
3688
|
+
}>>;
|
|
3689
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
3690
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
3691
|
+
readonly min: Schema.Number;
|
|
3692
|
+
readonly max: Schema.Number;
|
|
3693
|
+
}>>;
|
|
3694
|
+
}>;
|
|
3695
|
+
readonly plannedAttemptId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
|
|
3696
|
+
readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
|
|
3697
|
+
readonly attemptOrdinal: Schema.Number;
|
|
3698
|
+
readonly publication: Schema.Union<readonly [Schema.Struct<{
|
|
3699
|
+
readonly state: Schema.Literal<"pending">;
|
|
3700
|
+
}>, Schema.Struct<{
|
|
3701
3701
|
readonly originPlannedAttemptId: Schema.String;
|
|
3702
3702
|
readonly state: Schema.Literal<"published">;
|
|
3703
3703
|
readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
|
|
@@ -3791,36 +3791,9 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
3791
3791
|
readonly missing: Schema.Number;
|
|
3792
3792
|
}>;
|
|
3793
3793
|
readonly plannedAttempts: Schema.$Array<Schema.Struct<{
|
|
3794
|
-
readonly publication: Schema.Union<readonly [Schema.Struct<{
|
|
3795
|
-
readonly state: Schema.Literal<"pending">;
|
|
3796
|
-
}>, Schema.Struct<{
|
|
3797
|
-
readonly originPlannedAttemptId: Schema.String;
|
|
3798
|
-
readonly state: Schema.Literal<"published">;
|
|
3799
|
-
readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
|
|
3800
|
-
readonly attemptId: Schema.String;
|
|
3801
|
-
readonly attemptLocator: Schema.String;
|
|
3802
|
-
readonly originRunId: Schema.String;
|
|
3803
|
-
}>, Schema.Struct<{
|
|
3804
|
-
readonly state: Schema.Literal<"absent">;
|
|
3805
|
-
readonly reason: Schema.Literals<readonly ["early-exit-satisfied", "budget-exhausted", "stopped-by-failure", "interrupted-before-publication", "dispatch-failed"]>;
|
|
3806
|
-
}>]>;
|
|
3807
3794
|
readonly state: Schema.Literals<readonly ["executed", "carried", "accepted", "not-dispatched", "interrupted", "pending"]>;
|
|
3808
|
-
readonly locator: Schema.NullOr<Schema.String>;
|
|
3809
3795
|
readonly relation: Schema.NullOr<Schema.Literals<readonly ["origin", "reference"]>>;
|
|
3810
3796
|
readonly outcome: Schema.NullOr<Schema.Literals<readonly ["completed", "errored", "cancelled", "interrupted"]>>;
|
|
3811
|
-
readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
|
|
3812
|
-
readonly score: Schema.NullOr<Schema.Union<readonly [Schema.Struct<{
|
|
3813
|
-
readonly state: Schema.Literal<"not-scored">;
|
|
3814
|
-
}>, Schema.Struct<{
|
|
3815
|
-
readonly state: Schema.Literal<"complete">;
|
|
3816
|
-
readonly earned: Schema.Number;
|
|
3817
|
-
readonly possible: Schema.Number;
|
|
3818
|
-
}>, Schema.Struct<{
|
|
3819
|
-
readonly state: Schema.Literal<"unavailable">;
|
|
3820
|
-
readonly earned: Schema.Number;
|
|
3821
|
-
readonly possible: Schema.Number;
|
|
3822
|
-
readonly unavailable: Schema.Number;
|
|
3823
|
-
}>]>>;
|
|
3824
3797
|
readonly coverage: Schema.Struct<{
|
|
3825
3798
|
readonly state: Schema.Literals<readonly ["complete", "partial", "not-recorded", "invalid", "unavailable"]>;
|
|
3826
3799
|
readonly facts: Schema.$Array<Schema.Struct<{
|
|
@@ -4078,10 +4051,169 @@ export declare const inspectionProtocolRegistry: Readonly<{
|
|
|
4078
4051
|
readonly reason: Schema.String;
|
|
4079
4052
|
}>]>;
|
|
4080
4053
|
}>]>>;
|
|
4054
|
+
readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
|
|
4055
|
+
readonly score: Schema.Struct<{
|
|
4056
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
4057
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
4058
|
+
readonly samples: Schema.Number;
|
|
4059
|
+
readonly total: Schema.Number;
|
|
4060
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
4061
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
4062
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
4063
|
+
readonly locator: Schema.String;
|
|
4064
|
+
}>, Schema.Struct<{
|
|
4065
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
4066
|
+
readonly runId: Schema.String;
|
|
4067
|
+
readonly plannedAttemptId: Schema.String;
|
|
4068
|
+
readonly evalId: Schema.String;
|
|
4069
|
+
readonly attemptOrdinal: Schema.Number;
|
|
4070
|
+
}>, Schema.Struct<{
|
|
4071
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
4072
|
+
readonly locator: Schema.String;
|
|
4073
|
+
readonly entryId: Schema.String;
|
|
4074
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
4075
|
+
}>, Schema.Struct<{
|
|
4076
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
4077
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
4078
|
+
}>]>>;
|
|
4079
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
4080
|
+
readonly identity: Schema.Struct<{
|
|
4081
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
4082
|
+
readonly locator: Schema.String;
|
|
4083
|
+
}>;
|
|
4084
|
+
}>>;
|
|
4085
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
4086
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
4087
|
+
readonly min: Schema.Number;
|
|
4088
|
+
readonly max: Schema.Number;
|
|
4089
|
+
}>>;
|
|
4090
|
+
}>;
|
|
4091
|
+
readonly costUSD: Schema.Struct<{
|
|
4092
|
+
readonly source: Schema.NullOr<Schema.Literals<readonly ["reported", "estimated", "mixed"]>>;
|
|
4093
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
4094
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
4095
|
+
readonly samples: Schema.Number;
|
|
4096
|
+
readonly total: Schema.Number;
|
|
4097
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
4098
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
4099
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
4100
|
+
readonly locator: Schema.String;
|
|
4101
|
+
}>, Schema.Struct<{
|
|
4102
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
4103
|
+
readonly runId: Schema.String;
|
|
4104
|
+
readonly plannedAttemptId: Schema.String;
|
|
4105
|
+
readonly evalId: Schema.String;
|
|
4106
|
+
readonly attemptOrdinal: Schema.Number;
|
|
4107
|
+
}>, Schema.Struct<{
|
|
4108
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
4109
|
+
readonly locator: Schema.String;
|
|
4110
|
+
readonly entryId: Schema.String;
|
|
4111
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
4112
|
+
}>, Schema.Struct<{
|
|
4113
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
4114
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
4115
|
+
}>]>>;
|
|
4116
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
4117
|
+
readonly identity: Schema.Struct<{
|
|
4118
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
4119
|
+
readonly locator: Schema.String;
|
|
4120
|
+
}>;
|
|
4121
|
+
}>>;
|
|
4122
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
4123
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
4124
|
+
readonly min: Schema.Number;
|
|
4125
|
+
readonly max: Schema.Number;
|
|
4126
|
+
}>>;
|
|
4127
|
+
}>;
|
|
4128
|
+
readonly durationMs: Schema.Struct<{
|
|
4129
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
4130
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
4131
|
+
readonly samples: Schema.Number;
|
|
4132
|
+
readonly total: Schema.Number;
|
|
4133
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
4134
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
4135
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
4136
|
+
readonly locator: Schema.String;
|
|
4137
|
+
}>, Schema.Struct<{
|
|
4138
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
4139
|
+
readonly runId: Schema.String;
|
|
4140
|
+
readonly plannedAttemptId: Schema.String;
|
|
4141
|
+
readonly evalId: Schema.String;
|
|
4142
|
+
readonly attemptOrdinal: Schema.Number;
|
|
4143
|
+
}>, Schema.Struct<{
|
|
4144
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
4145
|
+
readonly locator: Schema.String;
|
|
4146
|
+
readonly entryId: Schema.String;
|
|
4147
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
4148
|
+
}>, Schema.Struct<{
|
|
4149
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
4150
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
4151
|
+
}>]>>;
|
|
4152
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
4153
|
+
readonly identity: Schema.Struct<{
|
|
4154
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
4155
|
+
readonly locator: Schema.String;
|
|
4156
|
+
}>;
|
|
4157
|
+
}>>;
|
|
4158
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
4159
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
4160
|
+
readonly min: Schema.Number;
|
|
4161
|
+
readonly max: Schema.Number;
|
|
4162
|
+
}>>;
|
|
4163
|
+
}>;
|
|
4164
|
+
readonly tokens: Schema.Struct<{
|
|
4165
|
+
readonly state: Schema.Literals<readonly ["available", "partial", "unavailable", "empty", "unsupported", "failed"]>;
|
|
4166
|
+
readonly value: Schema.NullOr<Schema.Number>;
|
|
4167
|
+
readonly samples: Schema.Number;
|
|
4168
|
+
readonly total: Schema.Number;
|
|
4169
|
+
readonly basis: Schema.Literals<readonly ["planned-attempt", "eval"]>;
|
|
4170
|
+
readonly issues: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
|
|
4171
|
+
readonly code: Schema.Literals<readonly ["assertions-not-recorded", "assertions-revision-unsupported", "assertions-current-invalid"]>;
|
|
4172
|
+
readonly locator: Schema.String;
|
|
4173
|
+
}>, Schema.Struct<{
|
|
4174
|
+
readonly code: Schema.Literals<readonly ["attempt-origin-missing", "attempt-not-observed"]>;
|
|
4175
|
+
readonly runId: Schema.String;
|
|
4176
|
+
readonly plannedAttemptId: Schema.String;
|
|
4177
|
+
readonly evalId: Schema.String;
|
|
4178
|
+
readonly attemptOrdinal: Schema.Number;
|
|
4179
|
+
}>, Schema.Struct<{
|
|
4180
|
+
readonly code: Schema.Literal<"score-contribution-unavailable">;
|
|
4181
|
+
readonly locator: Schema.String;
|
|
4182
|
+
readonly entryId: Schema.String;
|
|
4183
|
+
readonly reason: Schema.Literals<readonly ["source-unavailable", "evaluation-errored", "not-applicable"]>;
|
|
4184
|
+
}>, Schema.Struct<{
|
|
4185
|
+
readonly code: Schema.Literal<"score-partial">;
|
|
4186
|
+
readonly locator: Schema.NullOr<Schema.String>;
|
|
4187
|
+
}>]>>;
|
|
4188
|
+
readonly refs: Schema.$Array<Schema.Struct<{
|
|
4189
|
+
readonly identity: Schema.Struct<{
|
|
4190
|
+
readonly kind: Schema.Literal<"attempt">;
|
|
4191
|
+
readonly locator: Schema.String;
|
|
4192
|
+
}>;
|
|
4193
|
+
}>>;
|
|
4194
|
+
readonly unit: Schema.optional<Schema.Literals<readonly ["points", "USD", "ms", "tokens"]>>;
|
|
4195
|
+
readonly bounds: Schema.optional<Schema.Struct<{
|
|
4196
|
+
readonly min: Schema.Number;
|
|
4197
|
+
readonly max: Schema.Number;
|
|
4198
|
+
}>>;
|
|
4199
|
+
}>;
|
|
4200
|
+
readonly plannedAttemptId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
|
|
4081
4201
|
readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
|
|
4082
4202
|
readonly attemptOrdinal: Schema.Number;
|
|
4203
|
+
readonly publication: Schema.Union<readonly [Schema.Struct<{
|
|
4204
|
+
readonly state: Schema.Literal<"pending">;
|
|
4205
|
+
}>, Schema.Struct<{
|
|
4206
|
+
readonly originPlannedAttemptId: Schema.String;
|
|
4207
|
+
readonly state: Schema.Literal<"published">;
|
|
4208
|
+
readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
|
|
4209
|
+
readonly attemptId: Schema.String;
|
|
4210
|
+
readonly attemptLocator: Schema.String;
|
|
4211
|
+
readonly originRunId: Schema.String;
|
|
4212
|
+
}>, Schema.Struct<{
|
|
4213
|
+
readonly state: Schema.Literal<"absent">;
|
|
4214
|
+
readonly reason: Schema.Literals<readonly ["early-exit-satisfied", "budget-exhausted", "stopped-by-failure", "interrupted-before-publication", "dispatch-failed"]>;
|
|
4215
|
+
}>]>;
|
|
4083
4216
|
readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
|
|
4084
|
-
readonly plannedAttemptId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
|
|
4085
4217
|
}>>;
|
|
4086
4218
|
readonly coverage: Schema.Struct<{
|
|
4087
4219
|
readonly state: Schema.Literals<readonly ["complete", "partial", "not-recorded", "invalid", "unavailable"]>;
|
|
@@ -7394,7 +7526,7 @@ export type InspectionOperationId = keyof typeof inspectionProtocolRegistry;
|
|
|
7394
7526
|
export declare const INSPECTION_OPERATION_IDS: readonly ("run.list" | "run.get" | "project.get" | "overview.get" | "experiment.get" | "attempt.get" | "attempt.assertion.detail" | "attempt.assertion.image" | "attempt.trace" | "attempt.trace.detail" | "attempt.timing" | "attempt.usage" | "attempt.diff" | "attempt.sources" | "attempt.artifacts" | "attempt.artifact" | "runs.compare")[];
|
|
7395
7527
|
export declare const InspectionOperationIdSchema: Schema.Literals<readonly ("run.list" | "run.get" | "project.get" | "overview.get" | "experiment.get" | "attempt.get" | "attempt.assertion.detail" | "attempt.assertion.image" | "attempt.trace" | "attempt.trace.detail" | "attempt.timing" | "attempt.usage" | "attempt.diff" | "attempt.sources" | "attempt.artifacts" | "attempt.artifact" | "runs.compare")[]>;
|
|
7396
7528
|
export declare const InspectionRequestSchema: Schema.Struct<{
|
|
7397
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
7529
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
7398
7530
|
readonly operation: Schema.Union<(Schema.Struct<{
|
|
7399
7531
|
readonly kind: Schema.Literal<"project.get">;
|
|
7400
7532
|
} & {
|
|
@@ -7512,7 +7644,7 @@ export type InspectionOperationFor<Kind extends InspectionOperationId> = Extract
|
|
|
7512
7644
|
}>;
|
|
7513
7645
|
export declare const InspectionSuccessDocumentSchema: Schema.Union<readonly Schema.Codec<InspectionSuccessDocument, InspectionSuccessEncoded>[]>;
|
|
7514
7646
|
export declare const InspectionExplanationDocumentSchema: Schema.Union<Schema.Struct<{
|
|
7515
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
7647
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
7516
7648
|
readonly source: Schema.Struct<{
|
|
7517
7649
|
readonly kind: Schema.Literals<readonly ["project-record", "external-record"]>;
|
|
7518
7650
|
readonly publicationCutoffIdentity: Schema.String;
|
|
@@ -7536,14 +7668,14 @@ export declare const InspectionExplanationDocumentSchema: Schema.Union<Schema.St
|
|
|
7536
7668
|
readonly factKinds: Schema.Tuple<Schema.Literal<"assertions" | "core" | "sources" | "artifacts" | "runner-activities" | "adapter-usage" | "file-changes" | "agent-turns" | "sandbox-commands" | "runner-diagnostics" | "turn-contexts" | "current-project" | "attempt-cost" | "attempt-note" | "judge-usage" | "execution-traces">[]>;
|
|
7537
7669
|
}>[]>;
|
|
7538
7670
|
export declare const InspectionDiscoveryDocumentSchema: Schema.Struct<{
|
|
7539
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
7671
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
7540
7672
|
readonly outcome: Schema.Literal<"discovery">;
|
|
7541
7673
|
readonly operations: Schema.$Array<Schema.Union<Schema.Struct<{
|
|
7542
7674
|
readonly id: Schema.Literal<"run.list" | "run.get" | "project.get" | "overview.get" | "experiment.get" | "attempt.get" | "attempt.assertion.detail" | "attempt.assertion.image" | "attempt.trace" | "attempt.trace.detail" | "attempt.timing" | "attempt.usage" | "attempt.diff" | "attempt.sources" | "attempt.artifacts" | "attempt.artifact" | "runs.compare">;
|
|
7543
7675
|
}>[]>>;
|
|
7544
7676
|
}>;
|
|
7545
7677
|
export declare const InspectionFailureDocumentSchema: Schema.Struct<{
|
|
7546
|
-
readonly protocol: Schema.Literal<"niceeval.query/
|
|
7678
|
+
readonly protocol: Schema.Literal<"niceeval.query/v3">;
|
|
7547
7679
|
readonly outcome: Schema.Literal<"failure">;
|
|
7548
7680
|
readonly operation: Schema.NullOr<Schema.Literals<readonly ("run.list" | "run.get" | "project.get" | "overview.get" | "experiment.get" | "attempt.get" | "attempt.assertion.detail" | "attempt.assertion.image" | "attempt.trace" | "attempt.trace.detail" | "attempt.timing" | "attempt.usage" | "attempt.diff" | "attempt.sources" | "attempt.artifacts" | "attempt.artifact" | "runs.compare")[]>>;
|
|
7549
7681
|
readonly failure: Schema.Struct<{
|