@kortyx/agent 0.38.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +11 -0
- package/dist/evals.d.ts +414 -7
- package/dist/evals.js +495 -330
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +18 -3
- package/dist/index.js +1128 -2940
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-DCAw3v60.d.ts → studio-judge-caDvI8By.d.ts} +973 -43
- package/package.json +11 -10
package/dist/index.js
CHANGED
|
@@ -29,1602 +29,6 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
29
29
|
));
|
|
30
30
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
31
31
|
|
|
32
|
-
// ../telemetry-contracts/dist/index.js
|
|
33
|
-
var require_dist = __commonJS({
|
|
34
|
-
"../telemetry-contracts/dist/index.js"(exports2, module2) {
|
|
35
|
-
var __defProp2 = Object.defineProperty;
|
|
36
|
-
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
|
|
37
|
-
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
|
38
|
-
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
|
|
39
|
-
var __export2 = (target, all) => {
|
|
40
|
-
for (var name2 in all)
|
|
41
|
-
__defProp2(target, name2, { get: all[name2], enumerable: true });
|
|
42
|
-
};
|
|
43
|
-
var __copyProps2 = (to, from, except, desc) => {
|
|
44
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
45
|
-
for (let key of __getOwnPropNames2(from))
|
|
46
|
-
if (!__hasOwnProp2.call(to, key) && key !== except)
|
|
47
|
-
__defProp2(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc2(from, key)) || desc.enumerable });
|
|
48
|
-
}
|
|
49
|
-
return to;
|
|
50
|
-
};
|
|
51
|
-
var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
|
|
52
|
-
var index_exports2 = {};
|
|
53
|
-
__export2(index_exports2, {
|
|
54
|
-
ClearScoreResponseSchema: () => ClearScoreResponseSchema,
|
|
55
|
-
ClearUserFeedbackRequestSchema: () => ClearUserFeedbackRequestSchema,
|
|
56
|
-
DIAGNOSTIC_LIMITS: () => DIAGNOSTIC_LIMITS,
|
|
57
|
-
DiagnosticContentSchema: () => DiagnosticContentSchema,
|
|
58
|
-
DiagnosticCorrelationSchema: () => DiagnosticCorrelationSchema,
|
|
59
|
-
DiagnosticManifestSchema: () => DiagnosticManifestSchema,
|
|
60
|
-
DiagnosticNoteSchema: () => DiagnosticNoteSchema,
|
|
61
|
-
DiagnosticPartSchema: () => DiagnosticPartSchema,
|
|
62
|
-
DiagnosticResponseSchema: () => DiagnosticResponseSchema,
|
|
63
|
-
EnsureWorkflowTopologyRequestSchema: () => EnsureWorkflowTopologyRequestSchema,
|
|
64
|
-
EnsureWorkflowTopologyResponseSchema: () => EnsureWorkflowTopologyResponseSchema,
|
|
65
|
-
FEEDBACK_FILTERS: () => FEEDBACK_FILTERS,
|
|
66
|
-
FeedbackReasonSchema: () => FeedbackReasonSchema,
|
|
67
|
-
FeedbackSummarySchema: () => FeedbackSummarySchema,
|
|
68
|
-
PromptAssetSchema: () => PromptAssetSchema,
|
|
69
|
-
PromptAssignmentSchema: () => PromptAssignmentSchema,
|
|
70
|
-
PromptBundleSchema: () => PromptBundleSchema,
|
|
71
|
-
PromptCategorySchema: () => PromptCategorySchema,
|
|
72
|
-
PromptDetailSchema: () => PromptDetailSchema,
|
|
73
|
-
PromptEvidenceSchema: () => PromptEvidenceSchema,
|
|
74
|
-
PromptGroupSchema: () => PromptGroupSchema,
|
|
75
|
-
PromptLibrarySchema: () => PromptLibrarySchema,
|
|
76
|
-
PromptMutationSchema: () => PromptMutationSchema,
|
|
77
|
-
PromptPolicySchema: () => PromptPolicySchema,
|
|
78
|
-
PromptSelectionSchema: () => PromptSelectionSchema,
|
|
79
|
-
PromptStoredVersionSchema: () => PromptStoredVersionSchema,
|
|
80
|
-
PromptTransferRequestSchema: () => PromptTransferRequestSchema,
|
|
81
|
-
PromptUsageSchema: () => PromptUsageSchema,
|
|
82
|
-
ReviewVerdictSchema: () => ReviewVerdictSchema,
|
|
83
|
-
STUDIO_API_PROTOCOL_VERSION: () => STUDIO_API_PROTOCOL_VERSION,
|
|
84
|
-
STUDIO_TIME_RANGES: () => STUDIO_TIME_RANGES,
|
|
85
|
-
StudioCatalogsResponseSchema: () => StudioCatalogsResponseSchema,
|
|
86
|
-
StudioChangeResourceSchema: () => StudioChangeResourceSchema,
|
|
87
|
-
StudioChangeSchema: () => StudioChangeSchema,
|
|
88
|
-
StudioContextResponseSchema: () => StudioContextResponseSchema,
|
|
89
|
-
StudioCostSummarySchema: () => StudioCostSummarySchema,
|
|
90
|
-
StudioDetailEventSchema: () => StudioDetailEventSchema,
|
|
91
|
-
StudioInterruptDetailResponseSchema: () => StudioInterruptDetailResponseSchema,
|
|
92
|
-
StudioInterruptInteractionModeSchema: () => StudioInterruptInteractionModeSchema,
|
|
93
|
-
StudioInterruptOptionSchema: () => StudioInterruptOptionSchema,
|
|
94
|
-
StudioInterruptSchema: () => StudioInterruptSchema,
|
|
95
|
-
StudioInterruptStatusSchema: () => StudioInterruptStatusSchema,
|
|
96
|
-
StudioInterruptTypeSchema: () => StudioInterruptTypeSchema,
|
|
97
|
-
StudioInterruptValueSchema: () => StudioInterruptValueSchema,
|
|
98
|
-
StudioInterruptsResponseSchema: () => StudioInterruptsResponseSchema,
|
|
99
|
-
StudioMetricSchema: () => StudioMetricSchema,
|
|
100
|
-
StudioPricingSourceSchema: () => StudioPricingSourceSchema,
|
|
101
|
-
StudioPricingStatusSchema: () => StudioPricingStatusSchema,
|
|
102
|
-
StudioReleaseDeploymentSchema: () => StudioReleaseDeploymentSchema,
|
|
103
|
-
StudioReleaseSchema: () => StudioReleaseSchema,
|
|
104
|
-
StudioReleaseVersionSchema: () => StudioReleaseVersionSchema,
|
|
105
|
-
StudioResumeOutcomeSchema: () => StudioResumeOutcomeSchema,
|
|
106
|
-
StudioReviewRequestSchema: () => StudioReviewRequestSchema,
|
|
107
|
-
StudioRunDetailResponseSchema: () => StudioRunDetailResponseSchema,
|
|
108
|
-
StudioRunSchema: () => StudioRunSchema,
|
|
109
|
-
StudioRunStatusSchema: () => StudioRunStatusSchema,
|
|
110
|
-
StudioRunsResponseSchema: () => StudioRunsResponseSchema,
|
|
111
|
-
StudioScoreResponseSchema: () => StudioScoreResponseSchema,
|
|
112
|
-
StudioScoreSchema: () => StudioScoreSchema,
|
|
113
|
-
StudioSessionDetailResponseSchema: () => StudioSessionDetailResponseSchema,
|
|
114
|
-
StudioSessionSchema: () => StudioSessionSchema,
|
|
115
|
-
StudioSessionsResponseSchema: () => StudioSessionsResponseSchema,
|
|
116
|
-
StudioTimeRangeContextSchema: () => StudioTimeRangeContextSchema,
|
|
117
|
-
StudioTimeRangeQuerySchema: () => StudioTimeRangeQuerySchema,
|
|
118
|
-
StudioTimeRangeSchema: () => StudioTimeRangeSchema,
|
|
119
|
-
StudioToolSchema: () => StudioToolSchema,
|
|
120
|
-
StudioUpdateOperationSchema: () => StudioUpdateOperationSchema,
|
|
121
|
-
StudioUpdateSettingsSchema: () => StudioUpdateSettingsSchema,
|
|
122
|
-
StudioUpdateStatusSchema: () => StudioUpdateStatusSchema,
|
|
123
|
-
StudioWorkflowHealthSchema: () => StudioWorkflowHealthSchema,
|
|
124
|
-
StudioWorkflowInternalEdgeSchema: () => StudioWorkflowInternalEdgeSchema,
|
|
125
|
-
StudioWorkflowNodeSchema: () => StudioWorkflowNodeSchema,
|
|
126
|
-
StudioWorkflowSchema: () => StudioWorkflowSchema,
|
|
127
|
-
StudioWorkflowTransitionSchema: () => StudioWorkflowTransitionSchema,
|
|
128
|
-
StudioWorkflowsResponseSchema: () => StudioWorkflowsResponseSchema,
|
|
129
|
-
TELEMETRY_EVENT_TYPES: () => TELEMETRY_EVENT_TYPES,
|
|
130
|
-
TelemetryEventBatchResponseSchema: () => TelemetryEventBatchResponseSchema,
|
|
131
|
-
TelemetryEventBatchSchema: () => TelemetryEventBatchSchema,
|
|
132
|
-
TelemetryEventSchema: () => TelemetryEventSchema,
|
|
133
|
-
TelemetryEventTypeSchema: () => TelemetryEventTypeSchema,
|
|
134
|
-
TelemetryPricingHintSchema: () => TelemetryPricingHintSchema,
|
|
135
|
-
TelemetryPricingLineItemSchema: () => TelemetryPricingLineItemSchema,
|
|
136
|
-
TelemetryPricingSourceSchema: () => TelemetryPricingSourceSchema,
|
|
137
|
-
TelemetryPricingUnitSchema: () => TelemetryPricingUnitSchema,
|
|
138
|
-
TelemetryPricingUsageTypeSchema: () => TelemetryPricingUsageTypeSchema,
|
|
139
|
-
TelemetryServiceSchema: () => TelemetryServiceSchema,
|
|
140
|
-
TelemetryUnitPriceSchema: () => TelemetryUnitPriceSchema,
|
|
141
|
-
TelemetryUsageItemSchema: () => TelemetryUsageItemSchema,
|
|
142
|
-
ToolDiscoverySchema: () => ToolDiscoverySchema,
|
|
143
|
-
ToolObservationSchema: () => ToolObservationSchema,
|
|
144
|
-
ToolOutcomeSchema: () => ToolOutcomeSchema,
|
|
145
|
-
UserFeedbackRequestSchema: () => UserFeedbackRequestSchema,
|
|
146
|
-
WorkflowToolSchema: () => WorkflowToolSchema,
|
|
147
|
-
WorkflowTopologyCallSchema: () => WorkflowTopologyCallSchema,
|
|
148
|
-
WorkflowTopologyEdgeSchema: () => WorkflowTopologyEdgeSchema,
|
|
149
|
-
WorkflowTopologyNodeSchema: () => WorkflowTopologyNodeSchema,
|
|
150
|
-
WorkflowTopologyTransitionSchema: () => WorkflowTopologyTransitionSchema,
|
|
151
|
-
isPrivateDiagnosticField: () => isPrivateDiagnosticField,
|
|
152
|
-
newerStudioVersion: () => newerStudioVersion,
|
|
153
|
-
projectWorkflowCalls: () => projectWorkflowCalls,
|
|
154
|
-
redactDiagnosticContent: () => redactDiagnosticContent,
|
|
155
|
-
redactDiagnosticText: () => redactDiagnosticText,
|
|
156
|
-
resolveStudioInterruptStatus: () => resolveStudioInterruptStatus,
|
|
157
|
-
resolveStudioTimeRange: () => resolveStudioTimeRange,
|
|
158
|
-
studioUpdateRunning: () => studioUpdateRunning,
|
|
159
|
-
summarizeDiagnostic: () => summarizeDiagnostic2,
|
|
160
|
-
summarizeUserFeedback: () => summarizeUserFeedback
|
|
161
|
-
});
|
|
162
|
-
module2.exports = __toCommonJS2(index_exports2);
|
|
163
|
-
var import_zod62 = require("zod");
|
|
164
|
-
var import_zod14 = require("zod");
|
|
165
|
-
var FEEDBACK_FILTERS = ["positive", "negative", "unrated"];
|
|
166
|
-
var FeedbackSummarySchema = import_zod14.z.object({
|
|
167
|
-
positive: import_zod14.z.number().int().nonnegative(),
|
|
168
|
-
negative: import_zod14.z.number().int().nonnegative()
|
|
169
|
-
}).strict();
|
|
170
|
-
var FeedbackReasonSchema = import_zod14.z.enum([
|
|
171
|
-
"incorrect",
|
|
172
|
-
"irrelevant",
|
|
173
|
-
"incomplete",
|
|
174
|
-
"unsafe",
|
|
175
|
-
"other"
|
|
176
|
-
]);
|
|
177
|
-
var ReviewVerdictSchema = import_zod14.z.enum([
|
|
178
|
-
"correct",
|
|
179
|
-
"partially-correct",
|
|
180
|
-
"incorrect"
|
|
181
|
-
]);
|
|
182
|
-
var runId = import_zod14.z.string().trim().min(1).max(512);
|
|
183
|
-
var actorId = import_zod14.z.string().trim().min(1).max(256);
|
|
184
|
-
var comment = import_zod14.z.string().trim().max(4e3).nullable().optional();
|
|
185
|
-
var UserFeedbackRequestSchema = import_zod14.z.object({
|
|
186
|
-
runId,
|
|
187
|
-
actorId,
|
|
188
|
-
value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)]),
|
|
189
|
-
reasons: import_zod14.z.array(FeedbackReasonSchema).max(5).optional(),
|
|
190
|
-
comment
|
|
191
|
-
}).strict();
|
|
192
|
-
var ClearUserFeedbackRequestSchema = UserFeedbackRequestSchema.pick({
|
|
193
|
-
runId: true,
|
|
194
|
-
actorId: true
|
|
195
|
-
});
|
|
196
|
-
var StudioReviewRequestSchema = import_zod14.z.object({ value: ReviewVerdictSchema, comment }).strict();
|
|
197
|
-
var BaseScoreSchema = import_zod14.z.object({
|
|
198
|
-
id: import_zod14.z.string().uuid(),
|
|
199
|
-
target: import_zod14.z.object({ type: import_zod14.z.literal("run"), runId }).strict(),
|
|
200
|
-
environment: import_zod14.z.string().min(1),
|
|
201
|
-
name: import_zod14.z.string().min(1),
|
|
202
|
-
source: import_zod14.z.enum(["end-user", "human-review", "evaluator"]),
|
|
203
|
-
actorId,
|
|
204
|
-
reasons: import_zod14.z.array(FeedbackReasonSchema),
|
|
205
|
-
comment: import_zod14.z.string().nullable(),
|
|
206
|
-
createdAt: import_zod14.z.string().datetime({ offset: true }),
|
|
207
|
-
updatedAt: import_zod14.z.string().datetime({ offset: true })
|
|
208
|
-
}).strict();
|
|
209
|
-
var StudioScoreSchema = import_zod14.z.discriminatedUnion("dataType", [
|
|
210
|
-
BaseScoreSchema.extend({
|
|
211
|
-
dataType: import_zod14.z.literal("BOOLEAN"),
|
|
212
|
-
value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)])
|
|
213
|
-
}),
|
|
214
|
-
BaseScoreSchema.extend({
|
|
215
|
-
dataType: import_zod14.z.literal("CATEGORICAL"),
|
|
216
|
-
value: import_zod14.z.string()
|
|
217
|
-
}),
|
|
218
|
-
BaseScoreSchema.extend({
|
|
219
|
-
dataType: import_zod14.z.literal("NUMERIC"),
|
|
220
|
-
value: import_zod14.z.number().finite()
|
|
221
|
-
})
|
|
222
|
-
]);
|
|
223
|
-
var StudioScoreResponseSchema = import_zod14.z.object({ score: StudioScoreSchema }).strict();
|
|
224
|
-
var ClearScoreResponseSchema = import_zod14.z.object({ ok: import_zod14.z.literal(true) }).strict();
|
|
225
|
-
function summarizeUserFeedback(scores) {
|
|
226
|
-
const feedback = { positive: 0, negative: 0 };
|
|
227
|
-
for (const score of scores) {
|
|
228
|
-
if (score.source !== "end-user" || score.name !== "user-feedback" || score.dataType !== "BOOLEAN")
|
|
229
|
-
continue;
|
|
230
|
-
if (score.value === 1) feedback.positive += 1;
|
|
231
|
-
if (score.value === 0) feedback.negative += 1;
|
|
232
|
-
}
|
|
233
|
-
return feedback;
|
|
234
|
-
}
|
|
235
|
-
var import_zod22 = require("zod");
|
|
236
|
-
var DIAGNOSTIC_LIMITS = {
|
|
237
|
-
bytes: 8 * 1024 * 1024,
|
|
238
|
-
partBytes: 64 * 1024,
|
|
239
|
-
parts: 128,
|
|
240
|
-
nodes: 1e4,
|
|
241
|
-
depth: 128,
|
|
242
|
-
properties: 1e3
|
|
243
|
-
};
|
|
244
|
-
var DiagnosticNoteSchema = import_zod22.z.object({
|
|
245
|
-
path: import_zod22.z.string().max(4096),
|
|
246
|
-
reason: import_zod22.z.string().max(128),
|
|
247
|
-
originalBytes: import_zod22.z.number().int().nonnegative().optional(),
|
|
248
|
-
originalCount: import_zod22.z.number().int().nonnegative().optional()
|
|
249
|
-
}).strict();
|
|
250
|
-
var DiagnosticContentSchema = import_zod22.z.object({
|
|
251
|
-
schemaVersion: import_zod22.z.literal(1),
|
|
252
|
-
data: import_zod22.z.record(import_zod22.z.string(), import_zod22.z.unknown()),
|
|
253
|
-
capture: import_zod22.z.object({
|
|
254
|
-
status: import_zod22.z.enum(["complete", "partial", "failed"]),
|
|
255
|
-
omissions: import_zod22.z.array(DiagnosticNoteSchema).max(1e4),
|
|
256
|
-
redactions: import_zod22.z.array(DiagnosticNoteSchema).max(1e4)
|
|
257
|
-
}).strict()
|
|
258
|
-
}).strict();
|
|
259
|
-
var DiagnosticCorrelationSchema = import_zod22.z.object({
|
|
260
|
-
runId: import_zod22.z.string().optional(),
|
|
261
|
-
workflowId: import_zod22.z.string().optional(),
|
|
262
|
-
nodeId: import_zod22.z.string().optional(),
|
|
263
|
-
sessionId: import_zod22.z.string().optional(),
|
|
264
|
-
invocationId: import_zod22.z.string().optional(),
|
|
265
|
-
parentInvocationId: import_zod22.z.string().optional(),
|
|
266
|
-
branchId: import_zod22.z.string().optional(),
|
|
267
|
-
toolCallId: import_zod22.z.string().optional(),
|
|
268
|
-
attemptId: import_zod22.z.string().optional(),
|
|
269
|
-
workflowRevisionId: import_zod22.z.string().optional(),
|
|
270
|
-
traceId: import_zod22.z.string().optional(),
|
|
271
|
-
spanId: import_zod22.z.string().optional(),
|
|
272
|
-
parentSpanId: import_zod22.z.string().optional()
|
|
273
|
-
}).strict();
|
|
274
|
-
var DiagnosticManifestSchema = import_zod22.z.object({
|
|
275
|
-
schemaVersion: import_zod22.z.literal(1),
|
|
276
|
-
diagnosticId: import_zod22.z.string().uuid(),
|
|
277
|
-
occurrenceId: import_zod22.z.string().uuid(),
|
|
278
|
-
occurredAt: import_zod22.z.string().datetime({ offset: true }),
|
|
279
|
-
environment: import_zod22.z.string().min(1).max(256),
|
|
280
|
-
service: import_zod22.z.object({
|
|
281
|
-
name: import_zod22.z.string().min(1).max(256),
|
|
282
|
-
deploymentRef: import_zod22.z.string().max(1024).optional()
|
|
283
|
-
}).strict(),
|
|
284
|
-
correlation: DiagnosticCorrelationSchema,
|
|
285
|
-
handled: import_zod22.z.boolean(),
|
|
286
|
-
severity: import_zod22.z.enum(["warning", "error"]),
|
|
287
|
-
mechanism: import_zod22.z.enum(["span", "report", "tool"]),
|
|
288
|
-
summary: import_zod22.z.object({ type: import_zod22.z.string().max(256), message: import_zod22.z.string().max(8192) }).strict(),
|
|
289
|
-
captureStatus: import_zod22.z.enum(["complete", "partial", "failed"]),
|
|
290
|
-
checksum: import_zod22.z.string().regex(/^[a-f0-9]{64}$/),
|
|
291
|
-
byteLength: import_zod22.z.number().int().min(1).max(DIAGNOSTIC_LIMITS.bytes),
|
|
292
|
-
partCount: import_zod22.z.number().int().min(1).max(DIAGNOSTIC_LIMITS.parts)
|
|
293
|
-
}).strict().superRefine((value, ctx) => {
|
|
294
|
-
if (value.partCount !== Math.ceil(value.byteLength / DIAGNOSTIC_LIMITS.partBytes))
|
|
295
|
-
ctx.addIssue({
|
|
296
|
-
code: "custom",
|
|
297
|
-
path: ["partCount"],
|
|
298
|
-
message: "Part count does not match byte length."
|
|
299
|
-
});
|
|
300
|
-
});
|
|
301
|
-
var DiagnosticPartSchema = import_zod22.z.object({
|
|
302
|
-
index: import_zod22.z.number().int().min(0).max(DIAGNOSTIC_LIMITS.parts - 1),
|
|
303
|
-
data: import_zod22.z.string().max(Math.ceil(DIAGNOSTIC_LIMITS.partBytes / 3) * 4).regex(
|
|
304
|
-
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
|
|
305
|
-
)
|
|
306
|
-
}).strict();
|
|
307
|
-
var DiagnosticResponseSchema = import_zod22.z.object({
|
|
308
|
-
manifest: DiagnosticManifestSchema,
|
|
309
|
-
state: import_zod22.z.enum(["pending", "available", "incomplete", "expired"]),
|
|
310
|
-
receivedParts: import_zod22.z.number().int().nonnegative(),
|
|
311
|
-
expiresAt: import_zod22.z.string().datetime({ offset: true }),
|
|
312
|
-
content: DiagnosticContentSchema.nullable(),
|
|
313
|
-
contentChecksum: import_zod22.z.string().regex(/^[a-f0-9]{64}$/).nullable(),
|
|
314
|
-
contentByteLength: import_zod22.z.number().int().nonnegative().nullable()
|
|
315
|
-
}).strict();
|
|
316
|
-
var isPrivateDiagnosticField = (key) => /^(?:password|passwd|secret|clientsecret|privatekey|credential|credentials|authorization|proxyauthorization|bearer|bearertoken|jwt|token|accesstoken|refreshtoken|authtoken|apikey|xapikey|apitoken|xauthtoken|xaccesstoken|awsaccesskeyid|awssecretaccesskey|awssessiontoken|xamzsecuritytoken|apisecret|cookie|setcookie|resumehandle|resumetoken)$/.test(
|
|
317
|
-
key.toLowerCase().replace(/[^a-z0-9]/g, "")
|
|
318
|
-
);
|
|
319
|
-
function redactDiagnosticText(value) {
|
|
320
|
-
return value.replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [REDACTED]").replace(
|
|
321
|
-
/\b(?:ktyx_(?:test|live)_[^\s"'<>]+|sk-(?:proj-|ant-)?[A-Za-z0-9_-]{12,}|AIza[A-Za-z0-9_-]{20,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/g,
|
|
322
|
-
"[REDACTED]"
|
|
323
|
-
).replace(
|
|
324
|
-
/(["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|passwd|secret|token|client[_-]?secret|authorization|cookie)["']?\s*[:=]\s*)(["'])(.*?)\2/gi,
|
|
325
|
-
"$1$2[REDACTED]$2"
|
|
326
|
-
).replace(
|
|
327
|
-
/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|passwd|secret|token|client[_-]?secret)\s*[:=]\s*)(?!["'[])[^\s,;}&]+/gi,
|
|
328
|
-
"$1[REDACTED]"
|
|
329
|
-
).replace(
|
|
330
|
-
/(\b(?:authorization|proxy-authorization)\s*:\s*Basic\s+)[A-Za-z0-9+/=]+/gi,
|
|
331
|
-
"$1[REDACTED]"
|
|
332
|
-
).replace(
|
|
333
|
-
/([?&](?:api[_-]?key|token|access[_-]?token|password)=)[^&#\s]+/gi,
|
|
334
|
-
"$1[REDACTED]"
|
|
335
|
-
).replace(/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@");
|
|
336
|
-
}
|
|
337
|
-
function summarizeDiagnostic2(data, depth = 0) {
|
|
338
|
-
return {
|
|
339
|
-
type: typeof data.type === "string" ? data.type.slice(0, 256) : "Error",
|
|
340
|
-
message: typeof data.message === "string" ? data.message.slice(0, 8192) : "Execution reported a fault.",
|
|
341
|
-
...typeof data.stack === "string" ? { stack: data.stack.slice(0, 32768) } : {},
|
|
342
|
-
...depth < 4 && data.cause && typeof data.cause === "object" && !Array.isArray(data.cause) ? {
|
|
343
|
-
cause: summarizeDiagnostic2(
|
|
344
|
-
data.cause,
|
|
345
|
-
depth + 1
|
|
346
|
-
)
|
|
347
|
-
} : {}
|
|
348
|
-
};
|
|
349
|
-
}
|
|
350
|
-
function redactDiagnosticContent(content) {
|
|
351
|
-
const redactions = [...content.capture.redactions];
|
|
352
|
-
const omissions = [...content.capture.omissions];
|
|
353
|
-
let nodes = 0;
|
|
354
|
-
const visit = (value, path, depth = 0) => {
|
|
355
|
-
if (++nodes > DIAGNOSTIC_LIMITS.nodes || depth > DIAGNOSTIC_LIMITS.depth) {
|
|
356
|
-
const reason = depth > DIAGNOSTIC_LIMITS.depth ? "server_depth_limit" : "server_node_limit";
|
|
357
|
-
if (omissions.length < 1e4)
|
|
358
|
-
omissions.push({ path: path.slice(0, 4096), reason });
|
|
359
|
-
return { $omitted: reason };
|
|
360
|
-
}
|
|
361
|
-
if (typeof value === "string") {
|
|
362
|
-
const clean = redactDiagnosticText(value);
|
|
363
|
-
if (clean !== value && redactions.length < 1e4)
|
|
364
|
-
redactions.push({
|
|
365
|
-
path: path.slice(0, 4096),
|
|
366
|
-
reason: "server_credential_text"
|
|
367
|
-
});
|
|
368
|
-
return clean;
|
|
369
|
-
}
|
|
370
|
-
if (Array.isArray(value)) {
|
|
371
|
-
if (value.length > DIAGNOSTIC_LIMITS.properties && omissions.length < 1e4)
|
|
372
|
-
omissions.push({
|
|
373
|
-
path: path.slice(0, 4096),
|
|
374
|
-
reason: "server_collection_limit",
|
|
375
|
-
originalCount: value.length
|
|
376
|
-
});
|
|
377
|
-
return value.slice(0, DIAGNOSTIC_LIMITS.properties).map((item, index) => visit(item, `${path}/${index}`, depth + 1));
|
|
378
|
-
}
|
|
379
|
-
if (value && typeof value === "object") {
|
|
380
|
-
const keys = Object.keys(value);
|
|
381
|
-
if (keys.length > DIAGNOSTIC_LIMITS.properties && omissions.length < 1e4)
|
|
382
|
-
omissions.push({
|
|
383
|
-
path: path.slice(0, 4096),
|
|
384
|
-
reason: "server_property_limit",
|
|
385
|
-
originalCount: keys.length
|
|
386
|
-
});
|
|
387
|
-
return Object.fromEntries(
|
|
388
|
-
keys.slice(0, DIAGNOSTIC_LIMITS.properties).map((key) => {
|
|
389
|
-
const nested = value[key];
|
|
390
|
-
if (isPrivateDiagnosticField(key)) {
|
|
391
|
-
if (redactions.length < 1e4 && nested !== "[REDACTED]")
|
|
392
|
-
redactions.push({
|
|
393
|
-
path: `${path}/${key}`.slice(0, 4096),
|
|
394
|
-
reason: "server_credential_field"
|
|
395
|
-
});
|
|
396
|
-
return [key, "[REDACTED]"];
|
|
397
|
-
}
|
|
398
|
-
return [key, visit(nested, `${path}/${key}`, depth + 1)];
|
|
399
|
-
})
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
return value;
|
|
403
|
-
};
|
|
404
|
-
return {
|
|
405
|
-
...content,
|
|
406
|
-
data: visit(content.data, "#/data"),
|
|
407
|
-
capture: {
|
|
408
|
-
...content.capture,
|
|
409
|
-
status: content.capture.status === "failed" ? "failed" : omissions.length ? "partial" : "complete",
|
|
410
|
-
omissions: omissions.map((note) => ({
|
|
411
|
-
...note,
|
|
412
|
-
path: redactDiagnosticText(note.path),
|
|
413
|
-
reason: redactDiagnosticText(note.reason)
|
|
414
|
-
})),
|
|
415
|
-
redactions: redactions.map((note) => ({
|
|
416
|
-
...note,
|
|
417
|
-
path: redactDiagnosticText(note.path),
|
|
418
|
-
reason: redactDiagnosticText(note.reason)
|
|
419
|
-
}))
|
|
420
|
-
}
|
|
421
|
-
};
|
|
422
|
-
}
|
|
423
|
-
var import_zod32 = require("zod");
|
|
424
|
-
var ToolOutcomeSchema = import_zod32.z.enum([
|
|
425
|
-
"success",
|
|
426
|
-
"denied",
|
|
427
|
-
"fault",
|
|
428
|
-
"cancelled"
|
|
429
|
-
]);
|
|
430
|
-
var ToolObservationSchema = import_zod32.z.object({
|
|
431
|
-
version: import_zod32.z.literal(1),
|
|
432
|
-
name: import_zod32.z.string().min(1).max(256),
|
|
433
|
-
toolCallId: import_zod32.z.string().min(1),
|
|
434
|
-
attemptId: import_zod32.z.string().min(1),
|
|
435
|
-
callingMode: import_zod32.z.enum(["direct", "model"]),
|
|
436
|
-
executed: import_zod32.z.boolean(),
|
|
437
|
-
outcome: ToolOutcomeSchema.optional(),
|
|
438
|
-
denialCode: import_zod32.z.string().regex(/^[A-Z][A-Z0-9_]{0,63}$/).optional(),
|
|
439
|
-
errorType: import_zod32.z.string().max(256).optional(),
|
|
440
|
-
errorMessage: import_zod32.z.string().max(8192).optional(),
|
|
441
|
-
diagnosticId: import_zod32.z.string().uuid().optional(),
|
|
442
|
-
durationMs: import_zod32.z.number().finite().nonnegative().optional(),
|
|
443
|
-
workflowId: import_zod32.z.string().optional(),
|
|
444
|
-
nodeId: import_zod32.z.string().optional(),
|
|
445
|
-
workflowRevisionId: import_zod32.z.string().optional(),
|
|
446
|
-
runId: import_zod32.z.string().optional(),
|
|
447
|
-
invocationId: import_zod32.z.string().optional(),
|
|
448
|
-
parentInvocationId: import_zod32.z.string().optional(),
|
|
449
|
-
branchId: import_zod32.z.string().optional(),
|
|
450
|
-
observationKind: import_zod32.z.enum(["reused", "waiting"]).optional(),
|
|
451
|
-
suspended: import_zod32.z.boolean().optional(),
|
|
452
|
-
source: import_zod32.z.object({
|
|
453
|
-
runId: import_zod32.z.string().optional(),
|
|
454
|
-
invocationId: import_zod32.z.string().optional(),
|
|
455
|
-
branchId: import_zod32.z.string().optional(),
|
|
456
|
-
toolCallId: import_zod32.z.string(),
|
|
457
|
-
attemptId: import_zod32.z.string()
|
|
458
|
-
}).strict().optional()
|
|
459
|
-
});
|
|
460
|
-
var WorkflowToolSchema = import_zod32.z.object({
|
|
461
|
-
name: import_zod32.z.string().min(1).max(256),
|
|
462
|
-
description: import_zod32.z.string().max(4096).optional(),
|
|
463
|
-
callingMode: import_zod32.z.enum(["direct", "model"]),
|
|
464
|
-
provenance: import_zod32.z.enum(["source", "observed"]),
|
|
465
|
-
inputFields: import_zod32.z.array(
|
|
466
|
-
import_zod32.z.object({ name: import_zod32.z.string(), type: import_zod32.z.string(), required: import_zod32.z.boolean() }).strict()
|
|
467
|
-
).optional()
|
|
468
|
-
}).strict();
|
|
469
|
-
var ToolDiscoverySchema = import_zod32.z.object({
|
|
470
|
-
status: import_zod32.z.enum(["complete", "unresolved"]),
|
|
471
|
-
publishedAt: import_zod32.z.string().datetime().optional(),
|
|
472
|
-
warnings: import_zod32.z.array(import_zod32.z.string()).optional()
|
|
473
|
-
}).strict();
|
|
474
|
-
var StudioToolSchema = WorkflowToolSchema.extend({
|
|
475
|
-
callingMode: import_zod32.z.enum(["direct", "model", "unknown"]),
|
|
476
|
-
calls: import_zod32.z.number().nonnegative(),
|
|
477
|
-
replays: import_zod32.z.number().nonnegative(),
|
|
478
|
-
successes: import_zod32.z.number().nonnegative(),
|
|
479
|
-
denials: import_zod32.z.number().nonnegative(),
|
|
480
|
-
faults: import_zod32.z.number().nonnegative(),
|
|
481
|
-
cancellations: import_zod32.z.number().nonnegative(),
|
|
482
|
-
p50DurationMs: import_zod32.z.number().nonnegative().nullable(),
|
|
483
|
-
p95DurationMs: import_zod32.z.number().nonnegative().nullable()
|
|
484
|
-
}).strict();
|
|
485
|
-
var import_browser4 = require("@kortyx/prompts/browser");
|
|
486
|
-
var import_zod42 = require("zod");
|
|
487
|
-
var name = import_zod42.z.string().trim().min(1).max(200);
|
|
488
|
-
var version = import_zod42.z.number().int().positive();
|
|
489
|
-
var revision = import_zod42.z.number().int().nonnegative();
|
|
490
|
-
var PromptCategorySchema = import_zod42.z.object({
|
|
491
|
-
id: import_zod42.z.uuid(),
|
|
492
|
-
name,
|
|
493
|
-
parentId: import_zod42.z.uuid().nullable(),
|
|
494
|
-
revision
|
|
495
|
-
});
|
|
496
|
-
var PromptGroupSchema = import_zod42.z.object({
|
|
497
|
-
id: import_zod42.z.uuid(),
|
|
498
|
-
name,
|
|
499
|
-
revision,
|
|
500
|
-
members: import_zod42.z.array(
|
|
501
|
-
import_zod42.z.object({
|
|
502
|
-
promptId: import_zod42.z.uuid(),
|
|
503
|
-
version,
|
|
504
|
-
key: import_browser4.PromptKeySchema.optional(),
|
|
505
|
-
name: name.optional(),
|
|
506
|
-
archived: import_zod42.z.boolean().optional()
|
|
507
|
-
})
|
|
508
|
-
).max(100),
|
|
509
|
-
updatedAt: import_zod42.z.string()
|
|
510
|
-
});
|
|
511
|
-
var PromptAssignmentSchema = import_zod42.z.object({
|
|
512
|
-
tag: import_browser4.PromptTagSchema,
|
|
513
|
-
version,
|
|
514
|
-
revision,
|
|
515
|
-
updatedAt: import_zod42.z.string()
|
|
516
|
-
});
|
|
517
|
-
var PromptAssetSchema = import_zod42.z.object({
|
|
518
|
-
id: import_zod42.z.uuid(),
|
|
519
|
-
key: import_browser4.PromptKeySchema,
|
|
520
|
-
name,
|
|
521
|
-
categoryId: import_zod42.z.uuid().nullable(),
|
|
522
|
-
latestVersion: version,
|
|
523
|
-
revision,
|
|
524
|
-
archived: import_zod42.z.boolean(),
|
|
525
|
-
updatedAt: import_zod42.z.string(),
|
|
526
|
-
assignments: import_zod42.z.array(PromptAssignmentSchema)
|
|
527
|
-
});
|
|
528
|
-
var PromptLibrarySchema = import_zod42.z.object({
|
|
529
|
-
schemaVersion: import_zod42.z.literal(1),
|
|
530
|
-
assets: import_zod42.z.array(PromptAssetSchema),
|
|
531
|
-
categories: import_zod42.z.array(PromptCategorySchema),
|
|
532
|
-
groups: import_zod42.z.array(PromptGroupSchema),
|
|
533
|
-
totalCount: import_zod42.z.number().int().nonnegative(),
|
|
534
|
-
nextCursor: import_zod42.z.string().nullable(),
|
|
535
|
-
permissions: import_zod42.z.object({
|
|
536
|
-
edit: import_zod42.z.boolean(),
|
|
537
|
-
promote: import_zod42.z.boolean(),
|
|
538
|
-
review: import_zod42.z.boolean(),
|
|
539
|
-
settings: import_zod42.z.boolean()
|
|
540
|
-
})
|
|
541
|
-
});
|
|
542
|
-
var PromptStoredVersionSchema = import_browser4.PromptVersionSchema.extend({
|
|
543
|
-
promptId: import_zod42.z.uuid(),
|
|
544
|
-
note: import_zod42.z.string(),
|
|
545
|
-
author: import_zod42.z.string(),
|
|
546
|
-
createdAt: import_zod42.z.string(),
|
|
547
|
-
origin: import_zod42.z.object({
|
|
548
|
-
apiUrl: import_zod42.z.string(),
|
|
549
|
-
projectId: import_zod42.z.string(),
|
|
550
|
-
key: import_zod42.z.string(),
|
|
551
|
-
version,
|
|
552
|
-
hash: import_zod42.z.string()
|
|
553
|
-
}).nullable()
|
|
554
|
-
});
|
|
555
|
-
var PromptPolicySchema = import_zod42.z.object({
|
|
556
|
-
revision,
|
|
557
|
-
requireTest: import_zod42.z.boolean(),
|
|
558
|
-
requireChangeNote: import_zod42.z.boolean().default(false),
|
|
559
|
-
requiredSuites: import_zod42.z.array(import_zod42.z.object({ targetId: name, suiteId: name })).max(100),
|
|
560
|
-
requiredReviews: import_zod42.z.number().int().min(0).max(10),
|
|
561
|
-
allowException: import_zod42.z.boolean()
|
|
562
|
-
});
|
|
563
|
-
var PromptEvidenceSchema = import_zod42.z.object({
|
|
564
|
-
environment: import_zod42.z.string(),
|
|
565
|
-
runId: import_zod42.z.string(),
|
|
566
|
-
evaluationId: import_zod42.z.string().nullable().optional(),
|
|
567
|
-
targetId: import_zod42.z.string(),
|
|
568
|
-
suiteId: import_zod42.z.string(),
|
|
569
|
-
suiteRevision: import_zod42.z.string().optional(),
|
|
570
|
-
status: import_zod42.z.string(),
|
|
571
|
-
version,
|
|
572
|
-
fullSuite: import_zod42.z.boolean(),
|
|
573
|
-
usage: import_zod42.z.enum([
|
|
574
|
-
"verified",
|
|
575
|
-
"pending",
|
|
576
|
-
"not-used",
|
|
577
|
-
"mismatch",
|
|
578
|
-
"context-differs"
|
|
579
|
-
]),
|
|
580
|
-
groupName: import_zod42.z.string().nullable(),
|
|
581
|
-
createdAt: import_zod42.z.string(),
|
|
582
|
-
companions: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version }))
|
|
583
|
-
});
|
|
584
|
-
var PromptUsageSchema = import_zod42.z.object({
|
|
585
|
-
eventId: import_zod42.z.string(),
|
|
586
|
-
runId: import_zod42.z.string(),
|
|
587
|
-
sessionId: import_zod42.z.string().nullable(),
|
|
588
|
-
nodeId: import_zod42.z.string().nullable(),
|
|
589
|
-
version: import_zod42.z.number().int(),
|
|
590
|
-
hash: import_zod42.z.string().nullable(),
|
|
591
|
-
source: import_zod42.z.string().nullable(),
|
|
592
|
-
model: import_zod42.z.string().nullable(),
|
|
593
|
-
environment: import_zod42.z.string(),
|
|
594
|
-
occurredAt: import_zod42.z.string(),
|
|
595
|
-
captured: import_zod42.z.boolean()
|
|
596
|
-
});
|
|
597
|
-
var PromptDetailSchema = import_zod42.z.object({
|
|
598
|
-
asset: PromptAssetSchema,
|
|
599
|
-
versions: import_zod42.z.array(PromptStoredVersionSchema),
|
|
600
|
-
versionsNextCursor: import_zod42.z.string().nullable().optional(),
|
|
601
|
-
draft: import_browser4.PromptContentSchema.nullable(),
|
|
602
|
-
draftBase: import_zod42.z.number().int().nullable(),
|
|
603
|
-
draftRevision: revision,
|
|
604
|
-
evidence: import_zod42.z.array(PromptEvidenceSchema),
|
|
605
|
-
usage: import_zod42.z.array(PromptUsageSchema),
|
|
606
|
-
activity: import_zod42.z.array(
|
|
607
|
-
import_zod42.z.object({
|
|
608
|
-
id: import_zod42.z.string(),
|
|
609
|
-
action: import_zod42.z.string(),
|
|
610
|
-
actor: import_zod42.z.string(),
|
|
611
|
-
details: import_zod42.z.record(import_zod42.z.string(), import_zod42.z.unknown()),
|
|
612
|
-
createdAt: import_zod42.z.string()
|
|
613
|
-
})
|
|
614
|
-
),
|
|
615
|
-
policies: import_zod42.z.array(PromptPolicySchema),
|
|
616
|
-
reviews: import_zod42.z.array(
|
|
617
|
-
import_zod42.z.object({
|
|
618
|
-
version,
|
|
619
|
-
hash: import_zod42.z.string(),
|
|
620
|
-
independent: import_zod42.z.boolean(),
|
|
621
|
-
reviewer: import_zod42.z.string(),
|
|
622
|
-
note: import_zod42.z.string(),
|
|
623
|
-
createdAt: import_zod42.z.string()
|
|
624
|
-
})
|
|
625
|
-
).optional()
|
|
626
|
-
});
|
|
627
|
-
var PromptMutationSchema = import_zod42.z.discriminatedUnion("action", [
|
|
628
|
-
import_zod42.z.object({
|
|
629
|
-
action: import_zod42.z.literal("tag-set"),
|
|
630
|
-
id: import_zod42.z.uuid(),
|
|
631
|
-
tag: import_browser4.PromptTagSchema.refine(
|
|
632
|
-
(tag) => tag !== "live",
|
|
633
|
-
"Use promotion to update live."
|
|
634
|
-
),
|
|
635
|
-
version,
|
|
636
|
-
expectedRevision: revision
|
|
637
|
-
}).strict(),
|
|
638
|
-
import_zod42.z.object({
|
|
639
|
-
action: import_zod42.z.literal("tag-remove"),
|
|
640
|
-
id: import_zod42.z.uuid(),
|
|
641
|
-
tag: import_browser4.PromptTagSchema.refine(
|
|
642
|
-
(tag) => tag !== "live",
|
|
643
|
-
"The live tag cannot be removed manually."
|
|
644
|
-
),
|
|
645
|
-
expectedRevision: revision
|
|
646
|
-
}).strict(),
|
|
647
|
-
import_zod42.z.object({
|
|
648
|
-
action: import_zod42.z.literal("bulk-update"),
|
|
649
|
-
assets: import_zod42.z.array(import_zod42.z.object({ id: import_zod42.z.uuid(), expectedRevision: revision })).min(1).max(100),
|
|
650
|
-
categoryId: import_zod42.z.uuid().nullable().optional(),
|
|
651
|
-
archived: import_zod42.z.boolean().optional()
|
|
652
|
-
}).strict().refine(
|
|
653
|
-
(value) => value.categoryId !== void 0 || value.archived !== void 0,
|
|
654
|
-
"Choose a category or archive state."
|
|
655
|
-
),
|
|
656
|
-
import_zod42.z.object({
|
|
657
|
-
action: import_zod42.z.literal("create"),
|
|
658
|
-
key: import_browser4.PromptKeySchema,
|
|
659
|
-
name,
|
|
660
|
-
categoryId: import_zod42.z.uuid().nullable().default(null),
|
|
661
|
-
content: import_browser4.PromptContentSchema,
|
|
662
|
-
note: import_zod42.z.string().trim().max(2e3).default(""),
|
|
663
|
-
idempotencyKey: import_zod42.z.uuid().optional()
|
|
664
|
-
}).strict(),
|
|
665
|
-
import_zod42.z.object({
|
|
666
|
-
action: import_zod42.z.literal("draft"),
|
|
667
|
-
id: import_zod42.z.uuid(),
|
|
668
|
-
content: import_browser4.PromptContentSchema,
|
|
669
|
-
baseVersion: version,
|
|
670
|
-
expectedRevision: revision
|
|
671
|
-
}).strict(),
|
|
672
|
-
import_zod42.z.object({
|
|
673
|
-
action: import_zod42.z.literal("discard-draft"),
|
|
674
|
-
id: import_zod42.z.uuid(),
|
|
675
|
-
expectedRevision: revision
|
|
676
|
-
}).strict(),
|
|
677
|
-
import_zod42.z.object({
|
|
678
|
-
action: import_zod42.z.literal("save"),
|
|
679
|
-
id: import_zod42.z.uuid(),
|
|
680
|
-
content: import_browser4.PromptContentSchema,
|
|
681
|
-
baseVersion: version,
|
|
682
|
-
expectedHash: import_zod42.z.string().regex(/^[a-f0-9]{64}$/),
|
|
683
|
-
expectedDraftRevision: revision.optional(),
|
|
684
|
-
note: import_zod42.z.string().trim().max(2e3).default(""),
|
|
685
|
-
idempotencyKey: import_zod42.z.uuid()
|
|
686
|
-
}).strict(),
|
|
687
|
-
import_zod42.z.object({
|
|
688
|
-
action: import_zod42.z.literal("update"),
|
|
689
|
-
id: import_zod42.z.uuid(),
|
|
690
|
-
expectedRevision: revision,
|
|
691
|
-
name: name.optional(),
|
|
692
|
-
categoryId: import_zod42.z.uuid().nullable().optional(),
|
|
693
|
-
archived: import_zod42.z.boolean().optional()
|
|
694
|
-
}).strict(),
|
|
695
|
-
import_zod42.z.object({
|
|
696
|
-
action: import_zod42.z.literal("category-create"),
|
|
697
|
-
path: import_zod42.z.string().trim().min(1).max(1e3)
|
|
698
|
-
}).strict(),
|
|
699
|
-
import_zod42.z.object({
|
|
700
|
-
action: import_zod42.z.literal("category-update"),
|
|
701
|
-
id: import_zod42.z.uuid(),
|
|
702
|
-
expectedRevision: revision,
|
|
703
|
-
name: name.optional(),
|
|
704
|
-
parentId: import_zod42.z.uuid().nullable().optional()
|
|
705
|
-
}).strict(),
|
|
706
|
-
import_zod42.z.object({
|
|
707
|
-
action: import_zod42.z.literal("category-delete"),
|
|
708
|
-
id: import_zod42.z.uuid(),
|
|
709
|
-
expectedRevision: revision,
|
|
710
|
-
destinationId: import_zod42.z.uuid().nullable()
|
|
711
|
-
}).strict(),
|
|
712
|
-
import_zod42.z.object({
|
|
713
|
-
action: import_zod42.z.literal("group-create"),
|
|
714
|
-
name,
|
|
715
|
-
members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
|
|
716
|
-
}).strict(),
|
|
717
|
-
import_zod42.z.object({
|
|
718
|
-
action: import_zod42.z.literal("group-update"),
|
|
719
|
-
id: import_zod42.z.uuid(),
|
|
720
|
-
expectedRevision: revision,
|
|
721
|
-
name: name.optional(),
|
|
722
|
-
members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
|
|
723
|
-
}).strict(),
|
|
724
|
-
import_zod42.z.object({
|
|
725
|
-
action: import_zod42.z.literal("group-delete"),
|
|
726
|
-
id: import_zod42.z.uuid(),
|
|
727
|
-
expectedRevision: revision
|
|
728
|
-
}).strict(),
|
|
729
|
-
import_zod42.z.object({
|
|
730
|
-
action: import_zod42.z.literal("promote"),
|
|
731
|
-
id: import_zod42.z.uuid(),
|
|
732
|
-
version,
|
|
733
|
-
expectedRevision: revision,
|
|
734
|
-
exceptionReason: import_zod42.z.string().trim().min(1).max(2e3).optional(),
|
|
735
|
-
rollback: import_zod42.z.boolean().default(false)
|
|
736
|
-
}).strict(),
|
|
737
|
-
import_zod42.z.object({
|
|
738
|
-
action: import_zod42.z.literal("review"),
|
|
739
|
-
id: import_zod42.z.uuid(),
|
|
740
|
-
version,
|
|
741
|
-
note: import_zod42.z.string().trim().min(1).max(2e3)
|
|
742
|
-
}).strict(),
|
|
743
|
-
import_zod42.z.object({
|
|
744
|
-
action: import_zod42.z.literal("policy"),
|
|
745
|
-
id: import_zod42.z.uuid(),
|
|
746
|
-
policy: PromptPolicySchema
|
|
747
|
-
}).strict()
|
|
748
|
-
]);
|
|
749
|
-
var PromptSelectionSchema = import_zod42.z.discriminatedUnion("type", [
|
|
750
|
-
import_zod42.z.object({ type: import_zod42.z.literal("live") }).strict(),
|
|
751
|
-
import_zod42.z.object({
|
|
752
|
-
type: import_zod42.z.literal("single"),
|
|
753
|
-
id: import_zod42.z.string().min(1),
|
|
754
|
-
version: import_zod42.z.number().int().positive()
|
|
755
|
-
}).strict(),
|
|
756
|
-
import_zod42.z.object({ type: import_zod42.z.literal("group"), groupId: import_zod42.z.uuid() }).strict()
|
|
757
|
-
]);
|
|
758
|
-
var PromptBundleSchema = import_zod42.z.object({
|
|
759
|
-
schemaVersion: import_zod42.z.literal(1),
|
|
760
|
-
origin: import_zod42.z.object({
|
|
761
|
-
apiUrl: import_zod42.z.url().refine((value) => {
|
|
762
|
-
const url = new URL(value);
|
|
763
|
-
return /^https?:$/.test(url.protocol) && !url.username && !url.password && !url.search;
|
|
764
|
-
}),
|
|
765
|
-
projectId: import_zod42.z.string().min(1)
|
|
766
|
-
}).strict(),
|
|
767
|
-
prompts: import_zod42.z.array(
|
|
768
|
-
import_zod42.z.object({
|
|
769
|
-
key: import_browser4.PromptKeySchema,
|
|
770
|
-
name,
|
|
771
|
-
categoryPath: import_zod42.z.string().optional(),
|
|
772
|
-
versions: import_zod42.z.array(import_browser4.PromptVersionSchema.extend({ note: import_zod42.z.string().max(2e3) })).min(1).max(200)
|
|
773
|
-
}).strict()
|
|
774
|
-
).min(1).max(200),
|
|
775
|
-
groups: import_zod42.z.array(
|
|
776
|
-
import_zod42.z.object({
|
|
777
|
-
name,
|
|
778
|
-
members: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version })).max(100)
|
|
779
|
-
}).strict()
|
|
780
|
-
).max(100).default([])
|
|
781
|
-
}).strict();
|
|
782
|
-
var PromptTransferRequestSchema = import_zod42.z.object({
|
|
783
|
-
bundle: PromptBundleSchema,
|
|
784
|
-
conflicts: import_zod42.z.enum(["error", "append"]).default("error"),
|
|
785
|
-
rename: import_zod42.z.record(import_browser4.PromptKeySchema, import_browser4.PromptKeySchema).default({}),
|
|
786
|
-
categories: import_zod42.z.boolean().default(true),
|
|
787
|
-
groups: import_zod42.z.boolean().default(false)
|
|
788
|
-
}).strict();
|
|
789
|
-
var import_zod52 = require("zod");
|
|
790
|
-
var StudioReleaseVersionSchema = import_zod52.z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/);
|
|
791
|
-
var StudioReleaseDeploymentSchema = import_zod52.z.object({
|
|
792
|
-
strategy: import_zod52.z.enum(["rolling", "recreate"])
|
|
793
|
-
});
|
|
794
|
-
var StudioReleaseSchema = import_zod52.z.object({
|
|
795
|
-
format: import_zod52.z.literal(1),
|
|
796
|
-
installer: import_zod52.z.literal(1),
|
|
797
|
-
version: StudioReleaseVersionSchema,
|
|
798
|
-
api: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-api@sha256:[a-f0-9]{64}$/),
|
|
799
|
-
studio: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-studio@sha256:[a-f0-9]{64}$/),
|
|
800
|
-
deployment: StudioReleaseDeploymentSchema.default({
|
|
801
|
-
strategy: "recreate"
|
|
802
|
-
})
|
|
803
|
-
});
|
|
804
|
-
var StudioUpdateSettingsSchema = import_zod52.z.object({
|
|
805
|
-
automatic: import_zod52.z.boolean(),
|
|
806
|
-
hourUtc: import_zod52.z.number().int().min(0).max(23)
|
|
807
|
-
});
|
|
808
|
-
var StudioUpdateOperationSchema = import_zod52.z.object({
|
|
809
|
-
id: import_zod52.z.uuid(),
|
|
810
|
-
from: import_zod52.z.string(),
|
|
811
|
-
release: StudioReleaseSchema,
|
|
812
|
-
phase: import_zod52.z.enum([
|
|
813
|
-
"starting",
|
|
814
|
-
"pulling",
|
|
815
|
-
"backup",
|
|
816
|
-
"installing",
|
|
817
|
-
"verifying",
|
|
818
|
-
"succeeded",
|
|
819
|
-
"failed"
|
|
820
|
-
]),
|
|
821
|
-
startedAt: import_zod52.z.iso.datetime(),
|
|
822
|
-
updatedAt: import_zod52.z.iso.datetime(),
|
|
823
|
-
backup: import_zod52.z.string().nullable(),
|
|
824
|
-
messages: import_zod52.z.array(import_zod52.z.string()).max(100)
|
|
825
|
-
});
|
|
826
|
-
var StudioUpdateStatusSchema = import_zod52.z.object({
|
|
827
|
-
current: import_zod52.z.string().nullable(),
|
|
828
|
-
available: StudioReleaseSchema.nullable(),
|
|
829
|
-
checkedAt: import_zod52.z.iso.datetime().nullable(),
|
|
830
|
-
checkError: import_zod52.z.string().nullable(),
|
|
831
|
-
settings: StudioUpdateSettingsSchema,
|
|
832
|
-
operation: StudioUpdateOperationSchema.nullable()
|
|
833
|
-
});
|
|
834
|
-
function newerStudioVersion(candidate, current) {
|
|
835
|
-
const next = StudioReleaseVersionSchema.parse(candidate).split(".").map(BigInt);
|
|
836
|
-
const previous = StudioReleaseVersionSchema.parse(current).split(".").map(BigInt);
|
|
837
|
-
for (let i = 0; i < 3; i++) {
|
|
838
|
-
if (next[i] !== previous[i]) return (next[i] ?? 0n) > (previous[i] ?? 0n);
|
|
839
|
-
}
|
|
840
|
-
return false;
|
|
841
|
-
}
|
|
842
|
-
function studioUpdateRunning(operation) {
|
|
843
|
-
return !!operation && operation.phase !== "succeeded" && operation.phase !== "failed";
|
|
844
|
-
}
|
|
845
|
-
var str = (v, fallback = "") => typeof v === "string" ? v : fallback;
|
|
846
|
-
function projectWorkflowCalls(events) {
|
|
847
|
-
const groups = /* @__PURE__ */ new Map();
|
|
848
|
-
for (const event of events) {
|
|
849
|
-
if (!event.type.startsWith("workflow.call.") || !str(event.payload.invocationId) || !str(event.payload.targetWorkflowId))
|
|
850
|
-
continue;
|
|
851
|
-
const key = `${event.runId}:${str(event.payload.branchId, event.runId)}:${event.payload.invocationId}`;
|
|
852
|
-
const group = groups.get(key) ?? [];
|
|
853
|
-
if (!group.some((existing) => existing.id === event.id)) group.push(event);
|
|
854
|
-
groups.set(key, group);
|
|
855
|
-
}
|
|
856
|
-
return [...groups.entries()].map(([id3, group]) => {
|
|
857
|
-
group.sort(
|
|
858
|
-
(a, b) => Number(a.payload.sequence ?? 0) - Number(b.payload.sequence ?? 0) || a.occurredAt.localeCompare(b.occurredAt) || a.id.localeCompare(b.id)
|
|
859
|
-
);
|
|
860
|
-
const first = group[0];
|
|
861
|
-
const p = first.payload;
|
|
862
|
-
const call = {
|
|
863
|
-
id: id3,
|
|
864
|
-
invocationId: str(p.invocationId),
|
|
865
|
-
branchId: str(p.branchId, first.runId),
|
|
866
|
-
parentInvocationId: str(p.parentInvocationId) || null,
|
|
867
|
-
callId: str(p.callId),
|
|
868
|
-
callerNodeId: str(p.callerNodeId),
|
|
869
|
-
callerNodeExecutionId: str(p.callerNodeExecutionId),
|
|
870
|
-
sourceWorkflowId: str(p.sourceWorkflowId, first.workflowId),
|
|
871
|
-
targetWorkflowId: str(p.targetWorkflowId),
|
|
872
|
-
targetVersion: str(p.targetVersion) || null,
|
|
873
|
-
status: "incomplete",
|
|
874
|
-
startedAt: first.occurredAt,
|
|
875
|
-
endedAt: null,
|
|
876
|
-
activeMs: 0,
|
|
877
|
-
waitMs: 0,
|
|
878
|
-
attempts: 0,
|
|
879
|
-
reused: 0,
|
|
880
|
-
inherited: false,
|
|
881
|
-
sourceRunId: null,
|
|
882
|
-
leaf: null,
|
|
883
|
-
events: group
|
|
884
|
-
};
|
|
885
|
-
let active = null;
|
|
886
|
-
let waiting = null;
|
|
887
|
-
for (const event of group) {
|
|
888
|
-
const at = Date.parse(event.occurredAt);
|
|
889
|
-
const status2 = event.type.slice("workflow.call.".length);
|
|
890
|
-
for (const side of [
|
|
891
|
-
"input",
|
|
892
|
-
"output",
|
|
893
|
-
"inputOmitted",
|
|
894
|
-
"outputOmitted"
|
|
895
|
-
])
|
|
896
|
-
if (side in event.payload) call[side] = event.payload[side];
|
|
897
|
-
if (status2 === "restored") {
|
|
898
|
-
call.inherited = true;
|
|
899
|
-
call.sourceRunId = str(event.payload.sourceRunId) || null;
|
|
900
|
-
call.status = event.payload.status === "completed" ? "completed" : event.payload.status === "failed" ? "failed" : "interrupted";
|
|
901
|
-
if (call.status === "interrupted") waiting = at;
|
|
902
|
-
} else if (status2 === "started" || status2 === "resumed") {
|
|
903
|
-
if (waiting !== null) call.waitMs += Math.max(0, at - waiting);
|
|
904
|
-
waiting = null;
|
|
905
|
-
active = at;
|
|
906
|
-
call.attempts++;
|
|
907
|
-
call.status = "running";
|
|
908
|
-
call.endedAt = null;
|
|
909
|
-
} else if (status2 === "reused") {
|
|
910
|
-
call.reused++;
|
|
911
|
-
call.status = "completed";
|
|
912
|
-
} else if (status2 === "suspended" || status2 === "completed" || status2 === "failed") {
|
|
913
|
-
if (active !== null) call.activeMs += Math.max(0, at - active);
|
|
914
|
-
active = null;
|
|
915
|
-
call.status = status2 === "suspended" ? "interrupted" : status2;
|
|
916
|
-
if (status2 === "suspended") {
|
|
917
|
-
waiting = at;
|
|
918
|
-
const leaf = event.payload.leaf;
|
|
919
|
-
if (leaf && typeof leaf === "object") {
|
|
920
|
-
const value = leaf;
|
|
921
|
-
call.leaf = {
|
|
922
|
-
workflowId: str(value.workflowId),
|
|
923
|
-
nodeId: str(value.nodeId),
|
|
924
|
-
invocationId: str(value.invocationId)
|
|
925
|
-
};
|
|
926
|
-
}
|
|
927
|
-
} else {
|
|
928
|
-
call.endedAt = event.occurredAt;
|
|
929
|
-
call.leaf = null;
|
|
930
|
-
}
|
|
931
|
-
}
|
|
932
|
-
}
|
|
933
|
-
const cancellation = events.find(
|
|
934
|
-
(event) => event.type === "run.cancelled" && event.runId === first.runId && (event.payload.branchId ?? event.runId) === call.branchId && event.occurredAt >= (group.at(-1)?.occurredAt ?? first.occurredAt)
|
|
935
|
-
);
|
|
936
|
-
if (cancellation && (call.status === "interrupted" || call.status === "running")) {
|
|
937
|
-
call.status = "cancelled";
|
|
938
|
-
call.endedAt = cancellation.occurredAt;
|
|
939
|
-
}
|
|
940
|
-
return call;
|
|
941
|
-
}).sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
|
942
|
-
}
|
|
943
|
-
var STUDIO_API_PROTOCOL_VERSION = "1";
|
|
944
|
-
var TELEMETRY_EVENT_TYPES = [
|
|
945
|
-
"span.started",
|
|
946
|
-
"span.ended",
|
|
947
|
-
"span.failed",
|
|
948
|
-
"error.reported",
|
|
949
|
-
"generation.completed",
|
|
950
|
-
"tool.started",
|
|
951
|
-
"tool.completed",
|
|
952
|
-
"tool.failed",
|
|
953
|
-
"tool.denied",
|
|
954
|
-
"tool.cancelled",
|
|
955
|
-
"tool.reused",
|
|
956
|
-
"tool.waiting",
|
|
957
|
-
"tool.suspended",
|
|
958
|
-
"response.completed",
|
|
959
|
-
"output.emitted",
|
|
960
|
-
"output.invalidated",
|
|
961
|
-
"interrupt.created",
|
|
962
|
-
"interrupt.resolved",
|
|
963
|
-
"interrupt.expired",
|
|
964
|
-
"interrupt.cancelled",
|
|
965
|
-
"run.cancelled",
|
|
966
|
-
"run.limit_reached",
|
|
967
|
-
"workflow.suspended",
|
|
968
|
-
"workflow.resumed",
|
|
969
|
-
"workflow.transitioned",
|
|
970
|
-
"workflow.call.started",
|
|
971
|
-
"workflow.call.suspended",
|
|
972
|
-
"workflow.call.resumed",
|
|
973
|
-
"workflow.call.completed",
|
|
974
|
-
"workflow.call.failed",
|
|
975
|
-
"workflow.call.reused",
|
|
976
|
-
"workflow.call.restored",
|
|
977
|
-
"session.checkpointed",
|
|
978
|
-
"session.forked",
|
|
979
|
-
"session.rolled_back"
|
|
980
|
-
];
|
|
981
|
-
var TelemetryServiceSchema = import_zod62.z.object({
|
|
982
|
-
name: import_zod62.z.string().min(1),
|
|
983
|
-
deploymentRef: import_zod62.z.string().min(1).optional()
|
|
984
|
-
}).strict();
|
|
985
|
-
var WorkflowTopologyNodeSchema = import_zod62.z.object({
|
|
986
|
-
id: import_zod62.z.string().min(1),
|
|
987
|
-
label: import_zod62.z.string().optional(),
|
|
988
|
-
type: import_zod62.z.string().optional(),
|
|
989
|
-
provider: import_zod62.z.string().optional(),
|
|
990
|
-
model: import_zod62.z.string().optional(),
|
|
991
|
-
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional(),
|
|
992
|
-
tools: import_zod62.z.array(WorkflowToolSchema).optional(),
|
|
993
|
-
toolDiscovery: ToolDiscoverySchema.optional()
|
|
994
|
-
}).strict();
|
|
995
|
-
var WorkflowTopologyEdgeSchema = import_zod62.z.object({
|
|
996
|
-
sourceNodeId: import_zod62.z.string().min(1),
|
|
997
|
-
targetNodeId: import_zod62.z.string().min(1),
|
|
998
|
-
condition: import_zod62.z.string().optional()
|
|
999
|
-
}).strict();
|
|
1000
|
-
var WorkflowTopologyCallSchema = import_zod62.z.object({
|
|
1001
|
-
sourceNodeId: import_zod62.z.string().min(1),
|
|
1002
|
-
targetWorkflowId: import_zod62.z.string().min(1)
|
|
1003
|
-
}).strict();
|
|
1004
|
-
var WorkflowTopologyTransitionSchema = import_zod62.z.object({
|
|
1005
|
-
sourceNodeId: import_zod62.z.string().min(1).optional(),
|
|
1006
|
-
targetWorkflowId: import_zod62.z.string().min(1),
|
|
1007
|
-
condition: import_zod62.z.string().optional(),
|
|
1008
|
-
intent: import_zod62.z.string().optional(),
|
|
1009
|
-
kind: import_zod62.z.enum(["call", "handoff"]).optional()
|
|
1010
|
-
}).strict();
|
|
1011
|
-
var EnsureWorkflowTopologyRequestSchema = import_zod62.z.object({
|
|
1012
|
-
schemaVersion: import_zod62.z.literal(1),
|
|
1013
|
-
environment: import_zod62.z.string().min(1),
|
|
1014
|
-
service: TelemetryServiceSchema,
|
|
1015
|
-
workflow: import_zod62.z.object({
|
|
1016
|
-
id: import_zod62.z.string().min(1),
|
|
1017
|
-
declaredVersion: import_zod62.z.string().min(1),
|
|
1018
|
-
description: import_zod62.z.string().optional(),
|
|
1019
|
-
tags: import_zod62.z.array(import_zod62.z.string()).optional(),
|
|
1020
|
-
topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/),
|
|
1021
|
-
nodes: import_zod62.z.array(WorkflowTopologyNodeSchema),
|
|
1022
|
-
edges: import_zod62.z.array(WorkflowTopologyEdgeSchema),
|
|
1023
|
-
transitions: import_zod62.z.array(WorkflowTopologyTransitionSchema).optional(),
|
|
1024
|
-
calls: import_zod62.z.array(WorkflowTopologyCallSchema).optional()
|
|
1025
|
-
}).strict()
|
|
1026
|
-
}).strict();
|
|
1027
|
-
var EnsureWorkflowTopologyResponseSchema = import_zod62.z.object({ workflowRevisionId: import_zod62.z.string().min(1), created: import_zod62.z.boolean() }).strict();
|
|
1028
|
-
var TelemetryEventTypeSchema = import_zod62.z.enum(TELEMETRY_EVENT_TYPES);
|
|
1029
|
-
var TelemetryEventSchema = import_zod62.z.object({
|
|
1030
|
-
schemaVersion: import_zod62.z.literal(1),
|
|
1031
|
-
eventId: import_zod62.z.string().min(1),
|
|
1032
|
-
occurredAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1033
|
-
environment: import_zod62.z.string().min(1),
|
|
1034
|
-
service: TelemetryServiceSchema,
|
|
1035
|
-
correlation: import_zod62.z.object({
|
|
1036
|
-
invocationId: import_zod62.z.string().optional(),
|
|
1037
|
-
parentInvocationId: import_zod62.z.string().optional(),
|
|
1038
|
-
branchId: import_zod62.z.string().optional(),
|
|
1039
|
-
traceId: import_zod62.z.string().optional(),
|
|
1040
|
-
spanId: import_zod62.z.string().optional(),
|
|
1041
|
-
parentSpanId: import_zod62.z.string().optional(),
|
|
1042
|
-
runId: import_zod62.z.string().min(1),
|
|
1043
|
-
sessionId: import_zod62.z.string().optional(),
|
|
1044
|
-
workflowId: import_zod62.z.string().min(1),
|
|
1045
|
-
workflowRevisionId: import_zod62.z.string().optional(),
|
|
1046
|
-
topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
1047
|
-
nodeId: import_zod62.z.string().optional()
|
|
1048
|
-
}).strict(),
|
|
1049
|
-
context: import_zod62.z.object({
|
|
1050
|
-
userId: import_zod62.z.string().optional(),
|
|
1051
|
-
tenantId: import_zod62.z.string().optional(),
|
|
1052
|
-
tags: import_zod62.z.array(import_zod62.z.string()).optional(),
|
|
1053
|
-
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
|
|
1054
|
-
}).strict().optional(),
|
|
1055
|
-
type: TelemetryEventTypeSchema,
|
|
1056
|
-
payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
|
|
1057
|
-
}).strict().superRefine((event, ctx) => {
|
|
1058
|
-
if (!event.type.startsWith("tool.")) return;
|
|
1059
|
-
const canonical3 = event.payload.version === 1 || [
|
|
1060
|
-
"tool.denied",
|
|
1061
|
-
"tool.cancelled",
|
|
1062
|
-
"tool.reused",
|
|
1063
|
-
"tool.waiting",
|
|
1064
|
-
"tool.suspended"
|
|
1065
|
-
].includes(event.type);
|
|
1066
|
-
if (!canonical3) return;
|
|
1067
|
-
const parsed = ToolObservationSchema.strict().safeParse(event.payload);
|
|
1068
|
-
if (!parsed.success)
|
|
1069
|
-
for (const issue2 of parsed.error.issues)
|
|
1070
|
-
ctx.addIssue({ ...issue2, path: ["payload", ...issue2.path] });
|
|
1071
|
-
else {
|
|
1072
|
-
if ((parsed.data.errorType !== void 0 || parsed.data.errorMessage !== void 0) && parsed.data.outcome !== "fault")
|
|
1073
|
-
ctx.addIssue({
|
|
1074
|
-
code: "custom",
|
|
1075
|
-
path: ["payload"],
|
|
1076
|
-
message: "Error diagnostics require a fault outcome."
|
|
1077
|
-
});
|
|
1078
|
-
const terminalOutcomes = {
|
|
1079
|
-
"tool.completed": "success",
|
|
1080
|
-
"tool.failed": "fault",
|
|
1081
|
-
"tool.denied": "denied",
|
|
1082
|
-
"tool.cancelled": "cancelled"
|
|
1083
|
-
};
|
|
1084
|
-
const expected = terminalOutcomes[event.type];
|
|
1085
|
-
if (expected && parsed.data.outcome !== expected)
|
|
1086
|
-
ctx.addIssue({
|
|
1087
|
-
code: "custom",
|
|
1088
|
-
path: ["payload", "outcome"],
|
|
1089
|
-
message: "Outcome must match the tool event type."
|
|
1090
|
-
});
|
|
1091
|
-
if (event.type === "tool.reused" && (parsed.data.executed || !parsed.data.source))
|
|
1092
|
-
ctx.addIssue({
|
|
1093
|
-
code: "custom",
|
|
1094
|
-
path: ["payload"],
|
|
1095
|
-
message: "Reuse must be unexecuted and reference its source."
|
|
1096
|
-
});
|
|
1097
|
-
}
|
|
1098
|
-
});
|
|
1099
|
-
var TelemetryEventBatchSchema = import_zod62.z.object({ events: import_zod62.z.array(TelemetryEventSchema).min(1) }).strict();
|
|
1100
|
-
var TelemetryEventBatchResponseSchema = import_zod62.z.object({
|
|
1101
|
-
accepted: import_zod62.z.number().int().nonnegative(),
|
|
1102
|
-
inserted: import_zod62.z.number().int().nonnegative(),
|
|
1103
|
-
duplicates: import_zod62.z.number().int().nonnegative()
|
|
1104
|
-
}).strict();
|
|
1105
|
-
var TelemetryPricingSourceSchema = import_zod62.z.enum([
|
|
1106
|
-
"sdk",
|
|
1107
|
-
"provider",
|
|
1108
|
-
"custom",
|
|
1109
|
-
"project-rate-card",
|
|
1110
|
-
"default-rate-card"
|
|
1111
|
-
]);
|
|
1112
|
-
var TelemetryPricingUnitSchema = import_zod62.z.enum([
|
|
1113
|
-
"token",
|
|
1114
|
-
"character",
|
|
1115
|
-
"request",
|
|
1116
|
-
"image",
|
|
1117
|
-
"audio_second",
|
|
1118
|
-
"audio_minute",
|
|
1119
|
-
"video_second",
|
|
1120
|
-
"video_minute",
|
|
1121
|
-
"second",
|
|
1122
|
-
"minute",
|
|
1123
|
-
"custom"
|
|
1124
|
-
]);
|
|
1125
|
-
var TelemetryPricingUsageTypeSchema = import_zod62.z.enum([
|
|
1126
|
-
"input",
|
|
1127
|
-
"output",
|
|
1128
|
-
"reasoning",
|
|
1129
|
-
"cache_read",
|
|
1130
|
-
"cache_write",
|
|
1131
|
-
"embedding",
|
|
1132
|
-
"image_input",
|
|
1133
|
-
"image_output",
|
|
1134
|
-
"audio_input",
|
|
1135
|
-
"audio_output",
|
|
1136
|
-
"video_input",
|
|
1137
|
-
"video_output",
|
|
1138
|
-
"request",
|
|
1139
|
-
"custom"
|
|
1140
|
-
]);
|
|
1141
|
-
var TelemetryUsageItemSchema = import_zod62.z.object({
|
|
1142
|
-
usageType: TelemetryPricingUsageTypeSchema,
|
|
1143
|
-
quantity: import_zod62.z.number().nonnegative(),
|
|
1144
|
-
unit: TelemetryPricingUnitSchema,
|
|
1145
|
-
label: import_zod62.z.string().optional()
|
|
1146
|
-
}).strict();
|
|
1147
|
-
var TelemetryUnitPriceSchema = import_zod62.z.object({
|
|
1148
|
-
usageType: TelemetryPricingUsageTypeSchema,
|
|
1149
|
-
unit: TelemetryPricingUnitSchema,
|
|
1150
|
-
unitQuantity: import_zod62.z.number().positive().default(1),
|
|
1151
|
-
priceMicros: import_zod62.z.number().int().nonnegative(),
|
|
1152
|
-
label: import_zod62.z.string().optional()
|
|
1153
|
-
}).strict();
|
|
1154
|
-
var TelemetryPricingLineItemSchema = import_zod62.z.object({
|
|
1155
|
-
usageType: TelemetryPricingUsageTypeSchema,
|
|
1156
|
-
quantity: import_zod62.z.number().nonnegative(),
|
|
1157
|
-
unit: TelemetryPricingUnitSchema,
|
|
1158
|
-
unitQuantity: import_zod62.z.number().positive().default(1),
|
|
1159
|
-
unitPriceMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1160
|
-
totalCostMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1161
|
-
label: import_zod62.z.string().optional()
|
|
1162
|
-
}).strict().refine(
|
|
1163
|
-
(value) => value.totalCostMicros !== void 0 || value.unitPriceMicros !== void 0,
|
|
1164
|
-
"line item requires totalCostMicros or unitPriceMicros"
|
|
1165
|
-
);
|
|
1166
|
-
var TelemetryPricingHintSchema = import_zod62.z.object({
|
|
1167
|
-
source: TelemetryPricingSourceSchema.extract(["sdk", "provider", "custom"]),
|
|
1168
|
-
currency: import_zod62.z.string().min(3).max(12).default("USD"),
|
|
1169
|
-
actualCostMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1170
|
-
lineItems: import_zod62.z.array(TelemetryPricingLineItemSchema).optional(),
|
|
1171
|
-
unitPrices: import_zod62.z.array(TelemetryUnitPriceSchema).optional(),
|
|
1172
|
-
usageItems: import_zod62.z.array(TelemetryUsageItemSchema).optional(),
|
|
1173
|
-
pricingRef: import_zod62.z.string().min(1).optional(),
|
|
1174
|
-
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
|
|
1175
|
-
}).strict();
|
|
1176
|
-
var StudioRunStatusSchema = import_zod62.z.enum([
|
|
1177
|
-
"running",
|
|
1178
|
-
"completed",
|
|
1179
|
-
"interrupted",
|
|
1180
|
-
"incomplete",
|
|
1181
|
-
"failed",
|
|
1182
|
-
"cancelled"
|
|
1183
|
-
]);
|
|
1184
|
-
var STUDIO_TIME_RANGES = [
|
|
1185
|
-
"Last hour",
|
|
1186
|
-
"24 hours",
|
|
1187
|
-
"7 days",
|
|
1188
|
-
"30 days",
|
|
1189
|
-
"All time",
|
|
1190
|
-
"Custom range"
|
|
1191
|
-
];
|
|
1192
|
-
var StudioTimeRangeSchema = import_zod62.z.enum(STUDIO_TIME_RANGES);
|
|
1193
|
-
var StudioTimeRangeQuerySchema = import_zod62.z.object({
|
|
1194
|
-
range: StudioTimeRangeSchema.optional(),
|
|
1195
|
-
startedAfter: import_zod62.z.string().optional(),
|
|
1196
|
-
startedBefore: import_zod62.z.string().optional()
|
|
1197
|
-
}).strict();
|
|
1198
|
-
var StudioTimeRangeContextSchema = import_zod62.z.object({
|
|
1199
|
-
range: StudioTimeRangeSchema,
|
|
1200
|
-
startedAfter: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1201
|
-
startedBefore: import_zod62.z.string().datetime({ offset: true }).nullable()
|
|
1202
|
-
}).strict();
|
|
1203
|
-
var STUDIO_TIME_RANGE_MILLISECONDS = {
|
|
1204
|
-
"Last hour": 60 * 60 * 1e3,
|
|
1205
|
-
"24 hours": 24 * 60 * 60 * 1e3,
|
|
1206
|
-
"7 days": 7 * 24 * 60 * 60 * 1e3,
|
|
1207
|
-
"30 days": 30 * 24 * 60 * 60 * 1e3
|
|
1208
|
-
};
|
|
1209
|
-
var parseIsoBoundary = (value, label) => {
|
|
1210
|
-
if (!value) {
|
|
1211
|
-
return { ok: false, error: `${label} is required for a custom range.` };
|
|
1212
|
-
}
|
|
1213
|
-
if (!/(?:Z|[+-]\d{2}:\d{2})$/.test(value)) {
|
|
1214
|
-
return {
|
|
1215
|
-
ok: false,
|
|
1216
|
-
error: `${label} must include an explicit UTC offset.`
|
|
1217
|
-
};
|
|
1218
|
-
}
|
|
1219
|
-
const date = new Date(value);
|
|
1220
|
-
return Number.isFinite(date.getTime()) ? { ok: true, date } : { ok: false, error: `${label} must be a valid ISO date and time.` };
|
|
1221
|
-
};
|
|
1222
|
-
var resolveStudioTimeRange = (input, now = /* @__PURE__ */ new Date()) => {
|
|
1223
|
-
const parsedRange = StudioTimeRangeSchema.safeParse(
|
|
1224
|
-
input.range || "24 hours"
|
|
1225
|
-
);
|
|
1226
|
-
if (!parsedRange.success) {
|
|
1227
|
-
return {
|
|
1228
|
-
ok: false,
|
|
1229
|
-
error: `Unknown time range "${input.range}".`
|
|
1230
|
-
};
|
|
1231
|
-
}
|
|
1232
|
-
if (!Number.isFinite(now.getTime())) {
|
|
1233
|
-
return { ok: false, error: "The time range reference date is invalid." };
|
|
1234
|
-
}
|
|
1235
|
-
const range = parsedRange.data;
|
|
1236
|
-
if (range === "All time") {
|
|
1237
|
-
return {
|
|
1238
|
-
ok: true,
|
|
1239
|
-
value: { range, startedAfter: null, startedBefore: null }
|
|
1240
|
-
};
|
|
1241
|
-
}
|
|
1242
|
-
if (range === "Custom range") {
|
|
1243
|
-
const after = parseIsoBoundary(input.startedAfter, "Start time");
|
|
1244
|
-
if ("error" in after) return { ok: false, error: after.error };
|
|
1245
|
-
const before = parseIsoBoundary(input.startedBefore, "End time");
|
|
1246
|
-
if ("error" in before) return { ok: false, error: before.error };
|
|
1247
|
-
if (after.date.getTime() >= before.date.getTime()) {
|
|
1248
|
-
return {
|
|
1249
|
-
ok: false,
|
|
1250
|
-
error: "Start time must be before end time."
|
|
1251
|
-
};
|
|
1252
|
-
}
|
|
1253
|
-
return {
|
|
1254
|
-
ok: true,
|
|
1255
|
-
value: {
|
|
1256
|
-
range,
|
|
1257
|
-
startedAfter: after.date.toISOString(),
|
|
1258
|
-
startedBefore: before.date.toISOString()
|
|
1259
|
-
}
|
|
1260
|
-
};
|
|
1261
|
-
}
|
|
1262
|
-
const duration = STUDIO_TIME_RANGE_MILLISECONDS[range];
|
|
1263
|
-
if (!duration) {
|
|
1264
|
-
return { ok: false, error: `Unknown time range "${range}".` };
|
|
1265
|
-
}
|
|
1266
|
-
return {
|
|
1267
|
-
ok: true,
|
|
1268
|
-
value: {
|
|
1269
|
-
range,
|
|
1270
|
-
startedAfter: new Date(now.getTime() - duration).toISOString(),
|
|
1271
|
-
startedBefore: now.toISOString()
|
|
1272
|
-
}
|
|
1273
|
-
};
|
|
1274
|
-
};
|
|
1275
|
-
var StudioInterruptStatusSchema = import_zod62.z.enum([
|
|
1276
|
-
"pending",
|
|
1277
|
-
"resolved",
|
|
1278
|
-
"expired",
|
|
1279
|
-
"failed",
|
|
1280
|
-
"cancelled"
|
|
1281
|
-
]);
|
|
1282
|
-
var resolveStudioInterruptStatus = (input, now = Date.now()) => {
|
|
1283
|
-
if (input.status !== "pending" || !input.expiresAt) return input.status;
|
|
1284
|
-
const expiresAt = Date.parse(input.expiresAt);
|
|
1285
|
-
const nowMs = typeof now === "number" ? now : now.getTime();
|
|
1286
|
-
return Number.isFinite(expiresAt) && Number.isFinite(nowMs) && expiresAt <= nowMs ? "expired" : input.status;
|
|
1287
|
-
};
|
|
1288
|
-
var StudioInterruptTypeSchema = import_zod62.z.enum([
|
|
1289
|
-
"choice",
|
|
1290
|
-
"multi-choice",
|
|
1291
|
-
"text",
|
|
1292
|
-
"structured",
|
|
1293
|
-
"unknown"
|
|
1294
|
-
]);
|
|
1295
|
-
var StudioInterruptInteractionModeSchema = import_zod62.z.enum([
|
|
1296
|
-
"static-options",
|
|
1297
|
-
"dynamic-picker",
|
|
1298
|
-
"freeform",
|
|
1299
|
-
"unknown"
|
|
1300
|
-
]);
|
|
1301
|
-
var StudioInterruptOptionSchema = import_zod62.z.object({
|
|
1302
|
-
id: import_zod62.z.string().min(1),
|
|
1303
|
-
label: import_zod62.z.string().min(1),
|
|
1304
|
-
description: import_zod62.z.string().nullable()
|
|
1305
|
-
}).strip();
|
|
1306
|
-
var StudioInterruptValueSchema = import_zod62.z.union([
|
|
1307
|
-
import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()),
|
|
1308
|
-
import_zod62.z.array(import_zod62.z.unknown()),
|
|
1309
|
-
import_zod62.z.string(),
|
|
1310
|
-
import_zod62.z.number(),
|
|
1311
|
-
import_zod62.z.boolean()
|
|
1312
|
-
]);
|
|
1313
|
-
var StudioResumeOutcomeSchema = import_zod62.z.enum([
|
|
1314
|
-
"resumed",
|
|
1315
|
-
"resume failed",
|
|
1316
|
-
"expired before resume",
|
|
1317
|
-
"cancelled"
|
|
1318
|
-
]);
|
|
1319
|
-
var StudioWorkflowHealthSchema = import_zod62.z.enum([
|
|
1320
|
-
"unknown",
|
|
1321
|
-
"healthy",
|
|
1322
|
-
"degraded",
|
|
1323
|
-
"failing",
|
|
1324
|
-
"idle"
|
|
1325
|
-
]);
|
|
1326
|
-
var StudioPricingStatusSchema = import_zod62.z.enum([
|
|
1327
|
-
"priced",
|
|
1328
|
-
"unpriced",
|
|
1329
|
-
"unknown"
|
|
1330
|
-
]);
|
|
1331
|
-
var StudioPricingSourceSchema = TelemetryPricingSourceSchema.nullable();
|
|
1332
|
-
var StudioChangeResourceSchema = import_zod62.z.enum([
|
|
1333
|
-
"evals",
|
|
1334
|
-
"workflows",
|
|
1335
|
-
"runs",
|
|
1336
|
-
"sessions",
|
|
1337
|
-
"interrupts"
|
|
1338
|
-
]);
|
|
1339
|
-
var StudioChangeSchema = import_zod62.z.object({
|
|
1340
|
-
schemaVersion: import_zod62.z.literal(1),
|
|
1341
|
-
changeId: import_zod62.z.string().min(1),
|
|
1342
|
-
emittedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1343
|
-
organizationId: import_zod62.z.string().min(1),
|
|
1344
|
-
projectId: import_zod62.z.string().min(1),
|
|
1345
|
-
environment: import_zod62.z.string().min(1).optional(),
|
|
1346
|
-
resources: import_zod62.z.array(StudioChangeResourceSchema).min(1)
|
|
1347
|
-
}).strict();
|
|
1348
|
-
var StudioCostSummarySchema = import_zod62.z.object({
|
|
1349
|
-
amounts: import_zod62.z.array(
|
|
1350
|
-
import_zod62.z.object({
|
|
1351
|
-
amount: import_zod62.z.number().nonnegative(),
|
|
1352
|
-
currency: import_zod62.z.string().nullable()
|
|
1353
|
-
}).strip()
|
|
1354
|
-
),
|
|
1355
|
-
complete: import_zod62.z.boolean()
|
|
1356
|
-
}).strip();
|
|
1357
|
-
var StudioMetricSchema = import_zod62.z.object({
|
|
1358
|
-
runCount: import_zod62.z.number().int().nonnegative(),
|
|
1359
|
-
successRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1360
|
-
errorRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1361
|
-
retryCount: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1362
|
-
interruptRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1363
|
-
p50DurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1364
|
-
p95DurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1365
|
-
averageTokens: import_zod62.z.number().nonnegative().nullable(),
|
|
1366
|
-
averageCost: import_zod62.z.number().nonnegative().nullable(),
|
|
1367
|
-
averageCostSummary: StudioCostSummarySchema.optional(),
|
|
1368
|
-
currency: import_zod62.z.string().nullable()
|
|
1369
|
-
}).strip();
|
|
1370
|
-
var StudioRunSchema = import_zod62.z.object({
|
|
1371
|
-
id: import_zod62.z.string().min(1),
|
|
1372
|
-
feedback: FeedbackSummarySchema.optional(),
|
|
1373
|
-
callerNodeId: import_zod62.z.string().optional(),
|
|
1374
|
-
parentRunId: import_zod62.z.string().optional(),
|
|
1375
|
-
parentWorkflowId: import_zod62.z.string().optional(),
|
|
1376
|
-
invocationId: import_zod62.z.string().optional(),
|
|
1377
|
-
branchId: import_zod62.z.string().optional(),
|
|
1378
|
-
callId: import_zod62.z.string().optional(),
|
|
1379
|
-
status: StudioRunStatusSchema,
|
|
1380
|
-
startedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1381
|
-
endedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1382
|
-
workflowId: import_zod62.z.string().min(1),
|
|
1383
|
-
workflowIds: import_zod62.z.array(import_zod62.z.string().min(1)),
|
|
1384
|
-
workflowRefs: import_zod62.z.array(
|
|
1385
|
-
import_zod62.z.object({
|
|
1386
|
-
workflowId: import_zod62.z.string().min(1),
|
|
1387
|
-
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1388
|
-
declaredVersion: import_zod62.z.string().nullable()
|
|
1389
|
-
}).strip()
|
|
1390
|
-
),
|
|
1391
|
-
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1392
|
-
declaredVersion: import_zod62.z.string().nullable(),
|
|
1393
|
-
transitionIds: import_zod62.z.array(import_zod62.z.string().min(1)),
|
|
1394
|
-
path: import_zod62.z.array(import_zod62.z.string()),
|
|
1395
|
-
sessionId: import_zod62.z.string().nullable(),
|
|
1396
|
-
provider: import_zod62.z.string().nullable(),
|
|
1397
|
-
model: import_zod62.z.string().nullable(),
|
|
1398
|
-
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1399
|
-
durationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1400
|
-
tokens: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1401
|
-
cost: import_zod62.z.number().nonnegative().nullable(),
|
|
1402
|
-
costSummary: StudioCostSummarySchema.optional(),
|
|
1403
|
-
pricingStatus: StudioPricingStatusSchema,
|
|
1404
|
-
pricingSource: StudioPricingSourceSchema,
|
|
1405
|
-
currency: import_zod62.z.string().nullable(),
|
|
1406
|
-
result: import_zod62.z.string().nullable(),
|
|
1407
|
-
environment: import_zod62.z.string().min(1),
|
|
1408
|
-
userId: import_zod62.z.string().nullable(),
|
|
1409
|
-
tenantId: import_zod62.z.string().nullable(),
|
|
1410
|
-
hasTool: import_zod62.z.boolean(),
|
|
1411
|
-
hasRetry: import_zod62.z.boolean(),
|
|
1412
|
-
interruptNodeId: import_zod62.z.string().nullable(),
|
|
1413
|
-
interruptId: import_zod62.z.string().nullable().optional(),
|
|
1414
|
-
interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
|
|
1415
|
-
interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional()
|
|
1416
|
-
}).strip();
|
|
1417
|
-
var StudioSessionSchema = import_zod62.z.object({
|
|
1418
|
-
id: import_zod62.z.string().min(1),
|
|
1419
|
-
status: StudioRunStatusSchema,
|
|
1420
|
-
lastActivityAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1421
|
-
workflowIds: import_zod62.z.array(import_zod62.z.string()),
|
|
1422
|
-
workflowCount: import_zod62.z.number().int().nonnegative(),
|
|
1423
|
-
activeWorkflowId: import_zod62.z.string().nullable(),
|
|
1424
|
-
activeVersion: import_zod62.z.string().nullable(),
|
|
1425
|
-
userId: import_zod62.z.string().nullable(),
|
|
1426
|
-
tenantId: import_zod62.z.string().nullable(),
|
|
1427
|
-
runs: import_zod62.z.number().int().nonnegative(),
|
|
1428
|
-
succeeded: import_zod62.z.number().int().nonnegative(),
|
|
1429
|
-
failed: import_zod62.z.number().int().nonnegative(),
|
|
1430
|
-
interrupted: import_zod62.z.number().int().nonnegative(),
|
|
1431
|
-
checkpoints: import_zod62.z.number().int().nonnegative(),
|
|
1432
|
-
hasFork: import_zod62.z.boolean(),
|
|
1433
|
-
durationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1434
|
-
tokens: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1435
|
-
cost: import_zod62.z.number().nonnegative().nullable(),
|
|
1436
|
-
costSummary: StudioCostSummarySchema.optional(),
|
|
1437
|
-
pricingStatus: StudioPricingStatusSchema,
|
|
1438
|
-
pricingSource: StudioPricingSourceSchema,
|
|
1439
|
-
currency: import_zod62.z.string().nullable(),
|
|
1440
|
-
latestResult: import_zod62.z.string().nullable(),
|
|
1441
|
-
latestError: import_zod62.z.string().nullable(),
|
|
1442
|
-
pendingInterruptId: import_zod62.z.string().nullable(),
|
|
1443
|
-
interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
|
|
1444
|
-
interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional(),
|
|
1445
|
-
providers: import_zod62.z.array(import_zod62.z.string()),
|
|
1446
|
-
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1447
|
-
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1448
|
-
environment: import_zod62.z.string().min(1)
|
|
1449
|
-
}).strip();
|
|
1450
|
-
var StudioInterruptSchema = import_zod62.z.object({
|
|
1451
|
-
afterResponseCompleted: import_zod62.z.boolean().optional(),
|
|
1452
|
-
id: import_zod62.z.string().min(1),
|
|
1453
|
-
status: StudioInterruptStatusSchema,
|
|
1454
|
-
type: StudioInterruptTypeSchema,
|
|
1455
|
-
interactionMode: StudioInterruptInteractionModeSchema,
|
|
1456
|
-
contract: import_zod62.z.string().nullable().default(null),
|
|
1457
|
-
schemaId: import_zod62.z.string().nullable(),
|
|
1458
|
-
schemaVersion: import_zod62.z.string().nullable(),
|
|
1459
|
-
createdAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1460
|
-
resolvedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1461
|
-
expiresAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1462
|
-
question: import_zod62.z.string().nullable(),
|
|
1463
|
-
contentCaptured: import_zod62.z.boolean(),
|
|
1464
|
-
request: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable().default(null),
|
|
1465
|
-
requestCaptured: import_zod62.z.boolean().default(false),
|
|
1466
|
-
optionCount: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1467
|
-
options: import_zod62.z.array(StudioInterruptOptionSchema).nullable(),
|
|
1468
|
-
workflowId: import_zod62.z.string().min(1),
|
|
1469
|
-
nodeId: import_zod62.z.string().nullable(),
|
|
1470
|
-
sessionId: import_zod62.z.string().nullable(),
|
|
1471
|
-
userId: import_zod62.z.string().nullable(),
|
|
1472
|
-
tenantId: import_zod62.z.string().nullable(),
|
|
1473
|
-
response: import_zod62.z.string().nullable(),
|
|
1474
|
-
responseValue: StudioInterruptValueSchema.nullable().default(null),
|
|
1475
|
-
responseCaptured: import_zod62.z.boolean(),
|
|
1476
|
-
resumeOutcome: StudioResumeOutcomeSchema.nullable(),
|
|
1477
|
-
resumeError: import_zod62.z.string().nullable(),
|
|
1478
|
-
runId: import_zod62.z.string().min(1),
|
|
1479
|
-
resumeToken: import_zod62.z.string().nullable(),
|
|
1480
|
-
resolvedBy: import_zod62.z.string().nullable(),
|
|
1481
|
-
environment: import_zod62.z.string().min(1)
|
|
1482
|
-
}).strip();
|
|
1483
|
-
var StudioDetailEventSchema = import_zod62.z.object({
|
|
1484
|
-
id: import_zod62.z.string().min(1),
|
|
1485
|
-
// Studio readers must preserve future event kinds within the same API major.
|
|
1486
|
-
type: import_zod62.z.string().min(1),
|
|
1487
|
-
occurredAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1488
|
-
receivedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1489
|
-
environment: import_zod62.z.string().min(1),
|
|
1490
|
-
serviceName: import_zod62.z.string().min(1),
|
|
1491
|
-
deploymentRef: import_zod62.z.string().nullable(),
|
|
1492
|
-
traceId: import_zod62.z.string().nullable(),
|
|
1493
|
-
spanId: import_zod62.z.string().nullable(),
|
|
1494
|
-
parentSpanId: import_zod62.z.string().nullable(),
|
|
1495
|
-
runId: import_zod62.z.string().min(1),
|
|
1496
|
-
sessionId: import_zod62.z.string().nullable(),
|
|
1497
|
-
workflowId: import_zod62.z.string().min(1),
|
|
1498
|
-
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1499
|
-
nodeId: import_zod62.z.string().nullable(),
|
|
1500
|
-
userId: import_zod62.z.string().nullable(),
|
|
1501
|
-
tenantId: import_zod62.z.string().nullable(),
|
|
1502
|
-
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1503
|
-
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable(),
|
|
1504
|
-
payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
|
|
1505
|
-
}).strip();
|
|
1506
|
-
var StudioWorkflowNodeSchema = import_zod62.z.object({
|
|
1507
|
-
tools: import_zod62.z.array(StudioToolSchema).optional(),
|
|
1508
|
-
toolDiscovery: ToolDiscoverySchema.optional(),
|
|
1509
|
-
id: import_zod62.z.string().min(1),
|
|
1510
|
-
label: import_zod62.z.string(),
|
|
1511
|
-
type: import_zod62.z.string().nullable(),
|
|
1512
|
-
state: import_zod62.z.enum(["healthy", "warning", "failed", "interrupted", "retried"]).nullable(),
|
|
1513
|
-
provider: import_zod62.z.string().nullable(),
|
|
1514
|
-
model: import_zod62.z.string().nullable(),
|
|
1515
|
-
metrics: StudioMetricSchema
|
|
1516
|
-
}).strip();
|
|
1517
|
-
var StudioWorkflowInternalEdgeSchema = import_zod62.z.object({
|
|
1518
|
-
id: import_zod62.z.string().min(1),
|
|
1519
|
-
source: import_zod62.z.string().min(1),
|
|
1520
|
-
target: import_zod62.z.string().min(1),
|
|
1521
|
-
condition: import_zod62.z.string().nullable()
|
|
1522
|
-
}).strip();
|
|
1523
|
-
var StudioWorkflowTransitionSchema = import_zod62.z.object({
|
|
1524
|
-
kind: import_zod62.z.enum(["call", "handoff"]).optional(),
|
|
1525
|
-
id: import_zod62.z.string().min(1),
|
|
1526
|
-
sourceWorkflowId: import_zod62.z.string().min(1),
|
|
1527
|
-
sourceNodeId: import_zod62.z.string().nullable(),
|
|
1528
|
-
targetWorkflowId: import_zod62.z.string().min(1),
|
|
1529
|
-
condition: import_zod62.z.string().nullable(),
|
|
1530
|
-
volume: import_zod62.z.number().int().nonnegative(),
|
|
1531
|
-
successRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1532
|
-
medianDurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1533
|
-
errorRate: import_zod62.z.number().min(0).max(100).nullable()
|
|
1534
|
-
}).strip();
|
|
1535
|
-
var StudioWorkflowSchema = import_zod62.z.object({
|
|
1536
|
-
id: import_zod62.z.string().min(1),
|
|
1537
|
-
name: import_zod62.z.string().min(1),
|
|
1538
|
-
description: import_zod62.z.string().nullable(),
|
|
1539
|
-
versions: import_zod62.z.array(import_zod62.z.string()),
|
|
1540
|
-
activeVersion: import_zod62.z.string().nullable(),
|
|
1541
|
-
activeRevisionId: import_zod62.z.string().nullable(),
|
|
1542
|
-
health: StudioWorkflowHealthSchema,
|
|
1543
|
-
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1544
|
-
lastActivityAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1545
|
-
metrics: StudioMetricSchema,
|
|
1546
|
-
nodes: import_zod62.z.array(StudioWorkflowNodeSchema),
|
|
1547
|
-
internalEdges: import_zod62.z.array(StudioWorkflowInternalEdgeSchema)
|
|
1548
|
-
}).strip();
|
|
1549
|
-
var StudioRunsResponseSchema = import_zod62.z.object({
|
|
1550
|
-
runs: import_zod62.z.array(StudioRunSchema),
|
|
1551
|
-
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1552
|
-
}).strip();
|
|
1553
|
-
var StudioSessionsResponseSchema = import_zod62.z.object({
|
|
1554
|
-
sessions: import_zod62.z.array(StudioSessionSchema),
|
|
1555
|
-
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1556
|
-
}).strip();
|
|
1557
|
-
var StudioInterruptsResponseSchema = import_zod62.z.object({
|
|
1558
|
-
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1559
|
-
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1560
|
-
}).strip();
|
|
1561
|
-
var StudioRunDetailResponseSchema = import_zod62.z.object({
|
|
1562
|
-
run: StudioRunSchema,
|
|
1563
|
-
scores: import_zod62.z.array(StudioScoreSchema).optional(),
|
|
1564
|
-
canReview: import_zod62.z.boolean().optional(),
|
|
1565
|
-
reviewActorId: import_zod62.z.string().optional(),
|
|
1566
|
-
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1567
|
-
session: StudioSessionSchema.nullable(),
|
|
1568
|
-
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1569
|
-
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1570
|
-
}).strip();
|
|
1571
|
-
var StudioSessionDetailResponseSchema = import_zod62.z.object({
|
|
1572
|
-
session: StudioSessionSchema,
|
|
1573
|
-
runs: import_zod62.z.array(StudioRunSchema),
|
|
1574
|
-
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1575
|
-
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1576
|
-
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1577
|
-
}).strip();
|
|
1578
|
-
var StudioInterruptDetailResponseSchema = import_zod62.z.object({
|
|
1579
|
-
interrupt: StudioInterruptSchema,
|
|
1580
|
-
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1581
|
-
run: StudioRunSchema.nullable(),
|
|
1582
|
-
session: StudioSessionSchema.nullable(),
|
|
1583
|
-
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1584
|
-
}).strip();
|
|
1585
|
-
var StudioWorkflowsResponseSchema = import_zod62.z.object({
|
|
1586
|
-
workflows: import_zod62.z.array(StudioWorkflowSchema),
|
|
1587
|
-
observedCalls: import_zod62.z.array(
|
|
1588
|
-
StudioWorkflowTransitionSchema.extend({
|
|
1589
|
-
runId: import_zod62.z.string(),
|
|
1590
|
-
invocationId: import_zod62.z.string(),
|
|
1591
|
-
branchId: import_zod62.z.string()
|
|
1592
|
-
})
|
|
1593
|
-
).optional(),
|
|
1594
|
-
transitions: import_zod62.z.array(StudioWorkflowTransitionSchema),
|
|
1595
|
-
cohort: StudioTimeRangeContextSchema.extend({
|
|
1596
|
-
workflowId: import_zod62.z.string().nullable(),
|
|
1597
|
-
version: import_zod62.z.string().nullable()
|
|
1598
|
-
}).strip()
|
|
1599
|
-
}).strip();
|
|
1600
|
-
var StudioCatalogsResponseSchema = import_zod62.z.object({
|
|
1601
|
-
environments: import_zod62.z.array(import_zod62.z.string()),
|
|
1602
|
-
providers: import_zod62.z.array(import_zod62.z.string()),
|
|
1603
|
-
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1604
|
-
workflows: import_zod62.z.array(import_zod62.z.string()),
|
|
1605
|
-
tags: import_zod62.z.array(import_zod62.z.string())
|
|
1606
|
-
}).strip();
|
|
1607
|
-
var StudioContextResponseSchema = import_zod62.z.object({
|
|
1608
|
-
organization: import_zod62.z.object({
|
|
1609
|
-
name: import_zod62.z.string().min(1)
|
|
1610
|
-
}).strip(),
|
|
1611
|
-
project: import_zod62.z.object({
|
|
1612
|
-
name: import_zod62.z.string().min(1)
|
|
1613
|
-
}).strip(),
|
|
1614
|
-
environments: import_zod62.z.array(import_zod62.z.string()),
|
|
1615
|
-
apiKey: import_zod62.z.object({
|
|
1616
|
-
mode: import_zod62.z.enum(["test", "live"]),
|
|
1617
|
-
scopes: import_zod62.z.array(import_zod62.z.string())
|
|
1618
|
-
}).strip().nullable(),
|
|
1619
|
-
api: import_zod62.z.object({
|
|
1620
|
-
status: import_zod62.z.literal("ok"),
|
|
1621
|
-
service: import_zod62.z.literal("kortyx-api"),
|
|
1622
|
-
version: import_zod62.z.string().min(1)
|
|
1623
|
-
}).strip()
|
|
1624
|
-
}).strip();
|
|
1625
|
-
}
|
|
1626
|
-
});
|
|
1627
|
-
|
|
1628
32
|
// ../hooks/dist/internal.js
|
|
1629
33
|
var require_internal = __commonJS({
|
|
1630
34
|
"../hooks/dist/internal.js"(exports2, module2) {
|
|
@@ -1647,7 +51,7 @@ var require_internal = __commonJS({
|
|
|
1647
51
|
var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
|
|
1648
52
|
var internal_exports = {};
|
|
1649
53
|
__export2(internal_exports, {
|
|
1650
|
-
captureDiagnosticContent: () => captureDiagnosticContent,
|
|
54
|
+
captureDiagnosticContent: () => import_telemetry_contracts4.captureDiagnosticContent,
|
|
1651
55
|
errorDiagnostics: () => errorDiagnostics,
|
|
1652
56
|
exceptionDiagnostics: () => exceptionDiagnostics,
|
|
1653
57
|
isModelTraceSpan: () => isModelTraceSpan,
|
|
@@ -1655,255 +59,20 @@ var require_internal = __commonJS({
|
|
|
1655
59
|
safeSpan: () => safeSpan,
|
|
1656
60
|
safeStartSpan: () => safeStartSpan2,
|
|
1657
61
|
safeTelemetryMetadata: () => safeTelemetryMetadata2,
|
|
1658
|
-
summarizeDiagnostic: () =>
|
|
62
|
+
summarizeDiagnostic: () => import_telemetry_contracts32.summarizeDiagnostic,
|
|
1659
63
|
withSafeTraceSpan: () => withSafeTraceSpan2
|
|
1660
64
|
});
|
|
1661
65
|
module2.exports = __toCommonJS2(internal_exports);
|
|
1662
|
-
var
|
|
1663
|
-
var
|
|
1664
|
-
var
|
|
1665
|
-
var encoder = new TextEncoder();
|
|
1666
|
-
var nativeStackGetter = Object.getOwnPropertyDescriptor(
|
|
1667
|
-
new Error(),
|
|
1668
|
-
"stack"
|
|
1669
|
-
)?.get;
|
|
1670
|
-
function nativeStack(object2, getter) {
|
|
1671
|
-
if (!getter || getter !== nativeStackGetter)
|
|
1672
|
-
throw new Error("Custom stack accessor");
|
|
1673
|
-
for (const key of ["name", "message"]) {
|
|
1674
|
-
let cursor = object2;
|
|
1675
|
-
while (cursor) {
|
|
1676
|
-
const descriptor = Object.getOwnPropertyDescriptor(cursor, key);
|
|
1677
|
-
if (descriptor) {
|
|
1678
|
-
if (!("value" in descriptor) || typeof descriptor.value !== "string")
|
|
1679
|
-
throw new Error("Custom error formatting");
|
|
1680
|
-
break;
|
|
1681
|
-
}
|
|
1682
|
-
cursor = Object.getPrototypeOf(cursor);
|
|
1683
|
-
}
|
|
1684
|
-
}
|
|
1685
|
-
const formatter = Object.getOwnPropertyDescriptor(Error, "prepareStackTrace");
|
|
1686
|
-
if (formatter && (!("value" in formatter) || !formatter.writable))
|
|
1687
|
-
throw new Error("Custom stack formatter");
|
|
1688
|
-
Object.defineProperty(Error, "prepareStackTrace", {
|
|
1689
|
-
...formatter ?? { configurable: true, writable: true },
|
|
1690
|
-
value: void 0
|
|
1691
|
-
});
|
|
1692
|
-
try {
|
|
1693
|
-
return getter.call(object2);
|
|
1694
|
-
} finally {
|
|
1695
|
-
if (formatter) Object.defineProperty(Error, "prepareStackTrace", formatter);
|
|
1696
|
-
else Reflect.deleteProperty(Error, "prepareStackTrace");
|
|
1697
|
-
}
|
|
1698
|
-
}
|
|
1699
|
-
var pointer = (key) => key.replace(/~/g, "~0").replace(/\//g, "~1");
|
|
1700
|
-
function captureDiagnosticContent(error, project) {
|
|
1701
|
-
const omissions = [];
|
|
1702
|
-
const redactions = [];
|
|
1703
|
-
const seen = /* @__PURE__ */ new WeakMap();
|
|
1704
|
-
let nodes = 0;
|
|
1705
|
-
let remaining = import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes - 64 * 1024;
|
|
1706
|
-
const omit = (path, reason, extra = {}) => {
|
|
1707
|
-
if (omissions.length < 1e3)
|
|
1708
|
-
omissions.push({ path: path.slice(0, 4096), reason, ...extra });
|
|
1709
|
-
return { $omitted: reason };
|
|
1710
|
-
};
|
|
1711
|
-
const visit = (value, path, depth) => {
|
|
1712
|
-
if (++nodes > import_telemetry_contracts.DIAGNOSTIC_LIMITS.nodes) return omit(path, "node_limit");
|
|
1713
|
-
if (depth > import_telemetry_contracts.DIAGNOSTIC_LIMITS.depth) return omit(path, "depth_limit");
|
|
1714
|
-
if (typeof value === "string") {
|
|
1715
|
-
const clean = (0, import_telemetry_contracts.redactDiagnosticText)(value);
|
|
1716
|
-
if (clean !== value && redactions.length < 1e3)
|
|
1717
|
-
redactions.push({
|
|
1718
|
-
path: path.slice(0, 4096),
|
|
1719
|
-
reason: "credential_text"
|
|
1720
|
-
});
|
|
1721
|
-
const bytes = encoder.encode(JSON.stringify(clean)).length;
|
|
1722
|
-
if (bytes > remaining)
|
|
1723
|
-
return omit(path, "byte_limit", {
|
|
1724
|
-
originalBytes: encoder.encode(value).length
|
|
1725
|
-
});
|
|
1726
|
-
remaining -= bytes;
|
|
1727
|
-
return clean;
|
|
1728
|
-
}
|
|
1729
|
-
if (value === null || typeof value === "boolean") return value;
|
|
1730
|
-
if (typeof value === "number")
|
|
1731
|
-
return Number.isFinite(value) ? value : { $type: "number", value: String(value) };
|
|
1732
|
-
if (typeof value === "bigint")
|
|
1733
|
-
return { $type: "bigint", value: value.toString() };
|
|
1734
|
-
if (typeof value === "undefined") return { $type: "undefined" };
|
|
1735
|
-
if (typeof value === "function" || typeof value === "symbol")
|
|
1736
|
-
return omit(path, `unsupported_${typeof value}`);
|
|
1737
|
-
const object2 = value;
|
|
1738
|
-
const reference = seen.get(object2);
|
|
1739
|
-
if (reference) return { $ref: reference };
|
|
1740
|
-
seen.set(object2, path);
|
|
1741
|
-
remaining -= 32;
|
|
1742
|
-
if (remaining < 0) return omit(path, "byte_limit");
|
|
1743
|
-
try {
|
|
1744
|
-
if (value instanceof Date)
|
|
1745
|
-
return { $type: "Date", value: Date.prototype.toISOString.call(value) };
|
|
1746
|
-
if (value instanceof Map || value instanceof Set) {
|
|
1747
|
-
const isMap = value instanceof Map;
|
|
1748
|
-
const iterator = isMap ? Map.prototype.entries.call(value) : Set.prototype.values.call(value);
|
|
1749
|
-
const count = Object.getOwnPropertyDescriptor(
|
|
1750
|
-
isMap ? Map.prototype : Set.prototype,
|
|
1751
|
-
"size"
|
|
1752
|
-
)?.get?.call(value);
|
|
1753
|
-
const entries = [];
|
|
1754
|
-
for (let index = 0; index < Math.min(count, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties); index++)
|
|
1755
|
-
entries.push(iterator.next().value);
|
|
1756
|
-
if (count > entries.length)
|
|
1757
|
-
omissions.push({
|
|
1758
|
-
path,
|
|
1759
|
-
reason: "collection_limit",
|
|
1760
|
-
originalCount: count
|
|
1761
|
-
});
|
|
1762
|
-
return isMap ? {
|
|
1763
|
-
$type: "Map",
|
|
1764
|
-
entries: visit(entries, `${path}/entries`, depth + 1)
|
|
1765
|
-
} : {
|
|
1766
|
-
$type: "Set",
|
|
1767
|
-
values: visit(entries, `${path}/values`, depth + 1)
|
|
1768
|
-
};
|
|
1769
|
-
}
|
|
1770
|
-
if (Array.isArray(value)) {
|
|
1771
|
-
const count = Math.min(value.length, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties);
|
|
1772
|
-
const items = Array.from(
|
|
1773
|
-
{ length: count },
|
|
1774
|
-
(_, index) => read(value, String(index), `${path}/${index}`, depth)
|
|
1775
|
-
);
|
|
1776
|
-
if (count < value.length)
|
|
1777
|
-
items.push(
|
|
1778
|
-
omit(path, "collection_limit", { originalCount: value.length })
|
|
1779
|
-
);
|
|
1780
|
-
return items;
|
|
1781
|
-
}
|
|
1782
|
-
const keys = Object.getOwnPropertyNames(object2);
|
|
1783
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
1784
|
-
if (value instanceof Error) {
|
|
1785
|
-
const ownName = Object.getOwnPropertyDescriptor(object2, "name");
|
|
1786
|
-
let type = ownName && "value" in ownName ? ownName.value : void 0;
|
|
1787
|
-
if (typeof type !== "string") {
|
|
1788
|
-
const name = Object.getOwnPropertyDescriptor(
|
|
1789
|
-
Object.getPrototypeOf(value),
|
|
1790
|
-
"name"
|
|
1791
|
-
);
|
|
1792
|
-
type = name && "value" in name ? name.value : "Error";
|
|
1793
|
-
}
|
|
1794
|
-
result.type = visit(
|
|
1795
|
-
typeof type === "string" ? type : "Error",
|
|
1796
|
-
`${path}/type`,
|
|
1797
|
-
depth + 1
|
|
1798
|
-
);
|
|
1799
|
-
}
|
|
1800
|
-
for (const key of keys.slice(0, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)) {
|
|
1801
|
-
if (["__proto__", "constructor", "prototype"].includes(key)) {
|
|
1802
|
-
result[key] = omit(`${path}/${pointer(key)}`, "unsafe_key");
|
|
1803
|
-
continue;
|
|
1804
|
-
}
|
|
1805
|
-
remaining -= encoder.encode(key).length + 4;
|
|
1806
|
-
result[key] = read(object2, key, `${path}/${pointer(key)}`, depth);
|
|
1807
|
-
}
|
|
1808
|
-
if (keys.length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)
|
|
1809
|
-
result.$omittedProperties = omit(path, "property_limit", {
|
|
1810
|
-
originalCount: keys.length
|
|
1811
|
-
});
|
|
1812
|
-
return result;
|
|
1813
|
-
} catch {
|
|
1814
|
-
return omit(path, "unreadable_object");
|
|
1815
|
-
}
|
|
1816
|
-
};
|
|
1817
|
-
const read = (object2, key, path, depth) => {
|
|
1818
|
-
if ((0, import_telemetry_contracts.isPrivateDiagnosticField)(key)) {
|
|
1819
|
-
if (redactions.length < 1e3)
|
|
1820
|
-
redactions.push({
|
|
1821
|
-
path: path.slice(0, 4096),
|
|
1822
|
-
reason: "credential_field"
|
|
1823
|
-
});
|
|
1824
|
-
return "[REDACTED]";
|
|
1825
|
-
}
|
|
1826
|
-
try {
|
|
1827
|
-
const property = Object.getOwnPropertyDescriptor(object2, key);
|
|
1828
|
-
if (property && "value" in property)
|
|
1829
|
-
return visit(property.value, path, depth + 1);
|
|
1830
|
-
if (key === "stack" && object2 instanceof Error && property?.get && property.get === nativeStackGetter)
|
|
1831
|
-
return visit(nativeStack(object2, property.get), path, depth + 1);
|
|
1832
|
-
return omit(
|
|
1833
|
-
path,
|
|
1834
|
-
property ? "accessor_not_evaluated" : "unavailable_property"
|
|
1835
|
-
);
|
|
1836
|
-
} catch {
|
|
1837
|
-
return omit(path, "unreadable_property");
|
|
1838
|
-
}
|
|
1839
|
-
};
|
|
1840
|
-
try {
|
|
1841
|
-
const original = project ? project(error) : error;
|
|
1842
|
-
if (project && original === null) return null;
|
|
1843
|
-
const captured = visit(original, "#/data", 0);
|
|
1844
|
-
const upstreamOmissions = (0, import_errors16.errorProperty)(original, "diagnosticOmissions");
|
|
1845
|
-
if (Array.isArray(upstreamOmissions))
|
|
1846
|
-
for (const note of upstreamOmissions.slice(0, 1e3)) {
|
|
1847
|
-
const path = (0, import_errors16.errorProperty)(note, "path");
|
|
1848
|
-
const reason = (0, import_errors16.errorProperty)(note, "reason");
|
|
1849
|
-
if (typeof path === "string" && typeof reason === "string")
|
|
1850
|
-
omissions.push({
|
|
1851
|
-
path: (0, import_telemetry_contracts.redactDiagnosticText)(path).slice(0, 4096),
|
|
1852
|
-
reason: (0, import_telemetry_contracts.redactDiagnosticText)(reason).slice(0, 128)
|
|
1853
|
-
});
|
|
1854
|
-
}
|
|
1855
|
-
const data = captured && typeof captured === "object" && !Array.isArray(captured) ? captured : {
|
|
1856
|
-
type: "Error",
|
|
1857
|
-
message: typeof captured === "string" ? captured : "Execution reported a fault.",
|
|
1858
|
-
value: captured
|
|
1859
|
-
};
|
|
1860
|
-
if (typeof data.message !== "string" && !("message" in data))
|
|
1861
|
-
data.message = "Execution reported a fault.";
|
|
1862
|
-
if (!("type" in data))
|
|
1863
|
-
data.type = typeof data.name === "string" ? data.name : "Error";
|
|
1864
|
-
const content = {
|
|
1865
|
-
schemaVersion: 1,
|
|
1866
|
-
data,
|
|
1867
|
-
capture: {
|
|
1868
|
-
status: omissions.length ? "partial" : "complete",
|
|
1869
|
-
omissions,
|
|
1870
|
-
redactions
|
|
1871
|
-
}
|
|
1872
|
-
};
|
|
1873
|
-
if (encoder.encode(JSON.stringify(content)).length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes)
|
|
1874
|
-
return {
|
|
1875
|
-
schemaVersion: 1,
|
|
1876
|
-
data: {
|
|
1877
|
-
type: "Error",
|
|
1878
|
-
message: "Diagnostic exceeded the capture budget."
|
|
1879
|
-
},
|
|
1880
|
-
capture: {
|
|
1881
|
-
status: "partial",
|
|
1882
|
-
redactions: [],
|
|
1883
|
-
omissions: [{ path: "#/data", reason: "serialized_byte_limit" }]
|
|
1884
|
-
}
|
|
1885
|
-
};
|
|
1886
|
-
return content;
|
|
1887
|
-
} catch {
|
|
1888
|
-
return {
|
|
1889
|
-
schemaVersion: 1,
|
|
1890
|
-
data: { type: "Error", message: "Diagnostic capture failed." },
|
|
1891
|
-
capture: {
|
|
1892
|
-
status: "failed",
|
|
1893
|
-
redactions: [],
|
|
1894
|
-
omissions: [{ path: "#/data", reason: "projection_or_capture_failed" }]
|
|
1895
|
-
}
|
|
1896
|
-
};
|
|
1897
|
-
}
|
|
1898
|
-
}
|
|
1899
|
-
var import_telemetry_contracts2 = require_dist();
|
|
66
|
+
var import_telemetry_contracts32 = require("@kortyx/telemetry-contracts");
|
|
67
|
+
var import_telemetry_contracts4 = require("@kortyx/telemetry-contracts");
|
|
68
|
+
var import_telemetry_contracts22 = require("@kortyx/telemetry-contracts");
|
|
1900
69
|
function exceptionDiagnostics(error, project) {
|
|
1901
|
-
const content = captureDiagnosticContent(error, project);
|
|
70
|
+
const content = (0, import_telemetry_contracts4.captureDiagnosticContent)(error, project);
|
|
1902
71
|
return !content || content.capture.status === "failed" ? null : content.data;
|
|
1903
72
|
}
|
|
1904
73
|
function errorDiagnostics(error, project) {
|
|
1905
74
|
const captured = exceptionDiagnostics(error, project);
|
|
1906
|
-
const details = captured ? (0,
|
|
75
|
+
const details = captured ? (0, import_telemetry_contracts22.summarizeDiagnostic)(captured) : null;
|
|
1907
76
|
return details ? {
|
|
1908
77
|
errorMessage: details.message.slice(0, 8192),
|
|
1909
78
|
...details.type ? { errorType: details.type.slice(0, 256) } : {}
|
|
@@ -1914,7 +83,7 @@ var require_internal = __commonJS({
|
|
|
1914
83
|
}
|
|
1915
84
|
var import_node_crypto8 = require("crypto");
|
|
1916
85
|
var import_core6 = require("@kortyx/core");
|
|
1917
|
-
var
|
|
86
|
+
var import_errors16 = require("@kortyx/core/errors");
|
|
1918
87
|
function safeSpan(span) {
|
|
1919
88
|
const guard = (fn) => (...args) => {
|
|
1920
89
|
try {
|
|
@@ -1996,7 +165,7 @@ var require_internal = __commonJS({
|
|
|
1996
165
|
var import_node2 = require("@kortyx/usage-telemetry/node");
|
|
1997
166
|
var package_default2 = {
|
|
1998
167
|
name: "@kortyx/hooks",
|
|
1999
|
-
version: "0.
|
|
168
|
+
version: "0.35.0",
|
|
2000
169
|
private: false,
|
|
2001
170
|
description: "Lifecycle hooks and extension points for Kortyx.",
|
|
2002
171
|
keywords: [
|
|
@@ -2076,7 +245,7 @@ var require_internal = __commonJS({
|
|
|
2076
245
|
function assertModelFinish(result) {
|
|
2077
246
|
if (result.finishReason?.unified !== "content-filter") return;
|
|
2078
247
|
throw Object.assign(
|
|
2079
|
-
new
|
|
248
|
+
new import_errors16.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
|
|
2080
249
|
category: "provider",
|
|
2081
250
|
retryable: false,
|
|
2082
251
|
safeMessage: "The provider refused the request."
|
|
@@ -2393,7 +562,7 @@ var require_internal = __commonJS({
|
|
|
2393
562
|
});
|
|
2394
563
|
|
|
2395
564
|
// ../hooks/dist/index.js
|
|
2396
|
-
var
|
|
565
|
+
var require_dist = __commonJS({
|
|
2397
566
|
"../hooks/dist/index.js"(exports2, module2) {
|
|
2398
567
|
var __defProp2 = Object.defineProperty;
|
|
2399
568
|
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
|
|
@@ -2437,7 +606,7 @@ var require_dist2 = __commonJS({
|
|
|
2437
606
|
workflowCallFingerprint: () => workflowCallFingerprint
|
|
2438
607
|
});
|
|
2439
608
|
module2.exports = __toCommonJS2(index_exports2);
|
|
2440
|
-
var
|
|
609
|
+
var import_zod15 = require("zod");
|
|
2441
610
|
var import_node_async_hooks2 = require("async_hooks");
|
|
2442
611
|
var storage = new import_node_async_hooks2.AsyncLocalStorage();
|
|
2443
612
|
var isRecord5 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
@@ -2632,7 +801,7 @@ var require_dist2 = __commonJS({
|
|
|
2632
801
|
var import_node2 = require("@kortyx/usage-telemetry/node");
|
|
2633
802
|
var package_default2 = {
|
|
2634
803
|
name: "@kortyx/hooks",
|
|
2635
|
-
version: "0.
|
|
804
|
+
version: "0.35.0",
|
|
2636
805
|
private: false,
|
|
2637
806
|
description: "Lifecycle hooks and extension points for Kortyx.",
|
|
2638
807
|
keywords: [
|
|
@@ -2709,9 +878,9 @@ var require_dist2 = __commonJS({
|
|
|
2709
878
|
measure: (...args) => client2().measure(...args),
|
|
2710
879
|
flush: () => client2().flush()
|
|
2711
880
|
};
|
|
2712
|
-
var optionsSchema =
|
|
2713
|
-
message:
|
|
2714
|
-
data:
|
|
881
|
+
var optionsSchema = import_zod15.z.object({
|
|
882
|
+
message: import_zod15.z.string().optional(),
|
|
883
|
+
data: import_zod15.z.json().optional()
|
|
2715
884
|
}).strict();
|
|
2716
885
|
async function completeResponseInternal(options = {}) {
|
|
2717
886
|
const ctx = getHookContext();
|
|
@@ -3181,261 +1350,26 @@ var require_dist2 = __commonJS({
|
|
|
3181
1350
|
var import_node_async_hooks22 = require("async_hooks");
|
|
3182
1351
|
var import_node_crypto32 = require("crypto");
|
|
3183
1352
|
var import_core22 = require("@kortyx/core");
|
|
3184
|
-
var import_errors32 = require("@kortyx/core/errors");
|
|
3185
|
-
var import_telemetry_contracts2 = require_dist();
|
|
3186
1353
|
var import_errors22 = require("@kortyx/core/errors");
|
|
3187
|
-
var
|
|
3188
|
-
var
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
"
|
|
3192
|
-
|
|
3193
|
-
function
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
}
|
|
3207
|
-
}
|
|
3208
|
-
const formatter = Object.getOwnPropertyDescriptor(Error, "prepareStackTrace");
|
|
3209
|
-
if (formatter && (!("value" in formatter) || !formatter.writable))
|
|
3210
|
-
throw new Error("Custom stack formatter");
|
|
3211
|
-
Object.defineProperty(Error, "prepareStackTrace", {
|
|
3212
|
-
...formatter ?? { configurable: true, writable: true },
|
|
3213
|
-
value: void 0
|
|
3214
|
-
});
|
|
3215
|
-
try {
|
|
3216
|
-
return getter.call(object2);
|
|
3217
|
-
} finally {
|
|
3218
|
-
if (formatter) Object.defineProperty(Error, "prepareStackTrace", formatter);
|
|
3219
|
-
else Reflect.deleteProperty(Error, "prepareStackTrace");
|
|
3220
|
-
}
|
|
3221
|
-
}
|
|
3222
|
-
var pointer = (key) => key.replace(/~/g, "~0").replace(/\//g, "~1");
|
|
3223
|
-
function captureDiagnosticContent(error, project) {
|
|
3224
|
-
const omissions = [];
|
|
3225
|
-
const redactions = [];
|
|
3226
|
-
const seen = /* @__PURE__ */ new WeakMap();
|
|
3227
|
-
let nodes = 0;
|
|
3228
|
-
let remaining = import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes - 64 * 1024;
|
|
3229
|
-
const omit = (path, reason, extra = {}) => {
|
|
3230
|
-
if (omissions.length < 1e3)
|
|
3231
|
-
omissions.push({ path: path.slice(0, 4096), reason, ...extra });
|
|
3232
|
-
return { $omitted: reason };
|
|
3233
|
-
};
|
|
3234
|
-
const visit = (value, path, depth) => {
|
|
3235
|
-
if (++nodes > import_telemetry_contracts.DIAGNOSTIC_LIMITS.nodes) return omit(path, "node_limit");
|
|
3236
|
-
if (depth > import_telemetry_contracts.DIAGNOSTIC_LIMITS.depth) return omit(path, "depth_limit");
|
|
3237
|
-
if (typeof value === "string") {
|
|
3238
|
-
const clean = (0, import_telemetry_contracts.redactDiagnosticText)(value);
|
|
3239
|
-
if (clean !== value && redactions.length < 1e3)
|
|
3240
|
-
redactions.push({
|
|
3241
|
-
path: path.slice(0, 4096),
|
|
3242
|
-
reason: "credential_text"
|
|
3243
|
-
});
|
|
3244
|
-
const bytes = encoder.encode(JSON.stringify(clean)).length;
|
|
3245
|
-
if (bytes > remaining)
|
|
3246
|
-
return omit(path, "byte_limit", {
|
|
3247
|
-
originalBytes: encoder.encode(value).length
|
|
3248
|
-
});
|
|
3249
|
-
remaining -= bytes;
|
|
3250
|
-
return clean;
|
|
3251
|
-
}
|
|
3252
|
-
if (value === null || typeof value === "boolean") return value;
|
|
3253
|
-
if (typeof value === "number")
|
|
3254
|
-
return Number.isFinite(value) ? value : { $type: "number", value: String(value) };
|
|
3255
|
-
if (typeof value === "bigint")
|
|
3256
|
-
return { $type: "bigint", value: value.toString() };
|
|
3257
|
-
if (typeof value === "undefined") return { $type: "undefined" };
|
|
3258
|
-
if (typeof value === "function" || typeof value === "symbol")
|
|
3259
|
-
return omit(path, `unsupported_${typeof value}`);
|
|
3260
|
-
const object2 = value;
|
|
3261
|
-
const reference = seen.get(object2);
|
|
3262
|
-
if (reference) return { $ref: reference };
|
|
3263
|
-
seen.set(object2, path);
|
|
3264
|
-
remaining -= 32;
|
|
3265
|
-
if (remaining < 0) return omit(path, "byte_limit");
|
|
3266
|
-
try {
|
|
3267
|
-
if (value instanceof Date)
|
|
3268
|
-
return { $type: "Date", value: Date.prototype.toISOString.call(value) };
|
|
3269
|
-
if (value instanceof Map || value instanceof Set) {
|
|
3270
|
-
const isMap = value instanceof Map;
|
|
3271
|
-
const iterator = isMap ? Map.prototype.entries.call(value) : Set.prototype.values.call(value);
|
|
3272
|
-
const count = Object.getOwnPropertyDescriptor(
|
|
3273
|
-
isMap ? Map.prototype : Set.prototype,
|
|
3274
|
-
"size"
|
|
3275
|
-
)?.get?.call(value);
|
|
3276
|
-
const entries = [];
|
|
3277
|
-
for (let index = 0; index < Math.min(count, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties); index++)
|
|
3278
|
-
entries.push(iterator.next().value);
|
|
3279
|
-
if (count > entries.length)
|
|
3280
|
-
omissions.push({
|
|
3281
|
-
path,
|
|
3282
|
-
reason: "collection_limit",
|
|
3283
|
-
originalCount: count
|
|
3284
|
-
});
|
|
3285
|
-
return isMap ? {
|
|
3286
|
-
$type: "Map",
|
|
3287
|
-
entries: visit(entries, `${path}/entries`, depth + 1)
|
|
3288
|
-
} : {
|
|
3289
|
-
$type: "Set",
|
|
3290
|
-
values: visit(entries, `${path}/values`, depth + 1)
|
|
3291
|
-
};
|
|
3292
|
-
}
|
|
3293
|
-
if (Array.isArray(value)) {
|
|
3294
|
-
const count = Math.min(value.length, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties);
|
|
3295
|
-
const items = Array.from(
|
|
3296
|
-
{ length: count },
|
|
3297
|
-
(_, index) => read(value, String(index), `${path}/${index}`, depth)
|
|
3298
|
-
);
|
|
3299
|
-
if (count < value.length)
|
|
3300
|
-
items.push(
|
|
3301
|
-
omit(path, "collection_limit", { originalCount: value.length })
|
|
3302
|
-
);
|
|
3303
|
-
return items;
|
|
3304
|
-
}
|
|
3305
|
-
const keys = Object.getOwnPropertyNames(object2);
|
|
3306
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
3307
|
-
if (value instanceof Error) {
|
|
3308
|
-
const ownName = Object.getOwnPropertyDescriptor(object2, "name");
|
|
3309
|
-
let type = ownName && "value" in ownName ? ownName.value : void 0;
|
|
3310
|
-
if (typeof type !== "string") {
|
|
3311
|
-
const name = Object.getOwnPropertyDescriptor(
|
|
3312
|
-
Object.getPrototypeOf(value),
|
|
3313
|
-
"name"
|
|
3314
|
-
);
|
|
3315
|
-
type = name && "value" in name ? name.value : "Error";
|
|
3316
|
-
}
|
|
3317
|
-
result.type = visit(
|
|
3318
|
-
typeof type === "string" ? type : "Error",
|
|
3319
|
-
`${path}/type`,
|
|
3320
|
-
depth + 1
|
|
3321
|
-
);
|
|
3322
|
-
}
|
|
3323
|
-
for (const key of keys.slice(0, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)) {
|
|
3324
|
-
if (["__proto__", "constructor", "prototype"].includes(key)) {
|
|
3325
|
-
result[key] = omit(`${path}/${pointer(key)}`, "unsafe_key");
|
|
3326
|
-
continue;
|
|
3327
|
-
}
|
|
3328
|
-
remaining -= encoder.encode(key).length + 4;
|
|
3329
|
-
result[key] = read(object2, key, `${path}/${pointer(key)}`, depth);
|
|
3330
|
-
}
|
|
3331
|
-
if (keys.length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)
|
|
3332
|
-
result.$omittedProperties = omit(path, "property_limit", {
|
|
3333
|
-
originalCount: keys.length
|
|
3334
|
-
});
|
|
3335
|
-
return result;
|
|
3336
|
-
} catch {
|
|
3337
|
-
return omit(path, "unreadable_object");
|
|
3338
|
-
}
|
|
3339
|
-
};
|
|
3340
|
-
const read = (object2, key, path, depth) => {
|
|
3341
|
-
if ((0, import_telemetry_contracts.isPrivateDiagnosticField)(key)) {
|
|
3342
|
-
if (redactions.length < 1e3)
|
|
3343
|
-
redactions.push({
|
|
3344
|
-
path: path.slice(0, 4096),
|
|
3345
|
-
reason: "credential_field"
|
|
3346
|
-
});
|
|
3347
|
-
return "[REDACTED]";
|
|
3348
|
-
}
|
|
3349
|
-
try {
|
|
3350
|
-
const property = Object.getOwnPropertyDescriptor(object2, key);
|
|
3351
|
-
if (property && "value" in property)
|
|
3352
|
-
return visit(property.value, path, depth + 1);
|
|
3353
|
-
if (key === "stack" && object2 instanceof Error && property?.get && property.get === nativeStackGetter)
|
|
3354
|
-
return visit(nativeStack(object2, property.get), path, depth + 1);
|
|
3355
|
-
return omit(
|
|
3356
|
-
path,
|
|
3357
|
-
property ? "accessor_not_evaluated" : "unavailable_property"
|
|
3358
|
-
);
|
|
3359
|
-
} catch {
|
|
3360
|
-
return omit(path, "unreadable_property");
|
|
3361
|
-
}
|
|
3362
|
-
};
|
|
3363
|
-
try {
|
|
3364
|
-
const original = project ? project(error) : error;
|
|
3365
|
-
if (project && original === null) return null;
|
|
3366
|
-
const captured = visit(original, "#/data", 0);
|
|
3367
|
-
const upstreamOmissions = (0, import_errors22.errorProperty)(original, "diagnosticOmissions");
|
|
3368
|
-
if (Array.isArray(upstreamOmissions))
|
|
3369
|
-
for (const note of upstreamOmissions.slice(0, 1e3)) {
|
|
3370
|
-
const path = (0, import_errors22.errorProperty)(note, "path");
|
|
3371
|
-
const reason = (0, import_errors22.errorProperty)(note, "reason");
|
|
3372
|
-
if (typeof path === "string" && typeof reason === "string")
|
|
3373
|
-
omissions.push({
|
|
3374
|
-
path: (0, import_telemetry_contracts.redactDiagnosticText)(path).slice(0, 4096),
|
|
3375
|
-
reason: (0, import_telemetry_contracts.redactDiagnosticText)(reason).slice(0, 128)
|
|
3376
|
-
});
|
|
3377
|
-
}
|
|
3378
|
-
const data = captured && typeof captured === "object" && !Array.isArray(captured) ? captured : {
|
|
3379
|
-
type: "Error",
|
|
3380
|
-
message: typeof captured === "string" ? captured : "Execution reported a fault.",
|
|
3381
|
-
value: captured
|
|
3382
|
-
};
|
|
3383
|
-
if (typeof data.message !== "string" && !("message" in data))
|
|
3384
|
-
data.message = "Execution reported a fault.";
|
|
3385
|
-
if (!("type" in data))
|
|
3386
|
-
data.type = typeof data.name === "string" ? data.name : "Error";
|
|
3387
|
-
const content = {
|
|
3388
|
-
schemaVersion: 1,
|
|
3389
|
-
data,
|
|
3390
|
-
capture: {
|
|
3391
|
-
status: omissions.length ? "partial" : "complete",
|
|
3392
|
-
omissions,
|
|
3393
|
-
redactions
|
|
3394
|
-
}
|
|
3395
|
-
};
|
|
3396
|
-
if (encoder.encode(JSON.stringify(content)).length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes)
|
|
3397
|
-
return {
|
|
3398
|
-
schemaVersion: 1,
|
|
3399
|
-
data: {
|
|
3400
|
-
type: "Error",
|
|
3401
|
-
message: "Diagnostic exceeded the capture budget."
|
|
3402
|
-
},
|
|
3403
|
-
capture: {
|
|
3404
|
-
status: "partial",
|
|
3405
|
-
redactions: [],
|
|
3406
|
-
omissions: [{ path: "#/data", reason: "serialized_byte_limit" }]
|
|
3407
|
-
}
|
|
3408
|
-
};
|
|
3409
|
-
return content;
|
|
3410
|
-
} catch {
|
|
3411
|
-
return {
|
|
3412
|
-
schemaVersion: 1,
|
|
3413
|
-
data: { type: "Error", message: "Diagnostic capture failed." },
|
|
3414
|
-
capture: {
|
|
3415
|
-
status: "failed",
|
|
3416
|
-
redactions: [],
|
|
3417
|
-
omissions: [{ path: "#/data", reason: "projection_or_capture_failed" }]
|
|
3418
|
-
}
|
|
3419
|
-
};
|
|
3420
|
-
}
|
|
3421
|
-
}
|
|
3422
|
-
function exceptionDiagnostics(error, project) {
|
|
3423
|
-
const content = captureDiagnosticContent(error, project);
|
|
3424
|
-
return !content || content.capture.status === "failed" ? null : content.data;
|
|
3425
|
-
}
|
|
3426
|
-
function errorDiagnostics(error, project) {
|
|
3427
|
-
const captured = exceptionDiagnostics(error, project);
|
|
3428
|
-
const details = captured ? (0, import_telemetry_contracts2.summarizeDiagnostic)(captured) : null;
|
|
3429
|
-
return details ? {
|
|
3430
|
-
errorMessage: details.message.slice(0, 8192),
|
|
3431
|
-
...details.type ? { errorType: details.type.slice(0, 256) } : {}
|
|
3432
|
-
} : {};
|
|
3433
|
-
}
|
|
3434
|
-
var import_node_crypto22 = require("crypto");
|
|
3435
|
-
function emitWorkflowCall2(telemetry, type, payload) {
|
|
3436
|
-
const base = telemetry?.correlation;
|
|
3437
|
-
if (!telemetry?.reporter || !telemetry.environment || !telemetry.service || !base?.runId || !base.workflowId)
|
|
3438
|
-
return;
|
|
1354
|
+
var import_telemetry_contracts22 = require("@kortyx/telemetry-contracts");
|
|
1355
|
+
var import_telemetry_contracts4 = require("@kortyx/telemetry-contracts");
|
|
1356
|
+
function exceptionDiagnostics(error, project) {
|
|
1357
|
+
const content = (0, import_telemetry_contracts4.captureDiagnosticContent)(error, project);
|
|
1358
|
+
return !content || content.capture.status === "failed" ? null : content.data;
|
|
1359
|
+
}
|
|
1360
|
+
function errorDiagnostics(error, project) {
|
|
1361
|
+
const captured = exceptionDiagnostics(error, project);
|
|
1362
|
+
const details = captured ? (0, import_telemetry_contracts22.summarizeDiagnostic)(captured) : null;
|
|
1363
|
+
return details ? {
|
|
1364
|
+
errorMessage: details.message.slice(0, 8192),
|
|
1365
|
+
...details.type ? { errorType: details.type.slice(0, 256) } : {}
|
|
1366
|
+
} : {};
|
|
1367
|
+
}
|
|
1368
|
+
var import_node_crypto22 = require("crypto");
|
|
1369
|
+
function emitWorkflowCall2(telemetry, type, payload) {
|
|
1370
|
+
const base = telemetry?.correlation;
|
|
1371
|
+
if (!telemetry?.reporter || !telemetry.environment || !telemetry.service || !base?.runId || !base.workflowId)
|
|
1372
|
+
return;
|
|
3439
1373
|
try {
|
|
3440
1374
|
const active = telemetry.trace?.getActiveContext?.();
|
|
3441
1375
|
void Promise.resolve(
|
|
@@ -3681,7 +1615,7 @@ var require_dist2 = __commonJS({
|
|
|
3681
1615
|
return result;
|
|
3682
1616
|
} catch (error) {
|
|
3683
1617
|
if (args.abortSignal?.aborted) observation.outcome = "cancelled";
|
|
3684
|
-
else if (!(0,
|
|
1618
|
+
else if (!(0, import_errors22.isControlFlowError)(error)) {
|
|
3685
1619
|
const outcome = classification(
|
|
3686
1620
|
args.tool,
|
|
3687
1621
|
error,
|
|
@@ -3816,11 +1750,11 @@ var require_dist2 = __commonJS({
|
|
|
3816
1750
|
var import_prompts3 = require("@kortyx/prompts");
|
|
3817
1751
|
var import_node_crypto42 = require("crypto");
|
|
3818
1752
|
var import_core32 = require("@kortyx/core");
|
|
3819
|
-
var
|
|
1753
|
+
var import_errors32 = require("@kortyx/core/errors");
|
|
3820
1754
|
function assertModelFinish(result) {
|
|
3821
1755
|
if (result.finishReason?.unified !== "content-filter") return;
|
|
3822
1756
|
throw Object.assign(
|
|
3823
|
-
new
|
|
1757
|
+
new import_errors32.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
|
|
3824
1758
|
category: "provider",
|
|
3825
1759
|
retryable: false,
|
|
3826
1760
|
safeMessage: "The provider refused the request."
|
|
@@ -4884,7 +2818,7 @@ var require_dist2 = __commonJS({
|
|
|
4884
2818
|
}
|
|
4885
2819
|
};
|
|
4886
2820
|
}
|
|
4887
|
-
var
|
|
2821
|
+
var import_errors42 = require("@kortyx/core/errors");
|
|
4888
2822
|
var tryJsonParse = (value) => {
|
|
4889
2823
|
try {
|
|
4890
2824
|
return { ok: true, value: JSON.parse(value) };
|
|
@@ -4922,12 +2856,12 @@ var require_dist2 = __commonJS({
|
|
|
4922
2856
|
}
|
|
4923
2857
|
return text;
|
|
4924
2858
|
};
|
|
4925
|
-
var createTruncatedStructuredOutputError = (label, cause) => new
|
|
2859
|
+
var createTruncatedStructuredOutputError = (label, cause) => new import_errors42.ValidationError(
|
|
4926
2860
|
"OUTPUT_TRUNCATED",
|
|
4927
2861
|
`${label} was truncated before producing valid structured output. The model stopped due to output length. Increase maxOutputTokens or simplify the requested output.`,
|
|
4928
2862
|
cause
|
|
4929
2863
|
);
|
|
4930
|
-
var createInvalidStructuredOutputError = (label, cause) => new
|
|
2864
|
+
var createInvalidStructuredOutputError = (label, cause) => new import_errors42.ValidationError(
|
|
4931
2865
|
"INVALID_MODEL_JSON",
|
|
4932
2866
|
`${label} did not produce valid structured output. The model returned text instead of the expected JSON payload. Check the output contract or outputSchema and provider structured-output settings.`,
|
|
4933
2867
|
cause
|
|
@@ -4939,7 +2873,7 @@ var require_dist2 = __commonJS({
|
|
|
4939
2873
|
try {
|
|
4940
2874
|
return parseWithSchema(args.schema, candidate, args.label);
|
|
4941
2875
|
} catch (error) {
|
|
4942
|
-
if (!(error instanceof
|
|
2876
|
+
if (!(error instanceof import_errors42.ValidationError)) throw error;
|
|
4943
2877
|
const failure = args.finishReason?.unified === "length" ? createTruncatedStructuredOutputError(args.label, error) : typeof candidate === "string" && !validJson ? createInvalidStructuredOutputError(args.label, error) : error;
|
|
4944
2878
|
if (failure && typeof failure === "object")
|
|
4945
2879
|
Object.assign(failure, {
|
|
@@ -4959,7 +2893,7 @@ var require_dist2 = __commonJS({
|
|
|
4959
2893
|
if (typeof candidate === "string" && candidate.trim().length > 0) {
|
|
4960
2894
|
throw createInvalidStructuredOutputError("useReason output");
|
|
4961
2895
|
}
|
|
4962
|
-
throw new
|
|
2896
|
+
throw new import_errors42.ValidationError(
|
|
4963
2897
|
"MODEL_OUTPUT_SCHEMA",
|
|
4964
2898
|
"useReason first pass with interrupt must return a JSON object."
|
|
4965
2899
|
);
|
|
@@ -4968,7 +2902,7 @@ var require_dist2 = __commonJS({
|
|
|
4968
2902
|
let interruptRequired = true;
|
|
4969
2903
|
if (mode === "optional") {
|
|
4970
2904
|
if (candidate.decision !== "continue" && candidate.decision !== "interrupt") {
|
|
4971
|
-
throw new
|
|
2905
|
+
throw new import_errors42.ValidationError(
|
|
4972
2906
|
"MODEL_OUTPUT_SCHEMA",
|
|
4973
2907
|
'useReason optional interrupt first pass must include decision "continue" or "interrupt".'
|
|
4974
2908
|
);
|
|
@@ -4981,7 +2915,7 @@ var require_dist2 = __commonJS({
|
|
|
4981
2915
|
candidate.interrupt
|
|
4982
2916
|
].some((value) => value !== null && value !== void 0);
|
|
4983
2917
|
if (hasUnexpectedRequest) {
|
|
4984
|
-
throw new
|
|
2918
|
+
throw new import_errors42.ValidationError(
|
|
4985
2919
|
"MODEL_OUTPUT_SCHEMA",
|
|
4986
2920
|
'useReason optional interrupt first pass with decision "continue" must not include an interrupt request.'
|
|
4987
2921
|
);
|
|
@@ -5015,7 +2949,7 @@ var require_dist2 = __commonJS({
|
|
|
5015
2949
|
"useReason output"
|
|
5016
2950
|
);
|
|
5017
2951
|
} catch (error) {
|
|
5018
|
-
if (!(error instanceof
|
|
2952
|
+
if (!(error instanceof import_errors42.ValidationError)) throw error;
|
|
5019
2953
|
if (args.finishReason?.unified === "length") {
|
|
5020
2954
|
throw createTruncatedStructuredOutputError("useReason output", error);
|
|
5021
2955
|
}
|
|
@@ -5024,7 +2958,7 @@ var require_dist2 = __commonJS({
|
|
|
5024
2958
|
}
|
|
5025
2959
|
const explicitDraft = typeof candidate.draftText === "string" && candidate.draftText.length > 0 ? candidate.draftText : typeof candidate.text === "string" && candidate.text.length > 0 ? candidate.text : void 0;
|
|
5026
2960
|
if (!args.outputSchema && !explicitDraft) {
|
|
5027
|
-
throw new
|
|
2961
|
+
throw new import_errors42.ValidationError(
|
|
5028
2962
|
"MODEL_OUTPUT_SCHEMA",
|
|
5029
2963
|
"useReason first pass with interrupt requires `draftText` when outputSchema is not provided."
|
|
5030
2964
|
);
|
|
@@ -5290,7 +3224,7 @@ ${tryStringify2(args.response)}`
|
|
|
5290
3224
|
});
|
|
5291
3225
|
};
|
|
5292
3226
|
var import_core52 = require("@kortyx/core");
|
|
5293
|
-
var
|
|
3227
|
+
var import_errors52 = require("@kortyx/core/errors");
|
|
5294
3228
|
var import_zod52 = require("zod");
|
|
5295
3229
|
var record = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
5296
3230
|
function schemaPath(schema, path) {
|
|
@@ -5526,7 +3460,7 @@ ${tryStringify2(args.response)}`
|
|
|
5526
3460
|
const maxSteps = Math.max(1, useReasonArgs.toolExecution?.maxSteps ?? 3);
|
|
5527
3461
|
const missingReturnCorrections = useReasonArgs.toolExecution?.missingReturnCorrections ?? 0;
|
|
5528
3462
|
if (!Number.isSafeInteger(missingReturnCorrections) || missingReturnCorrections < 0)
|
|
5529
|
-
throw new
|
|
3463
|
+
throw new import_errors52.KortyxError(
|
|
5530
3464
|
"INVALID_REASON_CORRECTION_POLICY",
|
|
5531
3465
|
"Invalid missing return correction budget.",
|
|
5532
3466
|
{
|
|
@@ -5587,7 +3521,7 @@ ${tryStringify2(args.response)}`
|
|
|
5587
3521
|
let outputContractCorrections = checkpoint?.outputContractCorrections ?? 0;
|
|
5588
3522
|
const correctionExhausted = () => {
|
|
5589
3523
|
const message = `useReason could not complete output-contract correction within toolExecution.maxSteps (${maxSteps}). Select exactly one terminal return contract; multiple emit contracts are allowed. Increase maxSteps to allow correction and output generation.`;
|
|
5590
|
-
return new
|
|
3524
|
+
return new import_errors52.KortyxError(
|
|
5591
3525
|
"REASON_OUTPUT_CONTRACT_CORRECTION_EXHAUSTED",
|
|
5592
3526
|
message,
|
|
5593
3527
|
{
|
|
@@ -5684,7 +3618,7 @@ ${tryStringify2(args.response)}`
|
|
|
5684
3618
|
aggregatedWarnings = mergeWarnings(aggregatedWarnings, step.warnings);
|
|
5685
3619
|
const toolCalls = step.toolCalls ?? [];
|
|
5686
3620
|
if (missingReturnAttempts > 0 && toolCalls.some((call) => !outputByToolName.has(call.name)))
|
|
5687
|
-
throw new
|
|
3621
|
+
throw new import_errors52.KortyxError(
|
|
5688
3622
|
"REASON_OUTPUT_CORRECTION_TOOL_DISALLOWED",
|
|
5689
3623
|
"A return correction requested a non-output tool.",
|
|
5690
3624
|
{
|
|
@@ -5727,7 +3661,7 @@ ${tryStringify2(args.response)}`
|
|
|
5727
3661
|
save();
|
|
5728
3662
|
continue;
|
|
5729
3663
|
}
|
|
5730
|
-
throw new
|
|
3664
|
+
throw new import_errors52.KortyxError(
|
|
5731
3665
|
"REASON_TERMINAL_OUTPUT_MISSING",
|
|
5732
3666
|
"useReason outputs.return requires one terminal output contract call.",
|
|
5733
3667
|
{
|
|
@@ -5861,7 +3795,7 @@ ${tryStringify2(args.response)}`
|
|
|
5861
3795
|
);
|
|
5862
3796
|
if (steps.length >= maxSteps) {
|
|
5863
3797
|
const message = `useReason output generation requires another model pass within toolExecution.maxSteps (${maxSteps}). Increase maxSteps to allow outputs to use completed tool and interrupt results.`;
|
|
5864
|
-
throw new
|
|
3798
|
+
throw new import_errors52.ProviderRequestError(message, {
|
|
5865
3799
|
code: "REASON_OUTPUT_BUDGET_EXHAUSTED",
|
|
5866
3800
|
operation: "useReason",
|
|
5867
3801
|
safeMessage: message,
|
|
@@ -6056,7 +3990,7 @@ ${tryStringify2(args.response)}`
|
|
|
6056
3990
|
`useReason interrupt contract "${resolved.name}" request`
|
|
6057
3991
|
);
|
|
6058
3992
|
} catch (error) {
|
|
6059
|
-
if (!(error instanceof
|
|
3993
|
+
if (!(error instanceof import_errors52.ValidationError)) throw error;
|
|
6060
3994
|
const result3 = {
|
|
6061
3995
|
toolCallId: toolCall.id,
|
|
6062
3996
|
name: toolCall.name,
|
|
@@ -6266,12 +4200,12 @@ ${tryStringify2(args.response)}`
|
|
|
6266
4200
|
}
|
|
6267
4201
|
} catch (error) {
|
|
6268
4202
|
(0, import_core52.throwIfExecutionAborted)(abortSignal);
|
|
6269
|
-
if ((0,
|
|
4203
|
+
if ((0, import_errors52.isControlFlowError)(error)) throw error;
|
|
6270
4204
|
const result2 = {
|
|
6271
4205
|
toolCallId: toolCall.id,
|
|
6272
4206
|
name: tool.name,
|
|
6273
|
-
content: (0,
|
|
6274
|
-
failure: (0,
|
|
4207
|
+
content: (0, import_errors52.serializeFailure)(error).message,
|
|
4208
|
+
failure: (0, import_errors52.serializeFailure)(error),
|
|
6275
4209
|
isError: true
|
|
6276
4210
|
};
|
|
6277
4211
|
toolResults.push(result2);
|
|
@@ -6868,7 +4802,7 @@ ${tryStringify2(args.response)}`
|
|
|
6868
4802
|
return [getValue(), setValue];
|
|
6869
4803
|
}
|
|
6870
4804
|
var import_core7 = require("@kortyx/core");
|
|
6871
|
-
var
|
|
4805
|
+
var import_errors62 = require("@kortyx/core/errors");
|
|
6872
4806
|
var ParallelChildWaiting = class extends Error {
|
|
6873
4807
|
name = "ParallelChildWaiting";
|
|
6874
4808
|
};
|
|
@@ -6882,14 +4816,14 @@ ${tryStringify2(args.response)}`
|
|
|
6882
4816
|
);
|
|
6883
4817
|
this.results = results;
|
|
6884
4818
|
this.name = "ParallelError";
|
|
6885
|
-
this.failure = (0,
|
|
4819
|
+
this.failure = (0, import_errors62.serializeFailure)({
|
|
6886
4820
|
version: 1,
|
|
6887
4821
|
code: this.code,
|
|
6888
4822
|
category: "workflow",
|
|
6889
4823
|
message: this.message,
|
|
6890
4824
|
retryable: null,
|
|
6891
4825
|
results: results.map(
|
|
6892
|
-
(result) => result.status === "fulfilled" ? { status: "fulfilled" } : { status: "rejected", failure: (0,
|
|
4826
|
+
(result) => result.status === "fulfilled" ? { status: "fulfilled" } : { status: "rejected", failure: (0, import_errors62.serializeFailure)(result.reason) }
|
|
6893
4827
|
)
|
|
6894
4828
|
});
|
|
6895
4829
|
}
|
|
@@ -6897,7 +4831,7 @@ ${tryStringify2(args.response)}`
|
|
|
6897
4831
|
code = "PARALLEL_FAILED";
|
|
6898
4832
|
failure;
|
|
6899
4833
|
toJSON() {
|
|
6900
|
-
return (0,
|
|
4834
|
+
return (0, import_errors62.serializeFailure)(this);
|
|
6901
4835
|
}
|
|
6902
4836
|
};
|
|
6903
4837
|
function registerWorkflowTask(id3) {
|
|
@@ -7072,11 +5006,11 @@ ${tryStringify2(args.response)}`
|
|
|
7072
5006
|
}
|
|
7073
5007
|
var import_node_crypto62 = require("crypto");
|
|
7074
5008
|
var import_core8 = require("@kortyx/core");
|
|
7075
|
-
var
|
|
7076
|
-
var WorkflowCallError = class extends
|
|
5009
|
+
var import_errors72 = require("@kortyx/core/errors");
|
|
5010
|
+
var WorkflowCallError = class extends import_errors72.KortyxError {
|
|
7077
5011
|
constructor(workflow, message, cause, failure) {
|
|
7078
|
-
const detail = (0,
|
|
7079
|
-
failure ?? cause ?? new
|
|
5012
|
+
const detail = (0, import_errors72.serializeFailure)(
|
|
5013
|
+
failure ?? cause ?? new import_errors72.WorkflowContractError(message)
|
|
7080
5014
|
);
|
|
7081
5015
|
super(detail.code, `Workflow '${workflow}': ${message}`, {
|
|
7082
5016
|
...detail,
|
|
@@ -7097,12 +5031,12 @@ ${tryStringify2(args.response)}`
|
|
|
7097
5031
|
if (v === null || typeof v === "string" || typeof v === "boolean") return v;
|
|
7098
5032
|
if (typeof v === "number" && Number.isFinite(v)) return v;
|
|
7099
5033
|
if (typeof v !== "object" || !v || seen.has(v)) {
|
|
7100
|
-
throw new
|
|
5034
|
+
throw new import_errors72.WorkflowContractError(
|
|
7101
5035
|
"Workflow input and output must be JSON serializable."
|
|
7102
5036
|
);
|
|
7103
5037
|
}
|
|
7104
5038
|
if (!Array.isArray(v) && Object.getPrototypeOf(v) !== Object.prototype) {
|
|
7105
|
-
throw new
|
|
5039
|
+
throw new import_errors72.WorkflowContractError(
|
|
7106
5040
|
"Workflow input and output must contain plain JSON objects."
|
|
7107
5041
|
);
|
|
7108
5042
|
}
|
|
@@ -7123,7 +5057,7 @@ ${tryStringify2(args.response)}`
|
|
|
7123
5057
|
useWorkflow: (args) => {
|
|
7124
5058
|
const workflow = workflows[args.workflow];
|
|
7125
5059
|
if (!workflow || workflow.id !== args.workflow)
|
|
7126
|
-
throw new
|
|
5060
|
+
throw new import_errors72.WorkflowContractError(
|
|
7127
5061
|
"Workflow registry keys must match definition ids."
|
|
7128
5062
|
);
|
|
7129
5063
|
return useWorkflow({ ...args, workflow });
|
|
@@ -7149,23 +5083,23 @@ ${tryStringify2(args.response)}`
|
|
|
7149
5083
|
(0, import_core8.throwIfExecutionAborted)(ctx.node.abortSignal);
|
|
7150
5084
|
const service = ctx.node.callWorkflow;
|
|
7151
5085
|
if (!service)
|
|
7152
|
-
throw new
|
|
5086
|
+
throw new import_errors72.WorkflowContractError(
|
|
7153
5087
|
"useWorkflow requires an agent workflow registry."
|
|
7154
5088
|
);
|
|
7155
5089
|
if (!args.id.trim())
|
|
7156
|
-
throw new
|
|
5090
|
+
throw new import_errors72.WorkflowContractError(
|
|
7157
5091
|
"useWorkflow requires a stable, nonempty id."
|
|
7158
5092
|
);
|
|
7159
5093
|
if (ctx.workflowCallActive && !task.group)
|
|
7160
|
-
throw new
|
|
5094
|
+
throw new import_errors72.WorkflowContractError(
|
|
7161
5095
|
"Concurrent workflow calls require parallel([...]); otherwise await each call."
|
|
7162
5096
|
);
|
|
7163
5097
|
if (ctx.workflowCallIds.has(args.id))
|
|
7164
|
-
throw new
|
|
5098
|
+
throw new import_errors72.WorkflowContractError(
|
|
7165
5099
|
`Duplicate workflow call id '${args.id}' in one node activation.`
|
|
7166
5100
|
);
|
|
7167
5101
|
if (ctx.workflowCallIds.size >= 64)
|
|
7168
|
-
throw new
|
|
5102
|
+
throw new import_errors72.WorkflowContractError(
|
|
7169
5103
|
"A node may call at most 64 child workflows per activation."
|
|
7170
5104
|
);
|
|
7171
5105
|
ctx.workflowCallIds.add(args.id);
|
|
@@ -7246,7 +5180,7 @@ ${tryStringify2(args.response)}`
|
|
|
7246
5180
|
workflow,
|
|
7247
5181
|
record2.error ?? "child failed",
|
|
7248
5182
|
void 0,
|
|
7249
|
-
record2.failure ?? (0,
|
|
5183
|
+
record2.failure ?? (0, import_errors72.serializeFailure)(void 0)
|
|
7250
5184
|
);
|
|
7251
5185
|
if (task.group && record2.status === "interrupted" && record2.interrupts.at(-1)?.response === void 0)
|
|
7252
5186
|
throw new ParallelChildWaiting();
|
|
@@ -7254,7 +5188,7 @@ ${tryStringify2(args.response)}`
|
|
|
7254
5188
|
let outcome;
|
|
7255
5189
|
try {
|
|
7256
5190
|
if (Array.isArray(record2.snapshot) && record2.snapshot.length !== 1)
|
|
7257
|
-
throw new
|
|
5191
|
+
throw new import_errors72.WorkflowContractError("Invalid child workflow snapshot.");
|
|
7258
5192
|
const snapshot = Array.isArray(record2.snapshot) ? record2.snapshot[0] : record2.snapshot;
|
|
7259
5193
|
report(
|
|
7260
5194
|
snapshot ? "resumed" : "started",
|
|
@@ -7285,7 +5219,7 @@ ${tryStringify2(args.response)}`
|
|
|
7285
5219
|
throw error;
|
|
7286
5220
|
}
|
|
7287
5221
|
record2.status = "failed";
|
|
7288
|
-
record2.failure = (0,
|
|
5222
|
+
record2.failure = (0, import_errors72.serializeFailure)(error);
|
|
7289
5223
|
record2.error = record2.failure.message;
|
|
7290
5224
|
report("failed", { error: "Child workflow failed" });
|
|
7291
5225
|
throw new WorkflowCallError(
|
|
@@ -7349,6 +5283,7 @@ __export(index_exports, {
|
|
|
7349
5283
|
ChatLifecycleHookError: () => ChatLifecycleHookError,
|
|
7350
5284
|
EvalConfigurationError: () => EvalConfigurationError,
|
|
7351
5285
|
EvalEvidencePolicySchema: () => EvalEvidencePolicySchema,
|
|
5286
|
+
EvalExecutionSettingsSchema: () => EvalExecutionSettingsSchema,
|
|
7352
5287
|
EvalIssueSchema: () => EvalIssueSchema,
|
|
7353
5288
|
EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
|
|
7354
5289
|
EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
|
|
@@ -7356,10 +5291,14 @@ __export(index_exports, {
|
|
|
7356
5291
|
EvalProgressSchema: () => EvalProgressSchema,
|
|
7357
5292
|
EvalRemoteRunRequestSchema: () => EvalRemoteRunRequestSchema,
|
|
7358
5293
|
EvalRunResultSchema: () => EvalRunResultSchema,
|
|
5294
|
+
EvalSettingsResponseSchema: () => EvalSettingsResponseSchema,
|
|
5295
|
+
EvalSettingsSchema: () => EvalSettingsSchema,
|
|
7359
5296
|
EvalSuiteSchema: () => EvalSuiteSchema,
|
|
5297
|
+
EvalTerminationSchema: () => EvalTerminationSchema,
|
|
7360
5298
|
EvalWireEventSchema: () => EvalWireEventSchema,
|
|
7361
5299
|
EvaluationMetadataSchema: () => EvaluationMetadataSchema,
|
|
7362
5300
|
ExecutionRequestError: () => ExecutionRequestError,
|
|
5301
|
+
ResolvedEvalSettingsSchema: () => ResolvedEvalSettingsSchema,
|
|
7363
5302
|
StudioEvalJudgeRequestSchema: () => StudioEvalJudgeRequestSchema,
|
|
7364
5303
|
StudioEvalJudgeResponseSchema: () => StudioEvalJudgeResponseSchema,
|
|
7365
5304
|
StudioEvalStartRequestSchema: () => StudioEvalStartRequestSchema,
|
|
@@ -7390,6 +5329,7 @@ __export(index_exports, {
|
|
|
7390
5329
|
parseResumeMeta: () => parseResumeMeta,
|
|
7391
5330
|
projectWorkflowTopology: () => projectWorkflowTopology,
|
|
7392
5331
|
readRequestJson: () => readRequestJson,
|
|
5332
|
+
resolveEvalSettings: () => resolveEvalSettings,
|
|
7393
5333
|
streamChat: () => streamChat,
|
|
7394
5334
|
streamChatFromRoute: () => streamChatFromRoute,
|
|
7395
5335
|
transformGraphStreamForUI: () => transformGraphStreamForUI,
|
|
@@ -7445,6 +5385,7 @@ var ExecutionRequestError = class extends import_errors2.KortyxError {
|
|
|
7445
5385
|
var import_node_stream = require("stream");
|
|
7446
5386
|
var import_core2 = require("@kortyx/core");
|
|
7447
5387
|
var import_errors4 = require("@kortyx/core/errors");
|
|
5388
|
+
var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
|
|
7448
5389
|
var import_internal = __toESM(require_internal());
|
|
7449
5390
|
var import_runtime = require("@kortyx/runtime");
|
|
7450
5391
|
var import_stream = require("@kortyx/stream");
|
|
@@ -7906,7 +5847,7 @@ var import_node = require("@kortyx/usage-telemetry/node");
|
|
|
7906
5847
|
// package.json
|
|
7907
5848
|
var package_default = {
|
|
7908
5849
|
name: "@kortyx/agent",
|
|
7909
|
-
version: "0.
|
|
5850
|
+
version: "0.39.0",
|
|
7910
5851
|
private: false,
|
|
7911
5852
|
description: "Agent composition utilities for Kortyx.",
|
|
7912
5853
|
keywords: [
|
|
@@ -7964,7 +5905,8 @@ var package_default = {
|
|
|
7964
5905
|
"@langchain/langgraph": "catalog:",
|
|
7965
5906
|
zod: "catalog:",
|
|
7966
5907
|
"@kortyx/prompts": "workspace:*",
|
|
7967
|
-
"@kortyx/usage-telemetry": "workspace:*"
|
|
5908
|
+
"@kortyx/usage-telemetry": "workspace:*",
|
|
5909
|
+
"@kortyx/telemetry-contracts": "workspace:*"
|
|
7968
5910
|
},
|
|
7969
5911
|
devDependencies: {
|
|
7970
5912
|
"@kortyx/hooks": "workspace:*",
|
|
@@ -8170,7 +6112,8 @@ async function orchestrateGraphStream({
|
|
|
8170
6112
|
isRecord3(config.context) ? { userId: config.context.userId, tenantId: config.context.tenantId } : {}
|
|
8171
6113
|
);
|
|
8172
6114
|
const telemetryConfig = isRecord3(config.telemetry) ? config.telemetry : {};
|
|
8173
|
-
const errorChunk = (error, existingDiagnosticId) => {
|
|
6115
|
+
const errorChunk = (error, existingDiagnosticId, original = true) => {
|
|
6116
|
+
if (original) (0, import_eval_attribution2.observeEvalError)(error);
|
|
8174
6117
|
const failure = (0, import_errors4.serializeFailure)(error);
|
|
8175
6118
|
let diagnosticId = existingDiagnosticId;
|
|
8176
6119
|
if (!diagnosticId) {
|
|
@@ -8457,7 +6400,8 @@ async function orchestrateGraphStream({
|
|
|
8457
6400
|
write(
|
|
8458
6401
|
errorChunk(
|
|
8459
6402
|
outcomeError,
|
|
8460
|
-
payload?.diagnosticId
|
|
6403
|
+
payload?.diagnosticId,
|
|
6404
|
+
false
|
|
8461
6405
|
)
|
|
8462
6406
|
);
|
|
8463
6407
|
write({ type: "done" });
|
|
@@ -10195,7 +8139,7 @@ function createInterruptDiscovery(store) {
|
|
|
10195
8139
|
|
|
10196
8140
|
// src/telemetry/workflow-call-branch.ts
|
|
10197
8141
|
var import_node_crypto3 = require("crypto");
|
|
10198
|
-
var import_hooks = __toESM(
|
|
8142
|
+
var import_hooks = __toESM(require_dist());
|
|
10199
8143
|
function restoreWorkflowCallBranch(requests, telemetry, sourceRunId) {
|
|
10200
8144
|
const branchId = (0, import_node_crypto3.randomUUID)();
|
|
10201
8145
|
const emitted = /* @__PURE__ */ new Set();
|
|
@@ -10888,7 +8832,8 @@ function createAgent(args) {
|
|
|
10888
8832
|
// src/evals/contracts.ts
|
|
10889
8833
|
var import_errors8 = require("@kortyx/core/errors");
|
|
10890
8834
|
var import_browser2 = require("@kortyx/prompts/browser");
|
|
10891
|
-
var
|
|
8835
|
+
var import_telemetry_contracts = require("@kortyx/telemetry-contracts");
|
|
8836
|
+
var import_zod9 = require("zod");
|
|
10892
8837
|
|
|
10893
8838
|
// src/evals/evidence-policy.ts
|
|
10894
8839
|
var import_zod7 = require("zod");
|
|
@@ -10928,30 +8873,126 @@ var EvalEvidencePolicySchema = import_zod7.z.object({
|
|
|
10928
8873
|
]).optional()
|
|
10929
8874
|
}).strict();
|
|
10930
8875
|
|
|
8876
|
+
// src/evals/settings.ts
|
|
8877
|
+
var import_zod8 = require("zod");
|
|
8878
|
+
var timeout = import_zod8.z.number().int().min(1).max(2147483647);
|
|
8879
|
+
var EvalSettingsSchema = import_zod8.z.object({
|
|
8880
|
+
repetitions: import_zod8.z.number().int().min(1).max(20).optional(),
|
|
8881
|
+
concurrency: import_zod8.z.number().int().min(1).max(20).optional(),
|
|
8882
|
+
caseTimeoutMs: timeout.optional(),
|
|
8883
|
+
runTimeoutMs: timeout.optional(),
|
|
8884
|
+
judge: import_zod8.z.enum(["studio", "app"]).optional()
|
|
8885
|
+
}).strict();
|
|
8886
|
+
var EvalExecutionSettingsSchema = import_zod8.z.object({
|
|
8887
|
+
defaults: EvalSettingsSchema.omit({ runTimeoutMs: true, judge: true }),
|
|
8888
|
+
limits: import_zod8.z.object({
|
|
8889
|
+
concurrency: import_zod8.z.number().int().positive().max(20).optional(),
|
|
8890
|
+
caseTimeoutMs: timeout.optional()
|
|
8891
|
+
}).strict(),
|
|
8892
|
+
overrides: import_zod8.z.array(import_zod8.z.enum(["repetitions", "concurrency", "caseTimeoutMs"]))
|
|
8893
|
+
}).strict();
|
|
8894
|
+
var ResolvedEvalSettingsSchema = import_zod8.z.object({
|
|
8895
|
+
values: EvalSettingsSchema.required(),
|
|
8896
|
+
sources: import_zod8.z.record(
|
|
8897
|
+
import_zod8.z.enum([
|
|
8898
|
+
"repetitions",
|
|
8899
|
+
"concurrency",
|
|
8900
|
+
"caseTimeoutMs",
|
|
8901
|
+
"runTimeoutMs",
|
|
8902
|
+
"judge"
|
|
8903
|
+
]),
|
|
8904
|
+
import_zod8.z.enum(["run", "studio", "sdk", "sdk-default", "studio-default"])
|
|
8905
|
+
)
|
|
8906
|
+
}).strict();
|
|
8907
|
+
var EvalSettingsResponseSchema = import_zod8.z.object({
|
|
8908
|
+
defaults: EvalSettingsSchema,
|
|
8909
|
+
installation: EvalSettingsSchema,
|
|
8910
|
+
maxRunTimeoutMs: timeout,
|
|
8911
|
+
canEdit: import_zod8.z.boolean()
|
|
8912
|
+
});
|
|
8913
|
+
var EvalTerminationSchema = import_zod8.z.object({
|
|
8914
|
+
code: import_zod8.z.enum([
|
|
8915
|
+
"EVAL_RUN_TIMEOUT",
|
|
8916
|
+
"EVAL_CANCELLED",
|
|
8917
|
+
"EVAL_LEASE_LOST",
|
|
8918
|
+
"EVAL_EXECUTION_UNKNOWN",
|
|
8919
|
+
"EVAL_EXECUTOR_FAILED"
|
|
8920
|
+
]),
|
|
8921
|
+
message: import_zod8.z.string(),
|
|
8922
|
+
timeoutMs: timeout.optional(),
|
|
8923
|
+
phase: import_zod8.z.string().optional(),
|
|
8924
|
+
cleanup: import_zod8.z.enum(["signalled", "unknown"]).optional(),
|
|
8925
|
+
diagnosticId: import_zod8.z.string().uuid().optional()
|
|
8926
|
+
}).strict();
|
|
8927
|
+
function resolveEvalSettings(overrides = {}, studio = {}, execution, installation = {}, maxRunTimeoutMs = 2147483647) {
|
|
8928
|
+
const values = {
|
|
8929
|
+
repetitions: 1,
|
|
8930
|
+
concurrency: 1,
|
|
8931
|
+
caseTimeoutMs: 6e4,
|
|
8932
|
+
runTimeoutMs: 30 * 6e4,
|
|
8933
|
+
judge: "studio"
|
|
8934
|
+
};
|
|
8935
|
+
const sources = {
|
|
8936
|
+
repetitions: "sdk-default",
|
|
8937
|
+
concurrency: "sdk-default",
|
|
8938
|
+
caseTimeoutMs: "sdk-default",
|
|
8939
|
+
runTimeoutMs: "studio-default",
|
|
8940
|
+
judge: "studio-default"
|
|
8941
|
+
};
|
|
8942
|
+
for (const [input, source] of [
|
|
8943
|
+
[execution?.defaults ?? {}, "sdk"],
|
|
8944
|
+
[installation, "studio"],
|
|
8945
|
+
[studio, "studio"],
|
|
8946
|
+
[overrides, "run"]
|
|
8947
|
+
]) {
|
|
8948
|
+
const parsed = EvalSettingsSchema.parse(input);
|
|
8949
|
+
for (const key of Object.keys(parsed)) {
|
|
8950
|
+
if (parsed[key] !== void 0) {
|
|
8951
|
+
Object.assign(values, { [key]: parsed[key] });
|
|
8952
|
+
sources[key] = source;
|
|
8953
|
+
}
|
|
8954
|
+
}
|
|
8955
|
+
}
|
|
8956
|
+
if (values.runTimeoutMs > maxRunTimeoutMs)
|
|
8957
|
+
throw new Error(
|
|
8958
|
+
`Evaluation timeout exceeds the administrator maximum of ${maxRunTimeoutMs} ms.`
|
|
8959
|
+
);
|
|
8960
|
+
for (const key of ["concurrency", "caseTimeoutMs"]) {
|
|
8961
|
+
const max = execution?.limits[key];
|
|
8962
|
+
if (max !== void 0 && values[key] > max)
|
|
8963
|
+
throw new Error(`${key} exceeds the application maximum of ${max}.`);
|
|
8964
|
+
}
|
|
8965
|
+
if (!execution?.overrides.includes("caseTimeoutMs") && (overrides.caseTimeoutMs !== void 0 || studio.caseTimeoutMs !== void 0 || installation.caseTimeoutMs !== void 0))
|
|
8966
|
+
throw new Error(
|
|
8967
|
+
"Update the consumer SDK to support case timeout overrides."
|
|
8968
|
+
);
|
|
8969
|
+
return { values, sources };
|
|
8970
|
+
}
|
|
8971
|
+
|
|
10931
8972
|
// src/evals/contracts.ts
|
|
10932
|
-
var id2 =
|
|
10933
|
-
var failureDescriptor =
|
|
8973
|
+
var id2 = import_zod9.z.string().trim().min(1).max(256);
|
|
8974
|
+
var failureDescriptor = import_zod9.z.custom(import_errors8.isFailureDescriptor).transform(
|
|
10934
8975
|
(value) => JSON.parse(JSON.stringify((0, import_errors8.serializeFailure)(value)))
|
|
10935
8976
|
);
|
|
10936
|
-
var handler =
|
|
10937
|
-
var EvalResumeResponseSchema =
|
|
10938
|
-
|
|
10939
|
-
|
|
10940
|
-
|
|
10941
|
-
|
|
8977
|
+
var handler = import_zod9.z.object({ using: id2, params: import_zod9.z.json().optional() }).strict();
|
|
8978
|
+
var EvalResumeResponseSchema = import_zod9.z.discriminatedUnion("type", [
|
|
8979
|
+
import_zod9.z.object({ type: import_zod9.z.literal("text"), text: import_zod9.z.string() }).strict(),
|
|
8980
|
+
import_zod9.z.object({ type: import_zod9.z.literal("select"), ids: import_zod9.z.array(import_zod9.z.string()) }).strict(),
|
|
8981
|
+
import_zod9.z.object({ type: import_zod9.z.literal("value"), value: import_zod9.z.json() }).strict(),
|
|
8982
|
+
import_zod9.z.object({ type: import_zod9.z.literal("cancel") }).strict()
|
|
10942
8983
|
]);
|
|
10943
|
-
var expectation =
|
|
10944
|
-
type:
|
|
8984
|
+
var expectation = import_zod9.z.object({
|
|
8985
|
+
type: import_zod9.z.enum(["answer", "interrupt"]),
|
|
10945
8986
|
schemaId: id2.optional(),
|
|
10946
8987
|
schemaVersion: id2.optional(),
|
|
10947
|
-
outputs:
|
|
10948
|
-
criteria:
|
|
10949
|
-
|
|
10950
|
-
|
|
10951
|
-
|
|
8988
|
+
outputs: import_zod9.z.array(import_zod9.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
|
|
8989
|
+
criteria: import_zod9.z.array(
|
|
8990
|
+
import_zod9.z.union([
|
|
8991
|
+
import_zod9.z.string().trim().min(1),
|
|
8992
|
+
import_zod9.z.object({ id: id2, text: import_zod9.z.string().trim().min(1) }).strict()
|
|
10952
8993
|
])
|
|
10953
8994
|
).optional(),
|
|
10954
|
-
reference:
|
|
8995
|
+
reference: import_zod9.z.json().optional()
|
|
10955
8996
|
}).strict().superRefine((value, ctx) => {
|
|
10956
8997
|
if (value.type === "answer" && (value.schemaId || value.schemaVersion))
|
|
10957
8998
|
ctx.addIssue({
|
|
@@ -10967,21 +9008,21 @@ var expectation = import_zod8.z.object({
|
|
|
10967
9008
|
message: "Criterion IDs must be unique within a step."
|
|
10968
9009
|
});
|
|
10969
9010
|
});
|
|
10970
|
-
var EvalSuiteSchema =
|
|
9011
|
+
var EvalSuiteSchema = import_zod9.z.object({
|
|
10971
9012
|
evidence: EvalEvidencePolicySchema.optional(),
|
|
10972
9013
|
id: id2,
|
|
10973
|
-
name:
|
|
10974
|
-
cases:
|
|
10975
|
-
|
|
9014
|
+
name: import_zod9.z.string().optional(),
|
|
9015
|
+
cases: import_zod9.z.array(
|
|
9016
|
+
import_zod9.z.object({
|
|
10976
9017
|
id: id2,
|
|
10977
|
-
name:
|
|
10978
|
-
params:
|
|
9018
|
+
name: import_zod9.z.string().optional(),
|
|
9019
|
+
params: import_zod9.z.json().optional(),
|
|
10979
9020
|
workflowId: id2.optional(),
|
|
10980
|
-
steps:
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
|
|
10984
|
-
resume:
|
|
9021
|
+
steps: import_zod9.z.array(
|
|
9022
|
+
import_zod9.z.union([
|
|
9023
|
+
import_zod9.z.object({ message: import_zod9.z.string().min(1), expect: expectation }).strict(),
|
|
9024
|
+
import_zod9.z.object({
|
|
9025
|
+
resume: import_zod9.z.union([EvalResumeResponseSchema, handler]),
|
|
10985
9026
|
expect: expectation
|
|
10986
9027
|
}).strict()
|
|
10987
9028
|
])
|
|
@@ -10989,44 +9030,44 @@ var EvalSuiteSchema = import_zod8.z.object({
|
|
|
10989
9030
|
}).strict()
|
|
10990
9031
|
).min(1)
|
|
10991
9032
|
}).strict();
|
|
10992
|
-
var EvalVerdictSchema =
|
|
10993
|
-
passed:
|
|
10994
|
-
reason:
|
|
10995
|
-
evidence:
|
|
9033
|
+
var EvalVerdictSchema = import_zod9.z.object({
|
|
9034
|
+
passed: import_zod9.z.boolean(),
|
|
9035
|
+
reason: import_zod9.z.string().trim().min(1),
|
|
9036
|
+
evidence: import_zod9.z.array(import_zod9.z.string().min(1))
|
|
10996
9037
|
}).strict();
|
|
10997
|
-
var EvalObservationSchema =
|
|
9038
|
+
var EvalObservationSchema = import_zod9.z.object({
|
|
10998
9039
|
failure: failureDescriptor.optional(),
|
|
10999
|
-
diagnosticId:
|
|
11000
|
-
promptUsage:
|
|
11001
|
-
|
|
11002
|
-
id:
|
|
11003
|
-
version:
|
|
11004
|
-
hash:
|
|
11005
|
-
environment:
|
|
11006
|
-
snapshotRevision:
|
|
11007
|
-
model:
|
|
9040
|
+
diagnosticId: import_zod9.z.uuid().optional(),
|
|
9041
|
+
promptUsage: import_zod9.z.array(
|
|
9042
|
+
import_zod9.z.object({
|
|
9043
|
+
id: import_zod9.z.string(),
|
|
9044
|
+
version: import_zod9.z.number().int().positive(),
|
|
9045
|
+
hash: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
|
|
9046
|
+
environment: import_zod9.z.string(),
|
|
9047
|
+
snapshotRevision: import_zod9.z.string(),
|
|
9048
|
+
model: import_zod9.z.string().optional()
|
|
11008
9049
|
}).strict()
|
|
11009
9050
|
).optional(),
|
|
11010
|
-
type:
|
|
11011
|
-
text:
|
|
11012
|
-
structured:
|
|
11013
|
-
events:
|
|
11014
|
-
runId:
|
|
11015
|
-
checkpointId:
|
|
11016
|
-
interrupt:
|
|
11017
|
-
requestId:
|
|
11018
|
-
kind:
|
|
11019
|
-
question:
|
|
11020
|
-
schemaId:
|
|
11021
|
-
schemaVersion:
|
|
11022
|
-
options:
|
|
11023
|
-
|
|
11024
|
-
id:
|
|
11025
|
-
label:
|
|
11026
|
-
description:
|
|
9051
|
+
type: import_zod9.z.enum(["answer", "interrupt", "error", "cancelled"]),
|
|
9052
|
+
text: import_zod9.z.string(),
|
|
9053
|
+
structured: import_zod9.z.array(import_zod9.z.json()),
|
|
9054
|
+
events: import_zod9.z.array(import_zod9.z.json()).optional(),
|
|
9055
|
+
runId: import_zod9.z.string().optional(),
|
|
9056
|
+
checkpointId: import_zod9.z.string().optional(),
|
|
9057
|
+
interrupt: import_zod9.z.object({
|
|
9058
|
+
requestId: import_zod9.z.string(),
|
|
9059
|
+
kind: import_zod9.z.enum(["text", "choice", "multi-choice", "custom"]),
|
|
9060
|
+
question: import_zod9.z.string().optional(),
|
|
9061
|
+
schemaId: import_zod9.z.string().optional(),
|
|
9062
|
+
schemaVersion: import_zod9.z.string().optional(),
|
|
9063
|
+
options: import_zod9.z.array(
|
|
9064
|
+
import_zod9.z.object({
|
|
9065
|
+
id: import_zod9.z.string(),
|
|
9066
|
+
label: import_zod9.z.string(),
|
|
9067
|
+
description: import_zod9.z.string().optional()
|
|
11027
9068
|
}).strict()
|
|
11028
9069
|
),
|
|
11029
|
-
request:
|
|
9070
|
+
request: import_zod9.z.json().optional()
|
|
11030
9071
|
}).strict().optional()
|
|
11031
9072
|
}).strict().superRefine((value, ctx) => {
|
|
11032
9073
|
if (value.type === "interrupt" !== Boolean(value.interrupt))
|
|
@@ -11035,11 +9076,16 @@ var EvalObservationSchema = import_zod8.z.object({
|
|
|
11035
9076
|
message: "Interrupt observations require exactly one interrupt."
|
|
11036
9077
|
});
|
|
11037
9078
|
});
|
|
11038
|
-
var status =
|
|
11039
|
-
var EvalIssueSchema =
|
|
9079
|
+
var status = import_zod9.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
|
|
9080
|
+
var EvalIssueSchema = import_zod9.z.object({
|
|
9081
|
+
exception: import_telemetry_contracts.DiagnosticContentSchema.optional(),
|
|
9082
|
+
details: import_zod9.z.object({
|
|
9083
|
+
timeoutMs: import_zod9.z.number().positive().optional(),
|
|
9084
|
+
httpStatus: import_zod9.z.number().int().min(100).max(599).optional()
|
|
9085
|
+
}).strict().optional(),
|
|
11040
9086
|
failure: failureDescriptor.optional(),
|
|
11041
|
-
diagnosticId:
|
|
11042
|
-
phase:
|
|
9087
|
+
diagnosticId: import_zod9.z.uuid().optional(),
|
|
9088
|
+
phase: import_zod9.z.enum([
|
|
11043
9089
|
"params",
|
|
11044
9090
|
"setup",
|
|
11045
9091
|
"execute",
|
|
@@ -11050,196 +9096,200 @@ var EvalIssueSchema = import_zod8.z.object({
|
|
|
11050
9096
|
"cleanup",
|
|
11051
9097
|
"reporting"
|
|
11052
9098
|
]),
|
|
11053
|
-
code:
|
|
11054
|
-
message:
|
|
9099
|
+
code: import_zod9.z.string(),
|
|
9100
|
+
message: import_zod9.z.string()
|
|
11055
9101
|
}).strict();
|
|
11056
9102
|
var issue = EvalIssueSchema;
|
|
11057
|
-
var EvalJudgeUsageSchema =
|
|
11058
|
-
provider:
|
|
11059
|
-
model:
|
|
11060
|
-
occurredAt:
|
|
11061
|
-
usage:
|
|
11062
|
-
input:
|
|
11063
|
-
output:
|
|
11064
|
-
total:
|
|
11065
|
-
reasoning:
|
|
11066
|
-
cacheRead:
|
|
11067
|
-
cacheWrite:
|
|
11068
|
-
cacheWrite1h:
|
|
11069
|
-
outputIncludesReasoning:
|
|
11070
|
-
inputIncludesCacheRead:
|
|
11071
|
-
inputIncludesCacheWrite:
|
|
9103
|
+
var EvalJudgeUsageSchema = import_zod9.z.object({
|
|
9104
|
+
provider: import_zod9.z.string().min(1),
|
|
9105
|
+
model: import_zod9.z.string().min(1),
|
|
9106
|
+
occurredAt: import_zod9.z.iso.datetime(),
|
|
9107
|
+
usage: import_zod9.z.object({
|
|
9108
|
+
input: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9109
|
+
output: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9110
|
+
total: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9111
|
+
reasoning: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9112
|
+
cacheRead: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9113
|
+
cacheWrite: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9114
|
+
cacheWrite1h: import_zod9.z.number().finite().nonnegative().optional(),
|
|
9115
|
+
outputIncludesReasoning: import_zod9.z.boolean().optional(),
|
|
9116
|
+
inputIncludesCacheRead: import_zod9.z.boolean().optional(),
|
|
9117
|
+
inputIncludesCacheWrite: import_zod9.z.boolean().optional()
|
|
11072
9118
|
}).strict().optional(),
|
|
11073
|
-
pricingContext:
|
|
11074
|
-
serviceTier:
|
|
11075
|
-
inferenceGeo:
|
|
11076
|
-
speed:
|
|
9119
|
+
pricingContext: import_zod9.z.object({
|
|
9120
|
+
serviceTier: import_zod9.z.string().min(1).max(100).optional(),
|
|
9121
|
+
inferenceGeo: import_zod9.z.string().min(1).max(100).optional(),
|
|
9122
|
+
speed: import_zod9.z.string().min(1).max(100).optional()
|
|
11077
9123
|
}).strict().optional(),
|
|
11078
|
-
pricing:
|
|
11079
|
-
source:
|
|
11080
|
-
currency:
|
|
11081
|
-
actualCostMicros:
|
|
9124
|
+
pricing: import_zod9.z.object({
|
|
9125
|
+
source: import_zod9.z.literal("provider"),
|
|
9126
|
+
currency: import_zod9.z.literal("USD"),
|
|
9127
|
+
actualCostMicros: import_zod9.z.number().int().nonnegative().safe()
|
|
11082
9128
|
}).strict().optional()
|
|
11083
9129
|
}).strict();
|
|
11084
|
-
var EvalStepResultSchema =
|
|
11085
|
-
evidence:
|
|
11086
|
-
version:
|
|
11087
|
-
history:
|
|
9130
|
+
var EvalStepResultSchema = import_zod9.z.object({
|
|
9131
|
+
evidence: import_zod9.z.object({
|
|
9132
|
+
version: import_zod9.z.literal("compact-v1"),
|
|
9133
|
+
history: import_zod9.z.boolean(),
|
|
11088
9134
|
observation: EvalObservationSchema
|
|
11089
9135
|
}).strict().optional(),
|
|
11090
|
-
index:
|
|
11091
|
-
input:
|
|
11092
|
-
|
|
11093
|
-
|
|
9136
|
+
index: import_zod9.z.number().int().nonnegative(),
|
|
9137
|
+
input: import_zod9.z.union([
|
|
9138
|
+
import_zod9.z.object({ message: import_zod9.z.string() }).strict(),
|
|
9139
|
+
import_zod9.z.object({ resume: EvalResumeResponseSchema }).strict()
|
|
11094
9140
|
]),
|
|
11095
9141
|
expectation,
|
|
11096
9142
|
observation: EvalObservationSchema,
|
|
11097
|
-
reference:
|
|
9143
|
+
reference: import_zod9.z.json().optional(),
|
|
11098
9144
|
status,
|
|
11099
|
-
reason:
|
|
11100
|
-
judgeCalls:
|
|
11101
|
-
judgeUsage:
|
|
11102
|
-
criteria:
|
|
11103
|
-
EvalVerdictSchema.extend({ id:
|
|
9145
|
+
reason: import_zod9.z.string().optional(),
|
|
9146
|
+
judgeCalls: import_zod9.z.number().int().nonnegative().optional(),
|
|
9147
|
+
judgeUsage: import_zod9.z.array(EvalJudgeUsageSchema).optional(),
|
|
9148
|
+
criteria: import_zod9.z.array(
|
|
9149
|
+
EvalVerdictSchema.extend({ id: import_zod9.z.string(), text: import_zod9.z.string() })
|
|
11104
9150
|
)
|
|
11105
9151
|
}).strict();
|
|
11106
|
-
var runtimeExecution =
|
|
11107
|
-
var EvalCaseResultSchema =
|
|
9152
|
+
var runtimeExecution = import_zod9.z.object({ sessionId: import_zod9.z.string().min(1), runId: import_zod9.z.string().min(1) }).strict();
|
|
9153
|
+
var EvalCaseResultSchema = import_zod9.z.object({
|
|
11108
9154
|
verification: EvalVerdictSchema.optional(),
|
|
11109
|
-
phaseDurationsMs:
|
|
9155
|
+
phaseDurationsMs: import_zod9.z.partialRecord(issue.shape.phase, import_zod9.z.number().finite().nonnegative()).optional(),
|
|
11110
9156
|
caseId: id2,
|
|
11111
|
-
repetition:
|
|
11112
|
-
attemptId:
|
|
11113
|
-
runtimeExecutions:
|
|
11114
|
-
sessionId:
|
|
9157
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
9158
|
+
attemptId: import_zod9.z.uuid().optional(),
|
|
9159
|
+
runtimeExecutions: import_zod9.z.array(runtimeExecution).optional(),
|
|
9160
|
+
sessionId: import_zod9.z.string(),
|
|
11115
9161
|
status,
|
|
11116
|
-
durationMs:
|
|
11117
|
-
steps:
|
|
11118
|
-
errors:
|
|
9162
|
+
durationMs: import_zod9.z.number().nonnegative(),
|
|
9163
|
+
steps: import_zod9.z.array(EvalStepResultSchema),
|
|
9164
|
+
errors: import_zod9.z.array(issue)
|
|
11119
9165
|
}).strict();
|
|
11120
|
-
var EvalJudgeIdentitySchema =
|
|
9166
|
+
var EvalJudgeIdentitySchema = import_zod9.z.object({
|
|
11121
9167
|
id: id2,
|
|
11122
9168
|
version: id2,
|
|
11123
|
-
location:
|
|
9169
|
+
location: import_zod9.z.enum(["app", "studio"]).optional()
|
|
11124
9170
|
}).strict();
|
|
11125
9171
|
var judge = EvalJudgeIdentitySchema;
|
|
11126
|
-
var StudioEvalJudgeRequestSchema =
|
|
9172
|
+
var StudioEvalJudgeRequestSchema = import_zod9.z.object({
|
|
11127
9173
|
environment: id2,
|
|
11128
9174
|
judge: EvalJudgeIdentitySchema,
|
|
11129
|
-
criterion:
|
|
9175
|
+
criterion: import_zod9.z.object({ id: id2, text: import_zod9.z.string().trim().min(1).max(16384) }).strict(),
|
|
11130
9176
|
input: EvalStepResultSchema.shape.input,
|
|
11131
9177
|
observation: EvalObservationSchema,
|
|
11132
|
-
reference:
|
|
11133
|
-
conversation:
|
|
9178
|
+
reference: import_zod9.z.json().optional(),
|
|
9179
|
+
conversation: import_zod9.z.array(EvalStepResultSchema).max(100)
|
|
11134
9180
|
}).strict();
|
|
11135
|
-
var StudioEvalJudgeResponseSchema =
|
|
9181
|
+
var StudioEvalJudgeResponseSchema = import_zod9.z.object({
|
|
11136
9182
|
judge: EvalJudgeIdentitySchema,
|
|
11137
9183
|
verdict: EvalVerdictSchema,
|
|
11138
|
-
usage:
|
|
9184
|
+
usage: import_zod9.z.array(EvalJudgeUsageSchema).optional()
|
|
11139
9185
|
}).strict();
|
|
11140
|
-
var EvalRunResultSchema =
|
|
11141
|
-
id:
|
|
9186
|
+
var EvalRunResultSchema = import_zod9.z.object({
|
|
9187
|
+
id: import_zod9.z.uuid(),
|
|
11142
9188
|
suiteId: id2,
|
|
11143
|
-
suiteRevision:
|
|
9189
|
+
suiteRevision: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
|
|
11144
9190
|
suite: EvalSuiteSchema,
|
|
11145
9191
|
judge: judge.optional(),
|
|
11146
|
-
startedAt:
|
|
11147
|
-
durationMs:
|
|
9192
|
+
startedAt: import_zod9.z.iso.datetime(),
|
|
9193
|
+
durationMs: import_zod9.z.number().nonnegative(),
|
|
11148
9194
|
status,
|
|
11149
|
-
counts:
|
|
11150
|
-
passed:
|
|
11151
|
-
failed:
|
|
11152
|
-
error:
|
|
11153
|
-
cancelled:
|
|
11154
|
-
ungraded:
|
|
9195
|
+
counts: import_zod9.z.object({
|
|
9196
|
+
passed: import_zod9.z.number().int().nonnegative(),
|
|
9197
|
+
failed: import_zod9.z.number().int().nonnegative(),
|
|
9198
|
+
error: import_zod9.z.number().int().nonnegative(),
|
|
9199
|
+
cancelled: import_zod9.z.number().int().nonnegative(),
|
|
9200
|
+
ungraded: import_zod9.z.number().int().nonnegative().optional()
|
|
11155
9201
|
}).strict(),
|
|
11156
|
-
cases:
|
|
11157
|
-
errors:
|
|
9202
|
+
cases: import_zod9.z.array(EvalCaseResultSchema),
|
|
9203
|
+
errors: import_zod9.z.array(issue)
|
|
11158
9204
|
}).strict();
|
|
11159
|
-
var EvalManifestSchema =
|
|
11160
|
-
|
|
11161
|
-
|
|
11162
|
-
|
|
11163
|
-
|
|
11164
|
-
|
|
11165
|
-
|
|
11166
|
-
|
|
11167
|
-
|
|
9205
|
+
var EvalManifestSchema = import_zod9.z.object({
|
|
9206
|
+
execution: EvalExecutionSettingsSchema.optional(),
|
|
9207
|
+
schemaVersion: import_zod9.z.literal(1),
|
|
9208
|
+
studioJudging: import_zod9.z.literal(true).optional(),
|
|
9209
|
+
promptContracts: import_zod9.z.array(
|
|
9210
|
+
import_zod9.z.object({
|
|
9211
|
+
id: import_zod9.z.string(),
|
|
9212
|
+
format: import_zod9.z.enum(["system-user", "chat"]),
|
|
9213
|
+
variablesSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown()),
|
|
9214
|
+
configSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown())
|
|
11168
9215
|
}).strict()
|
|
11169
9216
|
).optional(),
|
|
11170
|
-
attemptScheduling:
|
|
11171
|
-
suites:
|
|
11172
|
-
responders:
|
|
11173
|
-
|
|
9217
|
+
attemptScheduling: import_zod9.z.literal(true).optional(),
|
|
9218
|
+
suites: import_zod9.z.array(EvalSuiteSchema),
|
|
9219
|
+
responders: import_zod9.z.array(
|
|
9220
|
+
import_zod9.z.object({
|
|
11174
9221
|
name: id2,
|
|
11175
9222
|
schemaId: id2.optional(),
|
|
11176
9223
|
schemaVersion: id2.optional()
|
|
11177
9224
|
}).strict()
|
|
11178
9225
|
),
|
|
11179
|
-
references:
|
|
11180
|
-
paramsSchema:
|
|
9226
|
+
references: import_zod9.z.array(id2),
|
|
9227
|
+
paramsSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown()).optional(),
|
|
11181
9228
|
judge: judge.optional()
|
|
11182
9229
|
}).strict();
|
|
11183
|
-
var EvalProgressSchema =
|
|
11184
|
-
|
|
11185
|
-
type:
|
|
9230
|
+
var EvalProgressSchema = import_zod9.z.discriminatedUnion("type", [
|
|
9231
|
+
import_zod9.z.object({
|
|
9232
|
+
type: import_zod9.z.literal("case-dispatch"),
|
|
11186
9233
|
caseId: id2,
|
|
11187
|
-
repetition:
|
|
11188
|
-
status:
|
|
11189
|
-
retryAt:
|
|
11190
|
-
httpStatus:
|
|
11191
|
-
executionId:
|
|
11192
|
-
generation:
|
|
9234
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
9235
|
+
status: import_zod9.z.enum(["queued", "retrying", "reconnecting"]),
|
|
9236
|
+
retryAt: import_zod9.z.iso.datetime().optional(),
|
|
9237
|
+
httpStatus: import_zod9.z.number().int().min(100).max(599).optional(),
|
|
9238
|
+
executionId: import_zod9.z.uuid().optional(),
|
|
9239
|
+
generation: import_zod9.z.uuid().optional()
|
|
11193
9240
|
}).strict(),
|
|
11194
|
-
|
|
11195
|
-
type:
|
|
9241
|
+
import_zod9.z.object({
|
|
9242
|
+
type: import_zod9.z.literal("case-runtime-associated"),
|
|
11196
9243
|
caseId: id2,
|
|
11197
|
-
repetition:
|
|
11198
|
-
attemptId:
|
|
9244
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
9245
|
+
attemptId: import_zod9.z.uuid(),
|
|
11199
9246
|
execution: runtimeExecution
|
|
11200
9247
|
}).strict(),
|
|
11201
|
-
|
|
11202
|
-
type:
|
|
11203
|
-
caseIds:
|
|
11204
|
-
repetitions:
|
|
11205
|
-
concurrency:
|
|
9248
|
+
import_zod9.z.object({
|
|
9249
|
+
type: import_zod9.z.literal("run-started"),
|
|
9250
|
+
caseIds: import_zod9.z.array(id2).min(1),
|
|
9251
|
+
repetitions: import_zod9.z.number().int().positive(),
|
|
9252
|
+
concurrency: import_zod9.z.number().int().positive()
|
|
11206
9253
|
}).strict(),
|
|
11207
|
-
|
|
11208
|
-
type:
|
|
9254
|
+
import_zod9.z.object({
|
|
9255
|
+
type: import_zod9.z.literal("case-progress"),
|
|
11209
9256
|
caseId: id2,
|
|
11210
|
-
repetition:
|
|
9257
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
11211
9258
|
phase: issue.shape.phase,
|
|
11212
|
-
stepIndex:
|
|
9259
|
+
stepIndex: import_zod9.z.number().int().nonnegative().optional()
|
|
11213
9260
|
}).strict(),
|
|
11214
|
-
|
|
11215
|
-
type:
|
|
11216
|
-
attemptId:
|
|
9261
|
+
import_zod9.z.object({
|
|
9262
|
+
type: import_zod9.z.literal("case-started"),
|
|
9263
|
+
attemptId: import_zod9.z.uuid().optional(),
|
|
11217
9264
|
caseId: id2,
|
|
11218
|
-
repetition:
|
|
11219
|
-
sessionId:
|
|
9265
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
9266
|
+
sessionId: import_zod9.z.string()
|
|
11220
9267
|
}).strict(),
|
|
11221
|
-
|
|
11222
|
-
type:
|
|
9268
|
+
import_zod9.z.object({
|
|
9269
|
+
type: import_zod9.z.literal("step-completed"),
|
|
11223
9270
|
caseId: id2,
|
|
11224
|
-
repetition:
|
|
9271
|
+
repetition: import_zod9.z.number().int().positive(),
|
|
11225
9272
|
step: EvalStepResultSchema
|
|
11226
9273
|
}).strict(),
|
|
11227
|
-
|
|
9274
|
+
import_zod9.z.object({ type: import_zod9.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
|
|
11228
9275
|
]);
|
|
11229
|
-
var EvalRemoteRunRequestSchema =
|
|
9276
|
+
var EvalRemoteRunRequestSchema = import_zod9.z.object({
|
|
11230
9277
|
promptSnapshot: import_browser2.PromptSnapshotSchema.optional(),
|
|
11231
|
-
promptGroupName:
|
|
9278
|
+
promptGroupName: import_zod9.z.string().optional(),
|
|
9279
|
+
caseTimeoutMs: EvalSettingsSchema.shape.caseTimeoutMs,
|
|
9280
|
+
runTimeoutMs: EvalSettingsSchema.shape.runTimeoutMs,
|
|
9281
|
+
resolvedSettings: ResolvedEvalSettingsSchema.optional(),
|
|
11232
9282
|
suiteId: id2,
|
|
11233
|
-
suiteRevision:
|
|
11234
|
-
grading:
|
|
9283
|
+
suiteRevision: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
|
|
9284
|
+
grading: import_zod9.z.enum(["app", "studio"]).optional(),
|
|
11235
9285
|
judge: EvalJudgeIdentitySchema.optional(),
|
|
11236
|
-
caseIds:
|
|
11237
|
-
repetitions:
|
|
11238
|
-
concurrency:
|
|
11239
|
-
attempt:
|
|
9286
|
+
caseIds: import_zod9.z.array(id2).min(1).max(100).optional(),
|
|
9287
|
+
repetitions: import_zod9.z.number().int().min(1).max(20).default(1),
|
|
9288
|
+
concurrency: import_zod9.z.number().int().min(1).max(20).default(1),
|
|
9289
|
+
attempt: import_zod9.z.object({
|
|
11240
9290
|
caseId: id2,
|
|
11241
|
-
repetition:
|
|
11242
|
-
evaluationId:
|
|
9291
|
+
repetition: import_zod9.z.number().int().min(1).max(20),
|
|
9292
|
+
evaluationId: import_zod9.z.uuid()
|
|
11243
9293
|
}).strict().optional()
|
|
11244
9294
|
}).strict();
|
|
11245
9295
|
var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
|
|
@@ -11249,39 +9299,42 @@ var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
|
|
|
11249
9299
|
promptGroupName: true,
|
|
11250
9300
|
attempt: true
|
|
11251
9301
|
}).extend({
|
|
11252
|
-
promptSelection:
|
|
11253
|
-
|
|
11254
|
-
|
|
11255
|
-
type:
|
|
11256
|
-
id:
|
|
11257
|
-
version:
|
|
9302
|
+
promptSelection: import_zod9.z.discriminatedUnion("type", [
|
|
9303
|
+
import_zod9.z.object({ type: import_zod9.z.literal("live") }).strict(),
|
|
9304
|
+
import_zod9.z.object({
|
|
9305
|
+
type: import_zod9.z.literal("single"),
|
|
9306
|
+
id: import_zod9.z.string(),
|
|
9307
|
+
version: import_zod9.z.number().int().positive()
|
|
11258
9308
|
}).strict(),
|
|
11259
|
-
|
|
9309
|
+
import_zod9.z.object({ type: import_zod9.z.literal("group"), groupId: import_zod9.z.uuid() }).strict()
|
|
11260
9310
|
]).optional(),
|
|
11261
|
-
targetId:
|
|
11262
|
-
judge:
|
|
9311
|
+
targetId: import_zod9.z.string().min(1).max(128),
|
|
9312
|
+
judge: import_zod9.z.enum(["studio", "app"]).optional(),
|
|
9313
|
+
repetitions: EvalSettingsSchema.shape.repetitions,
|
|
9314
|
+
concurrency: EvalSettingsSchema.shape.concurrency
|
|
11263
9315
|
});
|
|
11264
|
-
var EvalWireEventSchema =
|
|
11265
|
-
|
|
11266
|
-
type:
|
|
11267
|
-
state:
|
|
9316
|
+
var EvalWireEventSchema = import_zod9.z.discriminatedUnion("type", [
|
|
9317
|
+
import_zod9.z.object({
|
|
9318
|
+
type: import_zod9.z.literal("execution-status"),
|
|
9319
|
+
state: import_zod9.z.enum(["queued", "running", "completed", "cancelled", "unknown"])
|
|
11268
9320
|
}).strict(),
|
|
11269
|
-
|
|
11270
|
-
type:
|
|
9321
|
+
import_zod9.z.object({
|
|
9322
|
+
type: import_zod9.z.literal("progress"),
|
|
11271
9323
|
event: EvalProgressSchema,
|
|
11272
|
-
sequence:
|
|
9324
|
+
sequence: import_zod9.z.number().int().positive().optional()
|
|
11273
9325
|
}).strict(),
|
|
11274
|
-
|
|
11275
|
-
type:
|
|
9326
|
+
import_zod9.z.object({
|
|
9327
|
+
type: import_zod9.z.literal("result"),
|
|
11276
9328
|
result: EvalRunResultSchema,
|
|
11277
|
-
sequence:
|
|
9329
|
+
sequence: import_zod9.z.number().int().positive().optional()
|
|
11278
9330
|
}).strict(),
|
|
11279
|
-
|
|
11280
|
-
type:
|
|
11281
|
-
|
|
9331
|
+
import_zod9.z.object({
|
|
9332
|
+
type: import_zod9.z.literal("error"),
|
|
9333
|
+
exception: import_telemetry_contracts.DiagnosticContentSchema.optional(),
|
|
9334
|
+
message: import_zod9.z.string(),
|
|
11282
9335
|
failure: failureDescriptor.optional(),
|
|
11283
|
-
diagnosticId:
|
|
11284
|
-
sequence:
|
|
9336
|
+
diagnosticId: import_zod9.z.uuid().optional(),
|
|
9337
|
+
sequence: import_zod9.z.number().int().positive().optional()
|
|
11285
9338
|
}).strict()
|
|
11286
9339
|
]);
|
|
11287
9340
|
var EvalConfigurationError = class extends Error {
|
|
@@ -11311,13 +9364,14 @@ function parseEvalSuite(value) {
|
|
|
11311
9364
|
|
|
11312
9365
|
// src/evals/create-evals.ts
|
|
11313
9366
|
var import_node_crypto5 = require("crypto");
|
|
11314
|
-
var
|
|
9367
|
+
var import_eval_attribution3 = require("@kortyx/core/eval-attribution");
|
|
11315
9368
|
var import_usage_telemetry = require("@kortyx/usage-telemetry");
|
|
11316
|
-
var
|
|
9369
|
+
var import_zod11 = require("zod");
|
|
11317
9370
|
|
|
11318
9371
|
// src/evals/errors.ts
|
|
11319
9372
|
var import_errors9 = require("@kortyx/core/errors");
|
|
11320
|
-
var
|
|
9373
|
+
var import_telemetry_contracts2 = require("@kortyx/telemetry-contracts");
|
|
9374
|
+
var import_zod10 = require("zod");
|
|
11321
9375
|
var messages = {
|
|
11322
9376
|
params: "Case parameters failed validation.",
|
|
11323
9377
|
setup: "Case setup failed.",
|
|
@@ -11333,6 +9387,9 @@ function createEvalIssue(phase, error, options = {}) {
|
|
|
11333
9387
|
const control = (0, import_errors9.isControlFlowError)(error);
|
|
11334
9388
|
const declared = !control && ((0, import_errors9.isFailureDescriptor)(error) || (0, import_errors9.isFailureDescriptor)((0, import_errors9.errorProperty)(error, "failure")));
|
|
11335
9389
|
const failure = (0, import_errors9.serializeFailure)(control ? void 0 : error);
|
|
9390
|
+
const exception = !control && error !== void 0 ? (0, import_telemetry_contracts2.captureDiagnosticContent)(error) : null;
|
|
9391
|
+
const rawMessage = typeof error === "string" ? error : (0, import_errors9.errorProperty)(error, "message");
|
|
9392
|
+
const originalMessage = typeof rawMessage === "string" ? (0, import_telemetry_contracts2.redactDiagnosticText)(rawMessage) : void 0;
|
|
11336
9393
|
let diagnosticId = options.diagnosticId ?? (0, import_errors9.errorProperty)(error, "diagnosticId");
|
|
11337
9394
|
if (!control && !diagnosticId && options.reportError) {
|
|
11338
9395
|
try {
|
|
@@ -11340,12 +9397,16 @@ function createEvalIssue(phase, error, options = {}) {
|
|
|
11340
9397
|
} catch {
|
|
11341
9398
|
}
|
|
11342
9399
|
}
|
|
9400
|
+
const candidate = (0, import_errors9.errorProperty)(error, "statusCode") ?? (0, import_errors9.errorProperty)(error, "status");
|
|
9401
|
+
const httpStatus = typeof candidate === "number" && Number.isInteger(candidate) && candidate >= 100 && candidate <= 599 ? candidate : void 0;
|
|
11343
9402
|
return {
|
|
9403
|
+
...exception ? { exception } : {},
|
|
9404
|
+
...httpStatus ? { details: { httpStatus } } : {},
|
|
11344
9405
|
phase,
|
|
11345
9406
|
code: declared ? failure.code : options.code ?? "EVAL_HOOK_FAILED",
|
|
11346
|
-
message: declared ? failure.message : messages[phase],
|
|
9407
|
+
message: originalMessage ?? (declared ? failure.message : `${messages[phase]}${httpStatus ? ` HTTP ${httpStatus}.` : ""}`),
|
|
11347
9408
|
failure,
|
|
11348
|
-
...typeof diagnosticId === "string" &&
|
|
9409
|
+
...typeof diagnosticId === "string" && import_zod10.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
|
|
11349
9410
|
};
|
|
11350
9411
|
}
|
|
11351
9412
|
|
|
@@ -11852,12 +9913,28 @@ function createEvals(options) {
|
|
|
11852
9913
|
correlation,
|
|
11853
9914
|
...code ? { code } : {}
|
|
11854
9915
|
});
|
|
9916
|
+
for (const key of ["concurrency", "caseTimeoutMs"]) {
|
|
9917
|
+
const max = options.limits?.[key];
|
|
9918
|
+
if (max !== void 0 && defaults[key] > positive(max, `limits.${key}`))
|
|
9919
|
+
throw new EvalConfigurationError(
|
|
9920
|
+
`${key} default exceeds its application limit.`
|
|
9921
|
+
);
|
|
9922
|
+
}
|
|
11855
9923
|
return {
|
|
11856
9924
|
listSuites() {
|
|
11857
9925
|
return clone(suites);
|
|
11858
9926
|
},
|
|
11859
9927
|
describe() {
|
|
11860
9928
|
return {
|
|
9929
|
+
execution: {
|
|
9930
|
+
defaults: {
|
|
9931
|
+
...options.defaults?.repetitions !== void 0 ? { repetitions: options.defaults.repetitions } : {},
|
|
9932
|
+
...options.defaults?.concurrency !== void 0 ? { concurrency: options.defaults.concurrency } : {},
|
|
9933
|
+
...options.defaults?.caseTimeoutMs !== void 0 ? { caseTimeoutMs: options.defaults.caseTimeoutMs } : {}
|
|
9934
|
+
},
|
|
9935
|
+
limits: options.limits ?? {},
|
|
9936
|
+
overrides: ["repetitions", "concurrency", "caseTimeoutMs"]
|
|
9937
|
+
},
|
|
11861
9938
|
schemaVersion: 1,
|
|
11862
9939
|
studioJudging: true,
|
|
11863
9940
|
...options.agent.describePromptContracts ? { promptContracts: options.agent.describePromptContracts() } : {},
|
|
@@ -11872,7 +9949,7 @@ function createEvals(options) {
|
|
|
11872
9949
|
})),
|
|
11873
9950
|
references: Object.keys(references),
|
|
11874
9951
|
...options.paramsSchema ? {
|
|
11875
|
-
paramsSchema:
|
|
9952
|
+
paramsSchema: import_zod11.z.toJSONSchema(options.paramsSchema, {
|
|
11876
9953
|
io: "input"
|
|
11877
9954
|
})
|
|
11878
9955
|
} : {},
|
|
@@ -11886,252 +9963,368 @@ function createEvals(options) {
|
|
|
11886
9963
|
};
|
|
11887
9964
|
},
|
|
11888
9965
|
async run(args) {
|
|
11889
|
-
|
|
11890
|
-
if (!suite)
|
|
11891
|
-
throw new EvalConfigurationError(`Unknown suite: ${args.suiteId}`);
|
|
11892
|
-
if (args.caseIds && (!args.caseIds.length || new Set(args.caseIds).size !== args.caseIds.length || args.caseIds.some(
|
|
11893
|
-
(id3) => !suite.cases.some((item) => item.id === id3)
|
|
11894
|
-
)))
|
|
11895
|
-
throw new EvalConfigurationError("Select unique existing case IDs.");
|
|
11896
|
-
const repetitions = positive(
|
|
11897
|
-
args.repetitions ?? defaults.repetitions,
|
|
11898
|
-
"repetitions"
|
|
11899
|
-
);
|
|
11900
|
-
const concurrency = positive(
|
|
11901
|
-
args.concurrency ?? defaults.concurrency,
|
|
11902
|
-
"concurrency"
|
|
11903
|
-
);
|
|
11904
|
-
const selected = suite.cases.filter(
|
|
11905
|
-
(item) => !args.caseIds || args.caseIds.includes(item.id)
|
|
11906
|
-
);
|
|
11907
|
-
if (!Number.isSafeInteger(selected.length * repetitions))
|
|
11908
|
-
throw new EvalConfigurationError("Too many case repetitions.");
|
|
11909
|
-
if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
|
|
11910
|
-
throw new EvalConfigurationError("Invalid scheduled attempt.");
|
|
11911
|
-
const studioGrading = args.grading === "studio";
|
|
11912
|
-
if (!studioGrading && !options.judge && selected.some(
|
|
11913
|
-
(item) => item.steps.some((step) => step.expect.criteria?.length)
|
|
11914
|
-
))
|
|
11915
|
-
throw new EvalConfigurationError(
|
|
11916
|
-
"Local runs with criteria require a judge. Run from Studio or provide a code judge."
|
|
11917
|
-
);
|
|
11918
|
-
const selectedJudge = studioGrading ? args.judgeIdentity : options.judge;
|
|
11919
|
-
if (!studioGrading && args.judgeIdentity && (!options.judge || args.judgeIdentity.id !== options.judge.id || args.judgeIdentity.version !== options.judge.version || args.judgeIdentity.location !== options.judge.location))
|
|
11920
|
-
throw new EvalConfigurationError(
|
|
11921
|
-
"App judge changed. Refresh before running."
|
|
11922
|
-
);
|
|
9966
|
+
let usagePhase = "configuration";
|
|
11923
9967
|
const evalUsage = usage.start("sdk.eval", {
|
|
11924
|
-
|
|
9968
|
+
case_timeout_override: args.caseTimeoutMs !== void 0,
|
|
9969
|
+
settings_override: args.caseTimeoutMs !== void 0 || args.concurrency !== void 0 || args.repetitions !== void 0
|
|
11925
9970
|
});
|
|
11926
|
-
|
|
11927
|
-
|
|
11928
|
-
|
|
11929
|
-
|
|
11930
|
-
|
|
11931
|
-
|
|
11932
|
-
|
|
11933
|
-
|
|
11934
|
-
|
|
11935
|
-
|
|
11936
|
-
|
|
11937
|
-
|
|
11938
|
-
|
|
11939
|
-
|
|
11940
|
-
|
|
11941
|
-
|
|
11942
|
-
|
|
11943
|
-
|
|
11944
|
-
|
|
11945
|
-
|
|
11946
|
-
|
|
11947
|
-
|
|
11948
|
-
|
|
11949
|
-
try {
|
|
11950
|
-
await (0, import_eval_attribution2.withEvalAttribution)(
|
|
11951
|
-
void 0,
|
|
11952
|
-
() => args.onProgress(clone(event))
|
|
9971
|
+
try {
|
|
9972
|
+
const suite = suites.find((value) => value.id === args.suiteId);
|
|
9973
|
+
if (!suite)
|
|
9974
|
+
throw new EvalConfigurationError(`Unknown suite: ${args.suiteId}`);
|
|
9975
|
+
if (args.caseIds && (!args.caseIds.length || new Set(args.caseIds).size !== args.caseIds.length || args.caseIds.some(
|
|
9976
|
+
(id3) => !suite.cases.some((item) => item.id === id3)
|
|
9977
|
+
)))
|
|
9978
|
+
throw new EvalConfigurationError("Select unique existing case IDs.");
|
|
9979
|
+
const repetitions = positive(
|
|
9980
|
+
args.repetitions ?? defaults.repetitions,
|
|
9981
|
+
"repetitions"
|
|
9982
|
+
);
|
|
9983
|
+
const concurrency = positive(
|
|
9984
|
+
args.concurrency ?? defaults.concurrency,
|
|
9985
|
+
"concurrency"
|
|
9986
|
+
);
|
|
9987
|
+
const caseTimeoutMs = positive(
|
|
9988
|
+
args.caseTimeoutMs ?? defaults.caseTimeoutMs,
|
|
9989
|
+
"caseTimeoutMs"
|
|
9990
|
+
);
|
|
9991
|
+
if (caseTimeoutMs > 2147483647)
|
|
9992
|
+
throw new EvalConfigurationError(
|
|
9993
|
+
"caseTimeoutMs exceeds the supported timer range."
|
|
11953
9994
|
);
|
|
11954
|
-
|
|
11955
|
-
|
|
11956
|
-
|
|
9995
|
+
for (const [key, value] of [
|
|
9996
|
+
["concurrency", concurrency],
|
|
9997
|
+
["caseTimeoutMs", caseTimeoutMs]
|
|
9998
|
+
]) {
|
|
9999
|
+
const max = options.limits?.[key];
|
|
10000
|
+
if (max !== void 0 && value > max)
|
|
10001
|
+
throw new EvalConfigurationError(
|
|
10002
|
+
`${key} exceeds the application maximum of ${max}.`
|
|
10003
|
+
);
|
|
11957
10004
|
}
|
|
11958
|
-
|
|
11959
|
-
|
|
11960
|
-
|
|
11961
|
-
|
|
11962
|
-
|
|
11963
|
-
|
|
11964
|
-
|
|
11965
|
-
|
|
11966
|
-
|
|
11967
|
-
|
|
11968
|
-
|
|
11969
|
-
|
|
11970
|
-
|
|
11971
|
-
|
|
11972
|
-
const
|
|
11973
|
-
|
|
11974
|
-
|
|
11975
|
-
|
|
11976
|
-
|
|
11977
|
-
|
|
11978
|
-
|
|
11979
|
-
|
|
11980
|
-
|
|
11981
|
-
|
|
10005
|
+
const selected = suite.cases.filter(
|
|
10006
|
+
(item) => !args.caseIds || args.caseIds.includes(item.id)
|
|
10007
|
+
);
|
|
10008
|
+
if (!Number.isSafeInteger(selected.length * repetitions))
|
|
10009
|
+
throw new EvalConfigurationError("Too many case repetitions.");
|
|
10010
|
+
if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
|
|
10011
|
+
throw new EvalConfigurationError("Invalid scheduled attempt.");
|
|
10012
|
+
const studioGrading = args.grading === "studio";
|
|
10013
|
+
if (!studioGrading && !options.judge && selected.some(
|
|
10014
|
+
(item) => item.steps.some((step) => step.expect.criteria?.length)
|
|
10015
|
+
))
|
|
10016
|
+
throw new EvalConfigurationError(
|
|
10017
|
+
"Local runs with criteria require a judge. Run from Studio or provide a code judge."
|
|
10018
|
+
);
|
|
10019
|
+
const selectedJudge = studioGrading ? args.judgeIdentity : options.judge;
|
|
10020
|
+
if (!studioGrading && args.judgeIdentity && (!options.judge || args.judgeIdentity.id !== options.judge.id || args.judgeIdentity.version !== options.judge.version || args.judgeIdentity.location !== options.judge.location))
|
|
10021
|
+
throw new EvalConfigurationError(
|
|
10022
|
+
"App judge changed. Refresh before running."
|
|
10023
|
+
);
|
|
10024
|
+
usagePhase = "execution";
|
|
10025
|
+
const start = Date.now();
|
|
10026
|
+
const result = {
|
|
10027
|
+
id: (0, import_node_crypto5.randomUUID)(),
|
|
10028
|
+
suiteId: suite.id,
|
|
10029
|
+
suiteRevision: getEvalSuiteRevision(suite),
|
|
10030
|
+
suite: clone(suite),
|
|
10031
|
+
...selectedJudge ? {
|
|
10032
|
+
judge: {
|
|
10033
|
+
id: selectedJudge.id,
|
|
10034
|
+
version: selectedJudge.version,
|
|
10035
|
+
...selectedJudge.location ? { location: selectedJudge.location } : {}
|
|
10036
|
+
}
|
|
10037
|
+
} : {},
|
|
10038
|
+
startedAt: new Date(start).toISOString(),
|
|
11982
10039
|
durationMs: 0,
|
|
11983
|
-
|
|
11984
|
-
|
|
11985
|
-
|
|
10040
|
+
status: "passed",
|
|
10041
|
+
counts: { passed: 0, failed: 0, error: 0, cancelled: 0 },
|
|
10042
|
+
cases: [],
|
|
10043
|
+
errors: []
|
|
11986
10044
|
};
|
|
11987
|
-
let
|
|
11988
|
-
|
|
11989
|
-
|
|
11990
|
-
|
|
11991
|
-
|
|
11992
|
-
|
|
11993
|
-
|
|
11994
|
-
|
|
11995
|
-
|
|
11996
|
-
|
|
10045
|
+
let reporting = true;
|
|
10046
|
+
const emit = async (event) => {
|
|
10047
|
+
if (!args.onProgress || !reporting) return;
|
|
10048
|
+
try {
|
|
10049
|
+
await (0, import_eval_attribution3.withEvalAttribution)(
|
|
10050
|
+
void 0,
|
|
10051
|
+
() => args.onProgress?.(clone(event))
|
|
10052
|
+
);
|
|
10053
|
+
} catch (error) {
|
|
10054
|
+
reporting = false;
|
|
10055
|
+
result.errors.push(issueFor("reporting", error));
|
|
10056
|
+
}
|
|
10057
|
+
};
|
|
10058
|
+
const attempt = async (item, repetition) => {
|
|
10059
|
+
const started = Date.now();
|
|
10060
|
+
const controller = new AbortController();
|
|
10061
|
+
let timedOut = false;
|
|
10062
|
+
let timer;
|
|
10063
|
+
const deadline = (milliseconds) => {
|
|
10064
|
+
clearTimeout(timer);
|
|
10065
|
+
timer = setTimeout(() => {
|
|
10066
|
+
timedOut = true;
|
|
10067
|
+
controller.abort();
|
|
10068
|
+
}, milliseconds);
|
|
11997
10069
|
};
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
|
|
12003
|
-
|
|
12004
|
-
|
|
12005
|
-
|
|
12006
|
-
|
|
12007
|
-
|
|
12008
|
-
|
|
12009
|
-
|
|
12010
|
-
|
|
12011
|
-
|
|
12012
|
-
|
|
12013
|
-
|
|
12014
|
-
repetition,
|
|
12015
|
-
phase,
|
|
12016
|
-
...stepIndex !== void 0 ? { stepIndex } : {}
|
|
12017
|
-
});
|
|
10070
|
+
deadline(defaults.setupTimeoutMs);
|
|
10071
|
+
const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
|
|
10072
|
+
let originalExecutionError;
|
|
10073
|
+
const attemptId = (0, import_node_crypto5.randomUUID)();
|
|
10074
|
+
const runtimeExecutions = [];
|
|
10075
|
+
const attemptResult = {
|
|
10076
|
+
caseId: item.id,
|
|
10077
|
+
repetition,
|
|
10078
|
+
attemptId,
|
|
10079
|
+
runtimeExecutions,
|
|
10080
|
+
sessionId: `eval-${(0, import_node_crypto5.randomUUID)()}`,
|
|
10081
|
+
status: "passed",
|
|
10082
|
+
durationMs: 0,
|
|
10083
|
+
steps: [],
|
|
10084
|
+
errors: [],
|
|
10085
|
+
phaseDurationsMs: {}
|
|
12018
10086
|
};
|
|
12019
|
-
let
|
|
12020
|
-
let
|
|
12021
|
-
|
|
12022
|
-
|
|
12023
|
-
|
|
12024
|
-
|
|
12025
|
-
|
|
12026
|
-
|
|
12027
|
-
|
|
12028
|
-
|
|
12029
|
-
|
|
12030
|
-
|
|
12031
|
-
|
|
12032
|
-
|
|
12033
|
-
|
|
12034
|
-
|
|
12035
|
-
|
|
12036
|
-
|
|
12037
|
-
|
|
12038
|
-
|
|
12039
|
-
|
|
12040
|
-
command,
|
|
12041
|
-
continuation,
|
|
12042
|
-
history: clone(history),
|
|
12043
|
-
run
|
|
12044
|
-
}) : await run();
|
|
12045
|
-
continuation = execution.continuation;
|
|
12046
|
-
const observation = EvalObservationSchema.parse(
|
|
12047
|
-
execution.observation
|
|
10087
|
+
let associationProgress = Promise.resolve();
|
|
10088
|
+
let acceptingExecutions = true;
|
|
10089
|
+
const executeAttempt = async () => {
|
|
10090
|
+
let phase = "params";
|
|
10091
|
+
let phaseStarted = Date.now();
|
|
10092
|
+
const recordPhase = () => {
|
|
10093
|
+
const now = Date.now();
|
|
10094
|
+
const durations = attemptResult.phaseDurationsMs;
|
|
10095
|
+
durations[phase] = (durations[phase] ?? 0) + now - phaseStarted;
|
|
10096
|
+
phaseStarted = now;
|
|
10097
|
+
};
|
|
10098
|
+
const phaseIssue = (error, code) => issueFor(
|
|
10099
|
+
phase,
|
|
10100
|
+
error,
|
|
10101
|
+
{
|
|
10102
|
+
correlation: {
|
|
10103
|
+
sessionId: attemptResult.sessionId,
|
|
10104
|
+
attemptId
|
|
10105
|
+
}
|
|
10106
|
+
},
|
|
10107
|
+
code
|
|
12048
10108
|
);
|
|
12049
|
-
|
|
12050
|
-
|
|
12051
|
-
|
|
12052
|
-
|
|
12053
|
-
|
|
12054
|
-
|
|
12055
|
-
|
|
10109
|
+
let stepIndex;
|
|
10110
|
+
const reportPhase = async (next) => {
|
|
10111
|
+
recordPhase();
|
|
10112
|
+
phase = next;
|
|
10113
|
+
if (args.includeActivity)
|
|
10114
|
+
await emit({
|
|
10115
|
+
type: "case-progress",
|
|
10116
|
+
caseId: item.id,
|
|
10117
|
+
repetition,
|
|
10118
|
+
phase,
|
|
10119
|
+
...stepIndex !== void 0 ? { stepIndex } : {}
|
|
10120
|
+
});
|
|
12056
10121
|
};
|
|
12057
|
-
|
|
12058
|
-
|
|
12059
|
-
|
|
12060
|
-
|
|
12061
|
-
|
|
12062
|
-
|
|
12063
|
-
|
|
12064
|
-
|
|
12065
|
-
|
|
12066
|
-
|
|
12067
|
-
|
|
12068
|
-
|
|
12069
|
-
|
|
12070
|
-
|
|
12071
|
-
|
|
12072
|
-
|
|
12073
|
-
|
|
12074
|
-
|
|
12075
|
-
|
|
10122
|
+
let context;
|
|
10123
|
+
let continuation;
|
|
10124
|
+
let activeStep;
|
|
10125
|
+
let activeStepReported = false;
|
|
10126
|
+
const history = [];
|
|
10127
|
+
const call = async (readyContext, command, activeSignal) => {
|
|
10128
|
+
const run = (overrides) => executeEvalChat({
|
|
10129
|
+
agent: options.agent,
|
|
10130
|
+
...args.promptSnapshot ? { promptSnapshot: args.promptSnapshot } : {},
|
|
10131
|
+
command,
|
|
10132
|
+
continuation,
|
|
10133
|
+
history: clone(history),
|
|
10134
|
+
sessionId: attemptResult.sessionId,
|
|
10135
|
+
clientTurnId: (0, import_node_crypto5.randomUUID)(),
|
|
10136
|
+
signal: activeSignal,
|
|
10137
|
+
...item.workflowId ? { workflowId: item.workflowId } : {},
|
|
10138
|
+
...overrides
|
|
10139
|
+
});
|
|
10140
|
+
const execution = options.execute ? await options.execute({
|
|
10141
|
+
...readyContext,
|
|
10142
|
+
signal: activeSignal,
|
|
10143
|
+
command,
|
|
10144
|
+
continuation,
|
|
10145
|
+
history: clone(history),
|
|
10146
|
+
run
|
|
10147
|
+
}) : await run();
|
|
10148
|
+
continuation = execution.continuation;
|
|
10149
|
+
const observation = EvalObservationSchema.parse(
|
|
10150
|
+
execution.observation
|
|
10151
|
+
);
|
|
10152
|
+
if (observation.type === "interrupt" && execution.continuation === void 0)
|
|
10153
|
+
throw new Error(
|
|
10154
|
+
"A waiting interrupt requires a private continuation."
|
|
10155
|
+
);
|
|
10156
|
+
return {
|
|
10157
|
+
observation,
|
|
10158
|
+
...continuation !== void 0 ? { continuation } : {}
|
|
10159
|
+
};
|
|
12076
10160
|
};
|
|
12077
|
-
|
|
12078
|
-
|
|
12079
|
-
|
|
12080
|
-
|
|
12081
|
-
|
|
12082
|
-
|
|
12083
|
-
|
|
12084
|
-
|
|
12085
|
-
activeStepReported = false;
|
|
12086
|
-
signal.throwIfAborted();
|
|
12087
|
-
let command;
|
|
12088
|
-
if (step.message !== void 0)
|
|
12089
|
-
command = { type: "message", message: step.message };
|
|
12090
|
-
else {
|
|
12091
|
-
await reportPhase("responder");
|
|
12092
|
-
const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
|
|
12093
|
-
let response = step.resume;
|
|
12094
|
-
if (handlerRef(response)) {
|
|
12095
|
-
const responder = responders[response.using];
|
|
12096
|
-
if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
|
|
12097
|
-
throw new Error("Incompatible responder.");
|
|
12098
|
-
response = await (typeof responder === "function" ? responder : responder.respond)({
|
|
12099
|
-
...context,
|
|
12100
|
-
interrupt: clone(interrupt),
|
|
12101
|
-
...response.params !== void 0 ? { args: response.params } : {}
|
|
12102
|
-
});
|
|
12103
|
-
}
|
|
12104
|
-
command = {
|
|
12105
|
-
type: "resume",
|
|
12106
|
-
// z.json() is inferred as optional under the package's relaxed
|
|
12107
|
-
// declaration build. Runtime validation still requires value.
|
|
12108
|
-
response: EvalResumeResponseSchema.parse(
|
|
12109
|
-
response
|
|
12110
|
-
)
|
|
12111
|
-
};
|
|
12112
|
-
}
|
|
10161
|
+
try {
|
|
10162
|
+
await emit({
|
|
10163
|
+
type: "case-started",
|
|
10164
|
+
attemptId,
|
|
10165
|
+
caseId: item.id,
|
|
10166
|
+
repetition,
|
|
10167
|
+
sessionId: attemptResult.sessionId
|
|
10168
|
+
});
|
|
12113
10169
|
signal.throwIfAborted();
|
|
12114
|
-
await reportPhase("
|
|
12115
|
-
const
|
|
12116
|
-
const
|
|
12117
|
-
|
|
12118
|
-
|
|
12119
|
-
|
|
12120
|
-
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
criteria: []
|
|
10170
|
+
await reportPhase("params");
|
|
10171
|
+
const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
|
|
10172
|
+
const base = {
|
|
10173
|
+
suiteId: suite.id,
|
|
10174
|
+
case: item,
|
|
10175
|
+
repetition,
|
|
10176
|
+
sessionId: attemptResult.sessionId,
|
|
10177
|
+
params,
|
|
10178
|
+
signal
|
|
12124
10179
|
};
|
|
12125
|
-
|
|
12126
|
-
|
|
10180
|
+
await reportPhase("setup");
|
|
10181
|
+
const prepared = options.setup ? await options.setup(base) : void 0;
|
|
10182
|
+
context = { ...base, prepared };
|
|
12127
10183
|
signal.throwIfAborted();
|
|
12128
|
-
|
|
12129
|
-
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
10184
|
+
deadline(caseTimeoutMs);
|
|
10185
|
+
for (const [index, step] of item.steps.entries()) {
|
|
10186
|
+
stepIndex = index;
|
|
10187
|
+
activeStep = void 0;
|
|
10188
|
+
activeStepReported = false;
|
|
10189
|
+
signal.throwIfAborted();
|
|
10190
|
+
let command;
|
|
10191
|
+
if (step.message !== void 0)
|
|
10192
|
+
command = { type: "message", message: step.message };
|
|
10193
|
+
else {
|
|
10194
|
+
await reportPhase("responder");
|
|
10195
|
+
const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
|
|
10196
|
+
let response = step.resume;
|
|
10197
|
+
if (handlerRef(response)) {
|
|
10198
|
+
const responder = responders[response.using];
|
|
10199
|
+
if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
|
|
10200
|
+
throw new Error("Incompatible responder.");
|
|
10201
|
+
response = await (typeof responder === "function" ? responder : responder.respond)({
|
|
10202
|
+
...context,
|
|
10203
|
+
interrupt: clone(interrupt),
|
|
10204
|
+
...response.params !== void 0 ? { args: response.params } : {}
|
|
10205
|
+
});
|
|
10206
|
+
}
|
|
10207
|
+
command = {
|
|
10208
|
+
type: "resume",
|
|
10209
|
+
// z.json() is inferred as optional under the package's relaxed
|
|
10210
|
+
// declaration build. Runtime validation still requires value.
|
|
10211
|
+
response: EvalResumeResponseSchema.parse(
|
|
10212
|
+
response
|
|
10213
|
+
)
|
|
10214
|
+
};
|
|
10215
|
+
}
|
|
10216
|
+
signal.throwIfAborted();
|
|
10217
|
+
await reportPhase("execute");
|
|
10218
|
+
originalExecutionError = void 0;
|
|
10219
|
+
const { observation } = await call(context, command, signal);
|
|
10220
|
+
const input = command.type === "message" ? { message: command.message } : { resume: command.response };
|
|
10221
|
+
const evaluated = {
|
|
10222
|
+
index,
|
|
10223
|
+
input,
|
|
10224
|
+
expectation: clone(step.expect),
|
|
10225
|
+
observation,
|
|
10226
|
+
status: "passed",
|
|
10227
|
+
criteria: []
|
|
10228
|
+
};
|
|
10229
|
+
attemptResult.steps.push(evaluated);
|
|
10230
|
+
activeStep = evaluated;
|
|
10231
|
+
signal.throwIfAborted();
|
|
10232
|
+
if (observation.type === "error") {
|
|
10233
|
+
const issue2 = createEvalIssue(
|
|
10234
|
+
"execute",
|
|
10235
|
+
originalExecutionError ?? observation.failure ?? observation.text,
|
|
10236
|
+
{
|
|
10237
|
+
...observation.diagnosticId ? { diagnosticId: observation.diagnosticId } : {}
|
|
10238
|
+
}
|
|
10239
|
+
);
|
|
10240
|
+
attemptResult.errors.push(issue2);
|
|
10241
|
+
attemptResult.status = evaluated.status = "error";
|
|
10242
|
+
evaluated.reason = issue2.message;
|
|
10243
|
+
await emit({
|
|
10244
|
+
type: "step-completed",
|
|
10245
|
+
caseId: item.id,
|
|
10246
|
+
repetition,
|
|
10247
|
+
step: evaluated
|
|
10248
|
+
});
|
|
10249
|
+
activeStepReported = true;
|
|
10250
|
+
break;
|
|
10251
|
+
}
|
|
10252
|
+
if (observation.type === "cancelled") {
|
|
10253
|
+
evaluated.status = "cancelled";
|
|
10254
|
+
attemptResult.status = "cancelled";
|
|
10255
|
+
await emit({
|
|
10256
|
+
type: "step-completed",
|
|
10257
|
+
caseId: item.id,
|
|
10258
|
+
repetition,
|
|
10259
|
+
step: evaluated
|
|
10260
|
+
});
|
|
10261
|
+
activeStepReported = true;
|
|
10262
|
+
break;
|
|
10263
|
+
}
|
|
10264
|
+
if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
|
|
10265
|
+
evaluated.status = "failed";
|
|
10266
|
+
evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
|
|
10267
|
+
} else {
|
|
10268
|
+
const outputReason = missingOutputReason(
|
|
10269
|
+
step.expect.outputs,
|
|
10270
|
+
observation
|
|
10271
|
+
);
|
|
10272
|
+
if (outputReason) {
|
|
10273
|
+
evaluated.status = "failed";
|
|
10274
|
+
evaluated.reason = outputReason;
|
|
10275
|
+
} else {
|
|
10276
|
+
const reference = step.expect.reference;
|
|
10277
|
+
if (reference !== void 0) {
|
|
10278
|
+
await reportPhase("reference");
|
|
10279
|
+
evaluated.reference = handlerRef(reference) ? import_zod11.z.json().parse(
|
|
10280
|
+
await references[reference.using]?.({
|
|
10281
|
+
...context,
|
|
10282
|
+
observation: clone(observation),
|
|
10283
|
+
...reference.params !== void 0 ? { args: reference.params } : {}
|
|
10284
|
+
})
|
|
10285
|
+
) : clone(reference);
|
|
10286
|
+
}
|
|
10287
|
+
await reportPhase("grading");
|
|
10288
|
+
evaluated.evidence = createEvalJudgeEvidence(
|
|
10289
|
+
observation,
|
|
10290
|
+
suite.evidence,
|
|
10291
|
+
evidenceFilters
|
|
10292
|
+
);
|
|
10293
|
+
if (studioGrading && step.expect.criteria?.length) {
|
|
10294
|
+
evaluated.status = "ungraded";
|
|
10295
|
+
attemptResult.status = "ungraded";
|
|
10296
|
+
}
|
|
10297
|
+
for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
|
|
10298
|
+
signal.throwIfAborted();
|
|
10299
|
+
const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
|
|
10300
|
+
evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
|
|
10301
|
+
const verdict = EvalVerdictSchema.parse(
|
|
10302
|
+
await (0, import_eval_attribution3.withEvalAttribution)(
|
|
10303
|
+
void 0,
|
|
10304
|
+
() => options.judge.grade({
|
|
10305
|
+
criterion,
|
|
10306
|
+
onUsage: (usage2) => {
|
|
10307
|
+
if (!signal.aborted) {
|
|
10308
|
+
evaluated.judgeUsage ??= [];
|
|
10309
|
+
evaluated.judgeUsage.push(clone(usage2));
|
|
10310
|
+
}
|
|
10311
|
+
},
|
|
10312
|
+
input: clone(input),
|
|
10313
|
+
...getEvalGradeEvidence(
|
|
10314
|
+
evaluated,
|
|
10315
|
+
attemptResult.steps.slice(0, -1)
|
|
10316
|
+
),
|
|
10317
|
+
signal,
|
|
10318
|
+
...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
|
|
10319
|
+
})
|
|
10320
|
+
)
|
|
10321
|
+
);
|
|
10322
|
+
signal.throwIfAborted();
|
|
10323
|
+
evaluated.criteria.push({ ...criterion, ...verdict });
|
|
10324
|
+
if (!verdict.passed) evaluated.status = "failed";
|
|
10325
|
+
}
|
|
10326
|
+
}
|
|
10327
|
+
}
|
|
12135
10328
|
await emit({
|
|
12136
10329
|
type: "step-completed",
|
|
12137
10330
|
caseId: item.id,
|
|
@@ -12139,255 +10332,191 @@ function createEvals(options) {
|
|
|
12139
10332
|
step: evaluated
|
|
12140
10333
|
});
|
|
12141
10334
|
activeStepReported = true;
|
|
12142
|
-
|
|
10335
|
+
if (evaluated.status === "failed") {
|
|
10336
|
+
attemptResult.status = "failed";
|
|
10337
|
+
break;
|
|
10338
|
+
}
|
|
10339
|
+
history.push(
|
|
10340
|
+
{
|
|
10341
|
+
role: "user",
|
|
10342
|
+
content: command.type === "message" ? command.message : JSON.stringify(command.response)
|
|
10343
|
+
},
|
|
10344
|
+
{ role: "assistant", content: observation.text }
|
|
10345
|
+
);
|
|
10346
|
+
}
|
|
10347
|
+
if (options.verify && attemptResult.status !== "cancelled" && attemptResult.status !== "error") {
|
|
10348
|
+
stepIndex = void 0;
|
|
10349
|
+
deadline(defaults.verificationTimeoutMs);
|
|
10350
|
+
await reportPhase("verification");
|
|
10351
|
+
const verification = EvalVerdictSchema.parse(
|
|
10352
|
+
await options.verify({
|
|
10353
|
+
...context,
|
|
10354
|
+
result: clone(attemptResult)
|
|
10355
|
+
})
|
|
10356
|
+
);
|
|
10357
|
+
signal.throwIfAborted();
|
|
10358
|
+
attemptResult.verification = verification;
|
|
10359
|
+
if (!verification.passed) attemptResult.status = "failed";
|
|
12143
10360
|
}
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
attemptResult.status = "cancelled";
|
|
10361
|
+
} catch (error) {
|
|
10362
|
+
if (signal.aborted) {
|
|
10363
|
+
attemptResult.status = timedOut ? "error" : "cancelled";
|
|
10364
|
+
if (timedOut)
|
|
10365
|
+
attemptResult.errors.push({
|
|
10366
|
+
...phaseIssue(error),
|
|
10367
|
+
details: {
|
|
10368
|
+
timeoutMs: phase === "params" || phase === "setup" ? defaults.setupTimeoutMs : phase === "verification" ? defaults.verificationTimeoutMs : caseTimeoutMs
|
|
10369
|
+
},
|
|
10370
|
+
code: phase === "params" || phase === "setup" ? "EVAL_SETUP_TIMEOUT" : phase === "verification" ? "EVAL_VERIFICATION_TIMEOUT" : "EVAL_TIMEOUT",
|
|
10371
|
+
message: phase === "params" || phase === "setup" ? "Case setup exceeded its configured time limit." : phase === "verification" ? "Case verification exceeded its configured time limit." : "Case execution exceeded its configured time limit."
|
|
10372
|
+
});
|
|
10373
|
+
} else {
|
|
10374
|
+
attemptResult.status = "error";
|
|
10375
|
+
attemptResult.errors.push(phaseIssue(error));
|
|
10376
|
+
}
|
|
10377
|
+
if (activeStep && !activeStepReported) {
|
|
10378
|
+
activeStep.status = attemptResult.status;
|
|
10379
|
+
activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
|
|
12147
10380
|
await emit({
|
|
12148
10381
|
type: "step-completed",
|
|
12149
10382
|
caseId: item.id,
|
|
12150
10383
|
repetition,
|
|
12151
|
-
step:
|
|
10384
|
+
step: activeStep
|
|
12152
10385
|
});
|
|
12153
|
-
activeStepReported = true;
|
|
12154
|
-
break;
|
|
12155
10386
|
}
|
|
12156
|
-
|
|
12157
|
-
|
|
12158
|
-
|
|
12159
|
-
|
|
12160
|
-
const
|
|
12161
|
-
|
|
12162
|
-
|
|
10387
|
+
} finally {
|
|
10388
|
+
clearTimeout(timer);
|
|
10389
|
+
if (context) {
|
|
10390
|
+
await reportPhase("cleanup");
|
|
10391
|
+
const cleanup = new AbortController();
|
|
10392
|
+
const cleanupTimer = setTimeout(
|
|
10393
|
+
() => cleanup.abort(),
|
|
10394
|
+
defaults.cleanupTimeoutMs
|
|
12163
10395
|
);
|
|
12164
|
-
|
|
12165
|
-
|
|
12166
|
-
|
|
12167
|
-
|
|
12168
|
-
|
|
12169
|
-
|
|
12170
|
-
|
|
12171
|
-
|
|
12172
|
-
|
|
12173
|
-
|
|
12174
|
-
|
|
12175
|
-
|
|
12176
|
-
|
|
12177
|
-
|
|
12178
|
-
|
|
12179
|
-
|
|
12180
|
-
|
|
12181
|
-
|
|
12182
|
-
suite.evidence,
|
|
12183
|
-
evidenceFilters
|
|
12184
|
-
);
|
|
12185
|
-
if (studioGrading && step.expect.criteria?.length) {
|
|
12186
|
-
evaluated.status = "ungraded";
|
|
12187
|
-
attemptResult.status = "ungraded";
|
|
12188
|
-
}
|
|
12189
|
-
for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
|
|
12190
|
-
signal.throwIfAborted();
|
|
12191
|
-
const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
|
|
12192
|
-
evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
|
|
12193
|
-
const verdict = EvalVerdictSchema.parse(
|
|
12194
|
-
await (0, import_eval_attribution2.withEvalAttribution)(
|
|
12195
|
-
void 0,
|
|
12196
|
-
() => options.judge.grade({
|
|
12197
|
-
criterion,
|
|
12198
|
-
onUsage: (usage2) => {
|
|
12199
|
-
if (!signal.aborted) {
|
|
12200
|
-
evaluated.judgeUsage ??= [];
|
|
12201
|
-
evaluated.judgeUsage.push(clone(usage2));
|
|
10396
|
+
try {
|
|
10397
|
+
if (continuation !== void 0) {
|
|
10398
|
+
try {
|
|
10399
|
+
const cancelled = await call(
|
|
10400
|
+
context,
|
|
10401
|
+
{ type: "cancel" },
|
|
10402
|
+
cleanup.signal
|
|
10403
|
+
);
|
|
10404
|
+
if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
|
|
10405
|
+
attemptResult.errors.push(
|
|
10406
|
+
createEvalIssue(
|
|
10407
|
+
"cleanup",
|
|
10408
|
+
cancelled.observation.failure,
|
|
10409
|
+
{
|
|
10410
|
+
code: "EVAL_INTERRUPT_CLEANUP_FAILED",
|
|
10411
|
+
...cancelled.observation.diagnosticId ? {
|
|
10412
|
+
diagnosticId: cancelled.observation.diagnosticId
|
|
10413
|
+
} : {}
|
|
12202
10414
|
}
|
|
12203
|
-
|
|
12204
|
-
|
|
12205
|
-
|
|
12206
|
-
|
|
12207
|
-
attemptResult.steps.slice(0, -1)
|
|
12208
|
-
),
|
|
12209
|
-
signal,
|
|
12210
|
-
...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
|
|
12211
|
-
})
|
|
12212
|
-
)
|
|
12213
|
-
);
|
|
12214
|
-
signal.throwIfAborted();
|
|
12215
|
-
evaluated.criteria.push({ ...criterion, ...verdict });
|
|
12216
|
-
if (!verdict.passed) evaluated.status = "failed";
|
|
12217
|
-
}
|
|
12218
|
-
}
|
|
12219
|
-
}
|
|
12220
|
-
await emit({
|
|
12221
|
-
type: "step-completed",
|
|
12222
|
-
caseId: item.id,
|
|
12223
|
-
repetition,
|
|
12224
|
-
step: evaluated
|
|
12225
|
-
});
|
|
12226
|
-
activeStepReported = true;
|
|
12227
|
-
if (evaluated.status === "failed") {
|
|
12228
|
-
attemptResult.status = "failed";
|
|
12229
|
-
break;
|
|
12230
|
-
}
|
|
12231
|
-
history.push(
|
|
12232
|
-
{
|
|
12233
|
-
role: "user",
|
|
12234
|
-
content: command.type === "message" ? command.message : JSON.stringify(command.response)
|
|
12235
|
-
},
|
|
12236
|
-
{ role: "assistant", content: observation.text }
|
|
12237
|
-
);
|
|
12238
|
-
}
|
|
12239
|
-
if (options.verify && attemptResult.status !== "cancelled" && attemptResult.status !== "error") {
|
|
12240
|
-
stepIndex = void 0;
|
|
12241
|
-
deadline(defaults.verificationTimeoutMs);
|
|
12242
|
-
await reportPhase("verification");
|
|
12243
|
-
const verification = EvalVerdictSchema.parse(
|
|
12244
|
-
await options.verify({
|
|
12245
|
-
...context,
|
|
12246
|
-
result: clone(attemptResult)
|
|
12247
|
-
})
|
|
12248
|
-
);
|
|
12249
|
-
signal.throwIfAborted();
|
|
12250
|
-
attemptResult.verification = verification;
|
|
12251
|
-
if (!verification.passed) attemptResult.status = "failed";
|
|
12252
|
-
}
|
|
12253
|
-
} catch (error) {
|
|
12254
|
-
if (signal.aborted) {
|
|
12255
|
-
attemptResult.status = timedOut ? "error" : "cancelled";
|
|
12256
|
-
if (timedOut)
|
|
12257
|
-
attemptResult.errors.push({
|
|
12258
|
-
...phaseIssue(error),
|
|
12259
|
-
code: phase === "params" || phase === "setup" ? "EVAL_SETUP_TIMEOUT" : phase === "verification" ? "EVAL_VERIFICATION_TIMEOUT" : "EVAL_TIMEOUT",
|
|
12260
|
-
message: phase === "params" || phase === "setup" ? "Case setup exceeded its configured time limit." : phase === "verification" ? "Case verification exceeded its configured time limit." : "Case execution exceeded its configured time limit."
|
|
12261
|
-
});
|
|
12262
|
-
} else {
|
|
12263
|
-
attemptResult.status = "error";
|
|
12264
|
-
attemptResult.errors.push(phaseIssue(error));
|
|
12265
|
-
}
|
|
12266
|
-
if (activeStep && !activeStepReported) {
|
|
12267
|
-
activeStep.status = attemptResult.status;
|
|
12268
|
-
activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
|
|
12269
|
-
await emit({
|
|
12270
|
-
type: "step-completed",
|
|
12271
|
-
caseId: item.id,
|
|
12272
|
-
repetition,
|
|
12273
|
-
step: activeStep
|
|
12274
|
-
});
|
|
12275
|
-
}
|
|
12276
|
-
} finally {
|
|
12277
|
-
clearTimeout(timer);
|
|
12278
|
-
if (context) {
|
|
12279
|
-
await reportPhase("cleanup");
|
|
12280
|
-
const cleanup = new AbortController();
|
|
12281
|
-
const cleanupTimer = setTimeout(
|
|
12282
|
-
() => cleanup.abort(),
|
|
12283
|
-
defaults.cleanupTimeoutMs
|
|
12284
|
-
);
|
|
12285
|
-
try {
|
|
12286
|
-
if (continuation !== void 0) {
|
|
12287
|
-
try {
|
|
12288
|
-
const cancelled = await call(
|
|
12289
|
-
context,
|
|
12290
|
-
{ type: "cancel" },
|
|
12291
|
-
cleanup.signal
|
|
12292
|
-
);
|
|
12293
|
-
if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
|
|
10415
|
+
)
|
|
10416
|
+
);
|
|
10417
|
+
cleanup.signal.throwIfAborted();
|
|
10418
|
+
} catch (error) {
|
|
12294
10419
|
attemptResult.errors.push(
|
|
12295
|
-
|
|
12296
|
-
"cleanup",
|
|
12297
|
-
cancelled.observation.failure,
|
|
12298
|
-
{
|
|
12299
|
-
code: "EVAL_INTERRUPT_CLEANUP_FAILED",
|
|
12300
|
-
...cancelled.observation.diagnosticId ? {
|
|
12301
|
-
diagnosticId: cancelled.observation.diagnosticId
|
|
12302
|
-
} : {}
|
|
12303
|
-
}
|
|
12304
|
-
)
|
|
10420
|
+
phaseIssue(error, "EVAL_INTERRUPT_CLEANUP_FAILED")
|
|
12305
10421
|
);
|
|
10422
|
+
}
|
|
10423
|
+
}
|
|
10424
|
+
try {
|
|
10425
|
+
await options.teardown?.({
|
|
10426
|
+
...context,
|
|
10427
|
+
signal: cleanup.signal,
|
|
10428
|
+
result: clone(attemptResult)
|
|
10429
|
+
});
|
|
12306
10430
|
cleanup.signal.throwIfAborted();
|
|
12307
10431
|
} catch (error) {
|
|
12308
|
-
attemptResult.errors.push(
|
|
12309
|
-
phaseIssue(error, "EVAL_INTERRUPT_CLEANUP_FAILED")
|
|
12310
|
-
);
|
|
10432
|
+
attemptResult.errors.push(phaseIssue(error));
|
|
12311
10433
|
}
|
|
10434
|
+
} finally {
|
|
10435
|
+
clearTimeout(cleanupTimer);
|
|
12312
10436
|
}
|
|
12313
|
-
|
|
12314
|
-
|
|
12315
|
-
|
|
12316
|
-
|
|
12317
|
-
result: clone(attemptResult)
|
|
12318
|
-
});
|
|
12319
|
-
cleanup.signal.throwIfAborted();
|
|
12320
|
-
} catch (error) {
|
|
12321
|
-
attemptResult.errors.push(phaseIssue(error));
|
|
12322
|
-
}
|
|
12323
|
-
} finally {
|
|
12324
|
-
clearTimeout(cleanupTimer);
|
|
10437
|
+
if (attemptResult.errors.some(
|
|
10438
|
+
(error) => error.phase === "cleanup"
|
|
10439
|
+
))
|
|
10440
|
+
attemptResult.status = "error";
|
|
12325
10441
|
}
|
|
12326
|
-
|
|
12327
|
-
|
|
10442
|
+
recordPhase();
|
|
10443
|
+
attemptResult.durationMs = Date.now() - started;
|
|
12328
10444
|
}
|
|
12329
|
-
|
|
12330
|
-
|
|
12331
|
-
|
|
12332
|
-
|
|
12333
|
-
|
|
12334
|
-
|
|
12335
|
-
|
|
10445
|
+
acceptingExecutions = false;
|
|
10446
|
+
await associationProgress;
|
|
10447
|
+
await emit({ type: "case-completed", result: attemptResult });
|
|
10448
|
+
return attemptResult;
|
|
10449
|
+
};
|
|
10450
|
+
return (0, import_eval_attribution3.withEvalAttribution)(
|
|
10451
|
+
{
|
|
10452
|
+
attemptId,
|
|
10453
|
+
onError: (error) => {
|
|
10454
|
+
originalExecutionError = error;
|
|
10455
|
+
},
|
|
10456
|
+
onExecution: (execution) => {
|
|
10457
|
+
if (!acceptingExecutions || runtimeExecutions.some(
|
|
10458
|
+
(entry) => entry.runId === execution.runId && entry.sessionId === execution.sessionId
|
|
10459
|
+
))
|
|
10460
|
+
return;
|
|
10461
|
+
runtimeExecutions.push(execution);
|
|
10462
|
+
associationProgress = associationProgress.then(
|
|
10463
|
+
() => emit({
|
|
10464
|
+
type: "case-runtime-associated",
|
|
10465
|
+
caseId: item.id,
|
|
10466
|
+
repetition,
|
|
10467
|
+
attemptId,
|
|
10468
|
+
execution
|
|
10469
|
+
})
|
|
10470
|
+
);
|
|
10471
|
+
}
|
|
10472
|
+
},
|
|
10473
|
+
executeAttempt
|
|
10474
|
+
);
|
|
12336
10475
|
};
|
|
12337
|
-
|
|
12338
|
-
|
|
12339
|
-
|
|
12340
|
-
|
|
12341
|
-
|
|
12342
|
-
|
|
12343
|
-
|
|
12344
|
-
|
|
12345
|
-
|
|
12346
|
-
|
|
12347
|
-
|
|
12348
|
-
|
|
12349
|
-
|
|
12350
|
-
|
|
12351
|
-
|
|
12352
|
-
|
|
12353
|
-
|
|
10476
|
+
const total = args.attempt ? 1 : selected.length * repetitions;
|
|
10477
|
+
if (args.includeActivity)
|
|
10478
|
+
await emit({
|
|
10479
|
+
type: "run-started",
|
|
10480
|
+
caseIds: selected.map((item) => item.id),
|
|
10481
|
+
repetitions,
|
|
10482
|
+
concurrency
|
|
10483
|
+
});
|
|
10484
|
+
const completed = new Array(total);
|
|
10485
|
+
let cursor = 0;
|
|
10486
|
+
await Promise.all(
|
|
10487
|
+
Array.from({ length: Math.min(concurrency, total) }, async () => {
|
|
10488
|
+
while (cursor < total) {
|
|
10489
|
+
const index = cursor++;
|
|
10490
|
+
const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
|
|
10491
|
+
completed[index] = await attempt(
|
|
10492
|
+
item,
|
|
10493
|
+
args.attempt?.repetition ?? index % repetitions + 1
|
|
12354
10494
|
);
|
|
12355
10495
|
}
|
|
12356
|
-
}
|
|
12357
|
-
executeAttempt
|
|
10496
|
+
})
|
|
12358
10497
|
);
|
|
12359
|
-
|
|
12360
|
-
|
|
12361
|
-
|
|
12362
|
-
|
|
12363
|
-
|
|
12364
|
-
|
|
12365
|
-
|
|
12366
|
-
|
|
12367
|
-
|
|
12368
|
-
|
|
12369
|
-
|
|
12370
|
-
|
|
12371
|
-
|
|
12372
|
-
|
|
12373
|
-
|
|
12374
|
-
const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
|
|
12375
|
-
completed[index] = await attempt(
|
|
12376
|
-
item,
|
|
12377
|
-
args.attempt?.repetition ?? index % repetitions + 1
|
|
12378
|
-
);
|
|
10498
|
+
result.cases = completed;
|
|
10499
|
+
for (const item of completed)
|
|
10500
|
+
result.counts[item.status] = (result.counts[item.status] ?? 0) + 1;
|
|
10501
|
+
result.status = statusFor(result.counts, result.errors);
|
|
10502
|
+
result.durationMs = Date.now() - start;
|
|
10503
|
+
const failedIssue = result.errors[0] ?? result.cases.find((item) => item.errors.length)?.errors[0];
|
|
10504
|
+
evalUsage.end(
|
|
10505
|
+
result.status === "passed" ? "success" : result.status === "failed" ? "quality_failed" : result.status === "error" ? "failed" : result.status,
|
|
10506
|
+
{
|
|
10507
|
+
scale: (0, import_usage_telemetry.scale)(selected.length * repetitions),
|
|
10508
|
+
phase: usagePhase,
|
|
10509
|
+
...result.status === "error" ? {
|
|
10510
|
+
...(0, import_usage_telemetry.classifyUsageError)(failedIssue).properties,
|
|
10511
|
+
eval_phase: failedIssue?.phase
|
|
10512
|
+
} : {}
|
|
12379
10513
|
}
|
|
12380
|
-
|
|
12381
|
-
|
|
12382
|
-
|
|
12383
|
-
|
|
12384
|
-
|
|
12385
|
-
|
|
12386
|
-
result.durationMs = Date.now() - start;
|
|
12387
|
-
evalUsage.end(
|
|
12388
|
-
result.status === "passed" ? "success" : result.status === "failed" ? "quality_failed" : result.status === "error" ? "failed" : result.status
|
|
12389
|
-
);
|
|
12390
|
-
return result;
|
|
10514
|
+
);
|
|
10515
|
+
return result;
|
|
10516
|
+
} catch (error) {
|
|
10517
|
+
evalUsage.fail(error, { phase: usagePhase });
|
|
10518
|
+
throw error;
|
|
10519
|
+
}
|
|
12391
10520
|
}
|
|
12392
10521
|
};
|
|
12393
10522
|
}
|
|
@@ -12398,23 +10527,24 @@ function defineSuite(suite) {
|
|
|
12398
10527
|
}
|
|
12399
10528
|
|
|
12400
10529
|
// src/evals/evaluation-contracts.ts
|
|
12401
|
-
var
|
|
10530
|
+
var import_zod13 = require("zod");
|
|
12402
10531
|
|
|
12403
10532
|
// src/evals/studio-contracts.ts
|
|
12404
10533
|
var import_browser3 = require("@kortyx/prompts/browser");
|
|
12405
|
-
var
|
|
12406
|
-
var StudioEvalTargetsResponseSchema =
|
|
12407
|
-
canRun:
|
|
10534
|
+
var import_zod12 = require("zod");
|
|
10535
|
+
var StudioEvalTargetsResponseSchema = import_zod12.z.object({
|
|
10536
|
+
canRun: import_zod12.z.boolean(),
|
|
10537
|
+
settings: EvalSettingsResponseSchema.optional(),
|
|
12408
10538
|
studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
|
|
12409
|
-
targets:
|
|
12410
|
-
|
|
12411
|
-
id:
|
|
12412
|
-
name:
|
|
12413
|
-
environment:
|
|
12414
|
-
error:
|
|
10539
|
+
targets: import_zod12.z.array(
|
|
10540
|
+
import_zod12.z.object({
|
|
10541
|
+
id: import_zod12.z.string(),
|
|
10542
|
+
name: import_zod12.z.string(),
|
|
10543
|
+
environment: import_zod12.z.string(),
|
|
10544
|
+
error: import_zod12.z.string().nullable(),
|
|
12415
10545
|
// Safe discovery diagnostics; optional for compatibility with older APIs.
|
|
12416
|
-
diagnostic:
|
|
12417
|
-
code:
|
|
10546
|
+
diagnostic: import_zod12.z.object({
|
|
10547
|
+
code: import_zod12.z.enum([
|
|
12418
10548
|
"environment_forbidden",
|
|
12419
10549
|
"environment_unavailable",
|
|
12420
10550
|
"endpoint_not_found",
|
|
@@ -12423,44 +10553,47 @@ var StudioEvalTargetsResponseSchema = import_zod11.z.object({
|
|
|
12423
10553
|
"endpoint_unreachable",
|
|
12424
10554
|
"manifest_invalid"
|
|
12425
10555
|
]),
|
|
12426
|
-
httpStatus:
|
|
10556
|
+
httpStatus: import_zod12.z.number().int().min(100).max(599).optional()
|
|
12427
10557
|
}).nullable().optional(),
|
|
12428
10558
|
manifest: EvalManifestSchema.nullable(),
|
|
12429
|
-
revisions:
|
|
10559
|
+
revisions: import_zod12.z.record(import_zod12.z.string(), import_zod12.z.string())
|
|
12430
10560
|
})
|
|
12431
10561
|
)
|
|
12432
10562
|
});
|
|
12433
|
-
var EvalCostAmountSchema =
|
|
12434
|
-
costSummary:
|
|
12435
|
-
amounts:
|
|
12436
|
-
|
|
12437
|
-
amount:
|
|
12438
|
-
currency:
|
|
10563
|
+
var EvalCostAmountSchema = import_zod12.z.object({
|
|
10564
|
+
costSummary: import_zod12.z.object({
|
|
10565
|
+
amounts: import_zod12.z.array(
|
|
10566
|
+
import_zod12.z.object({
|
|
10567
|
+
amount: import_zod12.z.number().finite().nonnegative(),
|
|
10568
|
+
currency: import_zod12.z.string().nullable()
|
|
12439
10569
|
})
|
|
12440
10570
|
),
|
|
12441
|
-
complete:
|
|
10571
|
+
complete: import_zod12.z.boolean()
|
|
12442
10572
|
}).optional(),
|
|
12443
|
-
amount:
|
|
12444
|
-
currency:
|
|
12445
|
-
status:
|
|
12446
|
-
calls:
|
|
12447
|
-
unpricedCalls:
|
|
12448
|
-
estimated:
|
|
10573
|
+
amount: import_zod12.z.number().finite().nonnegative().nullable(),
|
|
10574
|
+
currency: import_zod12.z.string().nullable(),
|
|
10575
|
+
status: import_zod12.z.enum(["complete", "partial", "unavailable"]),
|
|
10576
|
+
calls: import_zod12.z.number().int().nonnegative(),
|
|
10577
|
+
unpricedCalls: import_zod12.z.number().int().nonnegative(),
|
|
10578
|
+
estimated: import_zod12.z.boolean()
|
|
12449
10579
|
});
|
|
12450
|
-
var EvalCostsSchema =
|
|
10580
|
+
var EvalCostsSchema = import_zod12.z.object({
|
|
12451
10581
|
workflow: EvalCostAmountSchema,
|
|
12452
10582
|
judge: EvalCostAmountSchema,
|
|
12453
10583
|
total: EvalCostAmountSchema
|
|
12454
10584
|
});
|
|
12455
|
-
var StudioEvalRunSummarySchema =
|
|
12456
|
-
|
|
12457
|
-
|
|
12458
|
-
|
|
12459
|
-
|
|
12460
|
-
|
|
12461
|
-
|
|
12462
|
-
|
|
12463
|
-
|
|
10585
|
+
var StudioEvalRunSummarySchema = import_zod12.z.object({
|
|
10586
|
+
issue: EvalIssueSchema.nullable().optional(),
|
|
10587
|
+
termination: EvalTerminationSchema.nullable().optional(),
|
|
10588
|
+
resolvedSettings: ResolvedEvalSettingsSchema.optional(),
|
|
10589
|
+
id: import_zod12.z.uuid(),
|
|
10590
|
+
evaluationId: import_zod12.z.uuid().nullable().optional(),
|
|
10591
|
+
targetId: import_zod12.z.string(),
|
|
10592
|
+
targetName: import_zod12.z.string(),
|
|
10593
|
+
environment: import_zod12.z.string(),
|
|
10594
|
+
suiteId: import_zod12.z.string(),
|
|
10595
|
+
suiteRevision: import_zod12.z.string(),
|
|
10596
|
+
status: import_zod12.z.enum([
|
|
12464
10597
|
"queued",
|
|
12465
10598
|
"running",
|
|
12466
10599
|
"passed",
|
|
@@ -12468,49 +10601,52 @@ var StudioEvalRunSummarySchema = import_zod11.z.object({
|
|
|
12468
10601
|
"error",
|
|
12469
10602
|
"cancelled"
|
|
12470
10603
|
]),
|
|
12471
|
-
createdAt:
|
|
12472
|
-
startedAt:
|
|
12473
|
-
endedAt:
|
|
12474
|
-
error:
|
|
12475
|
-
cancelRequestedAt:
|
|
10604
|
+
createdAt: import_zod12.z.string(),
|
|
10605
|
+
startedAt: import_zod12.z.string().nullable(),
|
|
10606
|
+
endedAt: import_zod12.z.string().nullable(),
|
|
10607
|
+
error: import_zod12.z.string().nullable(),
|
|
10608
|
+
cancelRequestedAt: import_zod12.z.string().nullable(),
|
|
12476
10609
|
costs: EvalCostsSchema.optional(),
|
|
12477
|
-
counts:
|
|
12478
|
-
passed:
|
|
12479
|
-
failed:
|
|
12480
|
-
error:
|
|
12481
|
-
cancelled:
|
|
10610
|
+
counts: import_zod12.z.object({
|
|
10611
|
+
passed: import_zod12.z.number(),
|
|
10612
|
+
failed: import_zod12.z.number(),
|
|
10613
|
+
error: import_zod12.z.number(),
|
|
10614
|
+
cancelled: import_zod12.z.number()
|
|
12482
10615
|
}).nullable().optional(),
|
|
12483
|
-
suiteName:
|
|
12484
|
-
totalAttempts:
|
|
12485
|
-
completedAttempts:
|
|
12486
|
-
phase:
|
|
10616
|
+
suiteName: import_zod12.z.string().nullable().optional(),
|
|
10617
|
+
totalAttempts: import_zod12.z.number().int().nonnegative().optional(),
|
|
10618
|
+
completedAttempts: import_zod12.z.number().int().nonnegative().optional(),
|
|
10619
|
+
phase: import_zod12.z.string().optional()
|
|
12487
10620
|
});
|
|
12488
|
-
var StudioEvalHistorySchema =
|
|
12489
|
-
runs:
|
|
10621
|
+
var StudioEvalHistorySchema = import_zod12.z.object({
|
|
10622
|
+
runs: import_zod12.z.array(StudioEvalRunSummarySchema)
|
|
12490
10623
|
});
|
|
12491
|
-
var StudioEvalDetailSchema =
|
|
10624
|
+
var StudioEvalDetailSchema = import_zod12.z.object({
|
|
12492
10625
|
run: StudioEvalRunSummarySchema.extend({
|
|
12493
|
-
caseCosts:
|
|
12494
|
-
request:
|
|
12495
|
-
caseIds:
|
|
12496
|
-
grading:
|
|
10626
|
+
caseCosts: import_zod12.z.record(import_zod12.z.string(), EvalCostsSchema).optional(),
|
|
10627
|
+
request: import_zod12.z.object({
|
|
10628
|
+
caseIds: import_zod12.z.array(import_zod12.z.string()).optional(),
|
|
10629
|
+
grading: import_zod12.z.enum(["app", "studio"]).optional(),
|
|
12497
10630
|
judge: EvalJudgeIdentitySchema.optional(),
|
|
12498
|
-
repetitions:
|
|
12499
|
-
concurrency:
|
|
10631
|
+
repetitions: import_zod12.z.number().int().positive(),
|
|
10632
|
+
concurrency: import_zod12.z.number().optional(),
|
|
10633
|
+
caseTimeoutMs: import_zod12.z.number().optional(),
|
|
10634
|
+
runTimeoutMs: import_zod12.z.number().optional(),
|
|
10635
|
+
resolvedSettings: ResolvedEvalSettingsSchema.optional(),
|
|
12500
10636
|
promptSnapshot: import_browser3.PromptSnapshotSchema.optional(),
|
|
12501
|
-
promptGroupName:
|
|
10637
|
+
promptGroupName: import_zod12.z.string().optional()
|
|
12502
10638
|
}).optional(),
|
|
12503
10639
|
suite: EvalSuiteSchema,
|
|
12504
10640
|
result: EvalRunResultSchema.nullable(),
|
|
12505
|
-
events:
|
|
10641
|
+
events: import_zod12.z.array(import_zod12.z.object({ id: import_zod12.z.number(), event: EvalProgressSchema }))
|
|
12506
10642
|
})
|
|
12507
10643
|
});
|
|
12508
10644
|
|
|
12509
10645
|
// src/evals/evaluation-contracts.ts
|
|
12510
|
-
var EvaluationMetadataSchema =
|
|
12511
|
-
source:
|
|
12512
|
-
commit:
|
|
12513
|
-
deploymentUrl:
|
|
10646
|
+
var EvaluationMetadataSchema = import_zod13.z.object({
|
|
10647
|
+
source: import_zod13.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
|
|
10648
|
+
commit: import_zod13.z.string().trim().min(1).max(100).optional(),
|
|
10649
|
+
deploymentUrl: import_zod13.z.url().refine((value) => {
|
|
12514
10650
|
try {
|
|
12515
10651
|
const url = new URL(value);
|
|
12516
10652
|
return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
|
|
@@ -12519,67 +10655,84 @@ var EvaluationMetadataSchema = import_zod12.z.object({
|
|
|
12519
10655
|
}
|
|
12520
10656
|
}, "Expected an HTTP(S) URL without credentials").optional()
|
|
12521
10657
|
}).strict();
|
|
12522
|
-
var StudioEvaluationStartRequestSchema =
|
|
12523
|
-
targetId:
|
|
12524
|
-
name:
|
|
10658
|
+
var StudioEvaluationStartRequestSchema = import_zod13.z.object({
|
|
10659
|
+
targetId: import_zod13.z.string().trim().min(1),
|
|
10660
|
+
name: import_zod13.z.string().trim().min(1).max(200).optional(),
|
|
12525
10661
|
promptSelection: StudioEvalStartRequestSchema.shape.promptSelection,
|
|
12526
|
-
selection:
|
|
12527
|
-
suites:
|
|
12528
|
-
|
|
12529
|
-
suiteId:
|
|
12530
|
-
suiteRevision:
|
|
12531
|
-
caseIds:
|
|
10662
|
+
selection: import_zod13.z.enum(["all", "selected"]),
|
|
10663
|
+
suites: import_zod13.z.array(
|
|
10664
|
+
import_zod13.z.object({
|
|
10665
|
+
suiteId: import_zod13.z.string().min(1),
|
|
10666
|
+
suiteRevision: import_zod13.z.string().regex(/^[a-f0-9]{64}$/),
|
|
10667
|
+
caseIds: import_zod13.z.array(import_zod13.z.string().min(1)).min(1).max(100).optional()
|
|
12532
10668
|
}).strict()
|
|
12533
10669
|
).min(1).max(100),
|
|
12534
|
-
|
|
12535
|
-
repetitions: import_zod12.z.number().int().min(1).max(20).default(1),
|
|
12536
|
-
concurrency: import_zod12.z.number().int().min(1).max(20).default(1),
|
|
10670
|
+
...EvalSettingsSchema.shape,
|
|
12537
10671
|
metadata: EvaluationMetadataSchema.default({ source: "manual" }),
|
|
12538
|
-
idempotencyKey:
|
|
10672
|
+
idempotencyKey: import_zod13.z.string().trim().min(1).max(200).optional()
|
|
12539
10673
|
}).strict();
|
|
12540
|
-
var StudioEvaluationSummarySchema =
|
|
12541
|
-
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
10674
|
+
var StudioEvaluationSummarySchema = import_zod13.z.object({
|
|
10675
|
+
termination: EvalTerminationSchema.nullable().optional(),
|
|
10676
|
+
resolvedSettings: ResolvedEvalSettingsSchema.optional(),
|
|
10677
|
+
interruptedSuites: import_zod13.z.number().int().nonnegative().optional(),
|
|
10678
|
+
failures: import_zod13.z.array(
|
|
10679
|
+
import_zod13.z.object({
|
|
10680
|
+
suiteRunId: import_zod13.z.uuid(),
|
|
10681
|
+
caseId: import_zod13.z.string().optional(),
|
|
10682
|
+
repetition: import_zod13.z.number().int().positive().optional(),
|
|
10683
|
+
phase: import_zod13.z.string(),
|
|
10684
|
+
code: import_zod13.z.string(),
|
|
10685
|
+
message: import_zod13.z.string(),
|
|
10686
|
+
diagnosticId: import_zod13.z.string().uuid().optional()
|
|
10687
|
+
})
|
|
10688
|
+
).optional(),
|
|
10689
|
+
pendingAttempts: import_zod13.z.object({
|
|
10690
|
+
notStarted: import_zod13.z.number().int().nonnegative(),
|
|
10691
|
+
executing: import_zod13.z.number().int().nonnegative(),
|
|
10692
|
+
awaitingGrading: import_zod13.z.number().int().nonnegative()
|
|
10693
|
+
}).optional(),
|
|
10694
|
+
id: import_zod13.z.uuid(),
|
|
10695
|
+
name: import_zod13.z.string(),
|
|
10696
|
+
targetId: import_zod13.z.string(),
|
|
10697
|
+
targetName: import_zod13.z.string(),
|
|
10698
|
+
environment: import_zod13.z.string(),
|
|
10699
|
+
selection: import_zod13.z.enum(["all", "selected"]),
|
|
12547
10700
|
metadata: EvaluationMetadataSchema,
|
|
12548
10701
|
judge: EvalJudgeIdentitySchema.optional(),
|
|
12549
10702
|
status: StudioEvalRunSummarySchema.shape.status,
|
|
12550
|
-
createdAt:
|
|
12551
|
-
startedAt:
|
|
12552
|
-
endedAt:
|
|
12553
|
-
cancelRequestedAt:
|
|
12554
|
-
suiteCount:
|
|
12555
|
-
completedSuites:
|
|
12556
|
-
totalAttempts:
|
|
12557
|
-
completedAttempts:
|
|
12558
|
-
counts:
|
|
12559
|
-
passed:
|
|
12560
|
-
failed:
|
|
12561
|
-
error:
|
|
12562
|
-
cancelled:
|
|
10703
|
+
createdAt: import_zod13.z.string(),
|
|
10704
|
+
startedAt: import_zod13.z.string().nullable(),
|
|
10705
|
+
endedAt: import_zod13.z.string().nullable(),
|
|
10706
|
+
cancelRequestedAt: import_zod13.z.string().nullable(),
|
|
10707
|
+
suiteCount: import_zod13.z.number().int().nonnegative(),
|
|
10708
|
+
completedSuites: import_zod13.z.number().int().nonnegative(),
|
|
10709
|
+
totalAttempts: import_zod13.z.number().int().nonnegative(),
|
|
10710
|
+
completedAttempts: import_zod13.z.number().int().nonnegative(),
|
|
10711
|
+
counts: import_zod13.z.object({
|
|
10712
|
+
passed: import_zod13.z.number(),
|
|
10713
|
+
failed: import_zod13.z.number(),
|
|
10714
|
+
error: import_zod13.z.number(),
|
|
10715
|
+
cancelled: import_zod13.z.number()
|
|
12563
10716
|
}),
|
|
12564
10717
|
costs: EvalCostsSchema.optional(),
|
|
12565
|
-
legacy:
|
|
10718
|
+
legacy: import_zod13.z.boolean().optional()
|
|
12566
10719
|
});
|
|
12567
|
-
var StudioEvaluationHistorySchema =
|
|
12568
|
-
runs:
|
|
10720
|
+
var StudioEvaluationHistorySchema = import_zod13.z.object({
|
|
10721
|
+
runs: import_zod13.z.array(StudioEvaluationSummarySchema)
|
|
12569
10722
|
});
|
|
12570
|
-
var StudioEvaluationDetailSchema =
|
|
10723
|
+
var StudioEvaluationDetailSchema = import_zod13.z.object({
|
|
12571
10724
|
run: StudioEvaluationSummarySchema.extend({
|
|
12572
|
-
suites:
|
|
10725
|
+
suites: import_zod13.z.array(StudioEvalRunSummarySchema)
|
|
12573
10726
|
})
|
|
12574
10727
|
});
|
|
12575
|
-
var StudioEvaluationResultsSchema =
|
|
10728
|
+
var StudioEvaluationResultsSchema = import_zod13.z.object({
|
|
12576
10729
|
run: StudioEvaluationSummarySchema.extend({
|
|
12577
|
-
suites:
|
|
10730
|
+
suites: import_zod13.z.array(StudioEvalDetailSchema.shape.run)
|
|
12578
10731
|
})
|
|
12579
10732
|
});
|
|
12580
10733
|
|
|
12581
10734
|
// src/evals/judge.ts
|
|
12582
|
-
var
|
|
10735
|
+
var import_zod14 = require("zod");
|
|
12583
10736
|
function createEvalJudge({
|
|
12584
10737
|
model,
|
|
12585
10738
|
id: id3 = `${model.provider.id}/${model.modelId}`,
|
|
@@ -12606,7 +10759,7 @@ function createEvalJudge({
|
|
|
12606
10759
|
responseFormat: {
|
|
12607
10760
|
type: "json",
|
|
12608
10761
|
name: "eval_verdict",
|
|
12609
|
-
schema:
|
|
10762
|
+
schema: import_zod14.z.toJSONSchema(EvalVerdictSchema)
|
|
12610
10763
|
}
|
|
12611
10764
|
});
|
|
12612
10765
|
const result = await judge2.invoke([
|
|
@@ -12730,6 +10883,7 @@ function createEvalExecutionTransport({
|
|
|
12730
10883
|
await store.append(job.id, owner, {
|
|
12731
10884
|
type: "error",
|
|
12732
10885
|
message: issue2.message,
|
|
10886
|
+
exception: issue2.exception,
|
|
12733
10887
|
failure: issue2.failure,
|
|
12734
10888
|
...issue2.diagnosticId ? { diagnosticId: issue2.diagnosticId } : {}
|
|
12735
10889
|
});
|
|
@@ -12771,6 +10925,7 @@ function createEvalExecutionTransport({
|
|
|
12771
10925
|
suiteId: body.suiteId,
|
|
12772
10926
|
repetitions: body.repetitions,
|
|
12773
10927
|
concurrency: body.concurrency,
|
|
10928
|
+
...body.caseTimeoutMs !== void 0 ? { caseTimeoutMs: body.caseTimeoutMs } : {},
|
|
12774
10929
|
signal: controller.signal,
|
|
12775
10930
|
...body.attempt ? { attempt: body.attempt } : {},
|
|
12776
10931
|
...body.caseIds ? { caseIds: body.caseIds } : {},
|
|
@@ -12846,10 +11001,16 @@ function createEvalExecutionTransport({
|
|
|
12846
11001
|
"x-kortyx-eval-generation": await store.generation()
|
|
12847
11002
|
});
|
|
12848
11003
|
const problem = (error, negotiated) => {
|
|
11004
|
+
const issue2 = createEvalIssue("reporting", error);
|
|
12849
11005
|
const failure = (0, import_errors13.serializeFailure)(error);
|
|
12850
11006
|
const busy = failure.code === "EVAL_EXECUTOR_BUSY";
|
|
12851
11007
|
return Response.json(
|
|
12852
|
-
{
|
|
11008
|
+
{
|
|
11009
|
+
error: issue2.message,
|
|
11010
|
+
issue: issue2,
|
|
11011
|
+
failure,
|
|
11012
|
+
...busy ? { accepted: false } : {}
|
|
11013
|
+
},
|
|
12853
11014
|
{
|
|
12854
11015
|
status: busy ? negotiated ? 503 : 429 : !negotiated && failure.code === "EVAL_CONCURRENCY_CHANGED" ? 429 : failure.status ?? 503,
|
|
12855
11016
|
headers: {
|
|
@@ -13186,6 +11347,15 @@ function createEvalRouteHandler({
|
|
|
13186
11347
|
{ error: "Unsupported or invalid scheduled attempt." },
|
|
13187
11348
|
{ status: 409 }
|
|
13188
11349
|
);
|
|
11350
|
+
for (const key of ["concurrency", "caseTimeoutMs"]) {
|
|
11351
|
+
const maximum = manifest.execution?.limits[key];
|
|
11352
|
+
const value = body[key] ?? manifest.execution?.defaults[key];
|
|
11353
|
+
if (maximum !== void 0 && value !== void 0 && value > maximum)
|
|
11354
|
+
return Response.json(
|
|
11355
|
+
{ error: `${key} exceeds the application maximum of ${maximum}.` },
|
|
11356
|
+
{ status: 400 }
|
|
11357
|
+
);
|
|
11358
|
+
}
|
|
13189
11359
|
if (request.signal.aborted)
|
|
13190
11360
|
return Response.json({ error: "Request cancelled." }, { status: 409 });
|
|
13191
11361
|
return execution.submit(request, body);
|
|
@@ -13194,11 +11364,12 @@ function createEvalRouteHandler({
|
|
|
13194
11364
|
}
|
|
13195
11365
|
|
|
13196
11366
|
// src/evals/studio-judge.ts
|
|
11367
|
+
var import_telemetry_contracts3 = require("@kortyx/telemetry-contracts");
|
|
13197
11368
|
async function readJson(response, signal) {
|
|
13198
|
-
if (!response.
|
|
13199
|
-
|
|
13200
|
-
|
|
13201
|
-
|
|
11369
|
+
if (!response.body) {
|
|
11370
|
+
throw new Error(
|
|
11371
|
+
`Studio judge returned HTTP ${response.status} with no response body.`
|
|
11372
|
+
);
|
|
13202
11373
|
}
|
|
13203
11374
|
const reader = response.body.getReader();
|
|
13204
11375
|
const abort = () => {
|
|
@@ -13220,7 +11391,18 @@ async function readJson(response, signal) {
|
|
|
13220
11391
|
text += decoder.decode(part.value, { stream: true });
|
|
13221
11392
|
}
|
|
13222
11393
|
signal.throwIfAborted();
|
|
13223
|
-
|
|
11394
|
+
text += decoder.decode();
|
|
11395
|
+
if (!response.ok)
|
|
11396
|
+
throw Object.assign(
|
|
11397
|
+
new Error(
|
|
11398
|
+
`Studio judge returned HTTP ${response.status}: ${(0, import_telemetry_contracts3.redactDiagnosticText)(text)}`
|
|
11399
|
+
),
|
|
11400
|
+
{
|
|
11401
|
+
statusCode: response.status,
|
|
11402
|
+
responseBody: (0, import_telemetry_contracts3.redactDiagnosticText)(text)
|
|
11403
|
+
}
|
|
11404
|
+
);
|
|
11405
|
+
return JSON.parse(text);
|
|
13224
11406
|
} finally {
|
|
13225
11407
|
signal.removeEventListener("abort", abort);
|
|
13226
11408
|
await reader.cancel().catch(() => {
|
|
@@ -13304,6 +11486,7 @@ async function createStudioEvalJudge(options) {
|
|
|
13304
11486
|
ChatLifecycleHookError,
|
|
13305
11487
|
EvalConfigurationError,
|
|
13306
11488
|
EvalEvidencePolicySchema,
|
|
11489
|
+
EvalExecutionSettingsSchema,
|
|
13307
11490
|
EvalIssueSchema,
|
|
13308
11491
|
EvalJudgeIdentitySchema,
|
|
13309
11492
|
EvalJudgeUsageSchema,
|
|
@@ -13311,10 +11494,14 @@ async function createStudioEvalJudge(options) {
|
|
|
13311
11494
|
EvalProgressSchema,
|
|
13312
11495
|
EvalRemoteRunRequestSchema,
|
|
13313
11496
|
EvalRunResultSchema,
|
|
11497
|
+
EvalSettingsResponseSchema,
|
|
11498
|
+
EvalSettingsSchema,
|
|
13314
11499
|
EvalSuiteSchema,
|
|
11500
|
+
EvalTerminationSchema,
|
|
13315
11501
|
EvalWireEventSchema,
|
|
13316
11502
|
EvaluationMetadataSchema,
|
|
13317
11503
|
ExecutionRequestError,
|
|
11504
|
+
ResolvedEvalSettingsSchema,
|
|
13318
11505
|
StudioEvalJudgeRequestSchema,
|
|
13319
11506
|
StudioEvalJudgeResponseSchema,
|
|
13320
11507
|
StudioEvalStartRequestSchema,
|
|
@@ -13345,6 +11532,7 @@ async function createStudioEvalJudge(options) {
|
|
|
13345
11532
|
parseResumeMeta,
|
|
13346
11533
|
projectWorkflowTopology,
|
|
13347
11534
|
readRequestJson,
|
|
11535
|
+
resolveEvalSettings,
|
|
13348
11536
|
streamChat,
|
|
13349
11537
|
streamChatFromRoute,
|
|
13350
11538
|
transformGraphStreamForUI,
|