@kortyx/agent 0.34.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/dist/evals.d.ts +134 -12
- package/dist/evals.js +57 -3
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +19 -5
- package/dist/index.js +1709 -692
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-DpRLvf8j.d.ts → studio-judge-Brk9R5oB.d.ts} +464 -41
- package/package.json +10 -8
package/dist/index.js
CHANGED
|
@@ -37,8 +37,8 @@ var require_dist = __commonJS({
|
|
|
37
37
|
var __getOwnPropNames2 = Object.getOwnPropertyNames;
|
|
38
38
|
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
|
|
39
39
|
var __export2 = (target, all) => {
|
|
40
|
-
for (var
|
|
41
|
-
__defProp2(target,
|
|
40
|
+
for (var name2 in all)
|
|
41
|
+
__defProp2(target, name2, { get: all[name2], enumerable: true });
|
|
42
42
|
};
|
|
43
43
|
var __copyProps2 = (to, from, except, desc) => {
|
|
44
44
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
@@ -65,6 +65,20 @@ var require_dist = __commonJS({
|
|
|
65
65
|
FEEDBACK_FILTERS: () => FEEDBACK_FILTERS,
|
|
66
66
|
FeedbackReasonSchema: () => FeedbackReasonSchema,
|
|
67
67
|
FeedbackSummarySchema: () => FeedbackSummarySchema,
|
|
68
|
+
PromptAssetSchema: () => PromptAssetSchema,
|
|
69
|
+
PromptAssignmentSchema: () => PromptAssignmentSchema,
|
|
70
|
+
PromptBundleSchema: () => PromptBundleSchema,
|
|
71
|
+
PromptCategorySchema: () => PromptCategorySchema,
|
|
72
|
+
PromptDetailSchema: () => PromptDetailSchema,
|
|
73
|
+
PromptEvidenceSchema: () => PromptEvidenceSchema,
|
|
74
|
+
PromptGroupSchema: () => PromptGroupSchema,
|
|
75
|
+
PromptLibrarySchema: () => PromptLibrarySchema,
|
|
76
|
+
PromptMutationSchema: () => PromptMutationSchema,
|
|
77
|
+
PromptPolicySchema: () => PromptPolicySchema,
|
|
78
|
+
PromptSelectionSchema: () => PromptSelectionSchema,
|
|
79
|
+
PromptStoredVersionSchema: () => PromptStoredVersionSchema,
|
|
80
|
+
PromptTransferRequestSchema: () => PromptTransferRequestSchema,
|
|
81
|
+
PromptUsageSchema: () => PromptUsageSchema,
|
|
68
82
|
ReviewVerdictSchema: () => ReviewVerdictSchema,
|
|
69
83
|
STUDIO_API_PROTOCOL_VERSION: () => STUDIO_API_PROTOCOL_VERSION,
|
|
70
84
|
STUDIO_TIME_RANGES: () => STUDIO_TIME_RANGES,
|
|
@@ -145,7 +159,7 @@ var require_dist = __commonJS({
|
|
|
145
159
|
summarizeUserFeedback: () => summarizeUserFeedback
|
|
146
160
|
});
|
|
147
161
|
module2.exports = __toCommonJS2(index_exports2);
|
|
148
|
-
var
|
|
162
|
+
var import_zod62 = require("zod");
|
|
149
163
|
var import_zod12 = require("zod");
|
|
150
164
|
var FEEDBACK_FILTERS = ["positive", "negative", "unrated"];
|
|
151
165
|
var FeedbackSummarySchema = import_zod12.z.object({
|
|
@@ -467,30 +481,334 @@ var require_dist = __commonJS({
|
|
|
467
481
|
p50DurationMs: import_zod32.z.number().nonnegative().nullable(),
|
|
468
482
|
p95DurationMs: import_zod32.z.number().nonnegative().nullable()
|
|
469
483
|
}).strict();
|
|
484
|
+
var import_browser4 = require("@kortyx/prompts/browser");
|
|
470
485
|
var import_zod42 = require("zod");
|
|
471
|
-
var
|
|
472
|
-
var
|
|
473
|
-
|
|
486
|
+
var name = import_zod42.z.string().trim().min(1).max(200);
|
|
487
|
+
var version = import_zod42.z.number().int().positive();
|
|
488
|
+
var revision = import_zod42.z.number().int().nonnegative();
|
|
489
|
+
var PromptCategorySchema = import_zod42.z.object({
|
|
490
|
+
id: import_zod42.z.uuid(),
|
|
491
|
+
name,
|
|
492
|
+
parentId: import_zod42.z.uuid().nullable(),
|
|
493
|
+
revision
|
|
494
|
+
});
|
|
495
|
+
var PromptGroupSchema = import_zod42.z.object({
|
|
496
|
+
id: import_zod42.z.uuid(),
|
|
497
|
+
name,
|
|
498
|
+
revision,
|
|
499
|
+
members: import_zod42.z.array(
|
|
500
|
+
import_zod42.z.object({
|
|
501
|
+
promptId: import_zod42.z.uuid(),
|
|
502
|
+
version,
|
|
503
|
+
key: import_browser4.PromptKeySchema.optional(),
|
|
504
|
+
name: name.optional(),
|
|
505
|
+
archived: import_zod42.z.boolean().optional()
|
|
506
|
+
})
|
|
507
|
+
).max(100),
|
|
508
|
+
updatedAt: import_zod42.z.string()
|
|
509
|
+
});
|
|
510
|
+
var PromptAssignmentSchema = import_zod42.z.object({
|
|
511
|
+
tag: import_browser4.PromptTagSchema,
|
|
512
|
+
version,
|
|
513
|
+
revision,
|
|
514
|
+
updatedAt: import_zod42.z.string()
|
|
515
|
+
});
|
|
516
|
+
var PromptAssetSchema = import_zod42.z.object({
|
|
517
|
+
id: import_zod42.z.uuid(),
|
|
518
|
+
key: import_browser4.PromptKeySchema,
|
|
519
|
+
name,
|
|
520
|
+
categoryId: import_zod42.z.uuid().nullable(),
|
|
521
|
+
latestVersion: version,
|
|
522
|
+
revision,
|
|
523
|
+
archived: import_zod42.z.boolean(),
|
|
524
|
+
updatedAt: import_zod42.z.string(),
|
|
525
|
+
assignments: import_zod42.z.array(PromptAssignmentSchema)
|
|
526
|
+
});
|
|
527
|
+
var PromptLibrarySchema = import_zod42.z.object({
|
|
528
|
+
schemaVersion: import_zod42.z.literal(1),
|
|
529
|
+
assets: import_zod42.z.array(PromptAssetSchema),
|
|
530
|
+
categories: import_zod42.z.array(PromptCategorySchema),
|
|
531
|
+
groups: import_zod42.z.array(PromptGroupSchema),
|
|
532
|
+
totalCount: import_zod42.z.number().int().nonnegative(),
|
|
533
|
+
nextCursor: import_zod42.z.string().nullable(),
|
|
534
|
+
permissions: import_zod42.z.object({
|
|
535
|
+
edit: import_zod42.z.boolean(),
|
|
536
|
+
promote: import_zod42.z.boolean(),
|
|
537
|
+
review: import_zod42.z.boolean(),
|
|
538
|
+
settings: import_zod42.z.boolean()
|
|
539
|
+
})
|
|
540
|
+
});
|
|
541
|
+
var PromptStoredVersionSchema = import_browser4.PromptVersionSchema.extend({
|
|
542
|
+
promptId: import_zod42.z.uuid(),
|
|
543
|
+
note: import_zod42.z.string(),
|
|
544
|
+
author: import_zod42.z.string(),
|
|
545
|
+
createdAt: import_zod42.z.string(),
|
|
546
|
+
origin: import_zod42.z.object({
|
|
547
|
+
apiUrl: import_zod42.z.string(),
|
|
548
|
+
projectId: import_zod42.z.string(),
|
|
549
|
+
key: import_zod42.z.string(),
|
|
550
|
+
version,
|
|
551
|
+
hash: import_zod42.z.string()
|
|
552
|
+
}).nullable()
|
|
553
|
+
});
|
|
554
|
+
var PromptPolicySchema = import_zod42.z.object({
|
|
555
|
+
revision,
|
|
556
|
+
requireTest: import_zod42.z.boolean(),
|
|
557
|
+
requireChangeNote: import_zod42.z.boolean().default(false),
|
|
558
|
+
requiredSuites: import_zod42.z.array(import_zod42.z.object({ targetId: name, suiteId: name })).max(100),
|
|
559
|
+
requiredReviews: import_zod42.z.number().int().min(0).max(10),
|
|
560
|
+
allowException: import_zod42.z.boolean()
|
|
561
|
+
});
|
|
562
|
+
var PromptEvidenceSchema = import_zod42.z.object({
|
|
563
|
+
environment: import_zod42.z.string(),
|
|
564
|
+
runId: import_zod42.z.string(),
|
|
565
|
+
evaluationId: import_zod42.z.string().nullable().optional(),
|
|
566
|
+
targetId: import_zod42.z.string(),
|
|
567
|
+
suiteId: import_zod42.z.string(),
|
|
568
|
+
suiteRevision: import_zod42.z.string().optional(),
|
|
569
|
+
status: import_zod42.z.string(),
|
|
570
|
+
version,
|
|
571
|
+
fullSuite: import_zod42.z.boolean(),
|
|
572
|
+
usage: import_zod42.z.enum([
|
|
573
|
+
"verified",
|
|
574
|
+
"pending",
|
|
575
|
+
"not-used",
|
|
576
|
+
"mismatch",
|
|
577
|
+
"context-differs"
|
|
578
|
+
]),
|
|
579
|
+
groupName: import_zod42.z.string().nullable(),
|
|
580
|
+
createdAt: import_zod42.z.string(),
|
|
581
|
+
companions: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version }))
|
|
582
|
+
});
|
|
583
|
+
var PromptUsageSchema = import_zod42.z.object({
|
|
584
|
+
eventId: import_zod42.z.string(),
|
|
585
|
+
runId: import_zod42.z.string(),
|
|
586
|
+
sessionId: import_zod42.z.string().nullable(),
|
|
587
|
+
nodeId: import_zod42.z.string().nullable(),
|
|
588
|
+
version: import_zod42.z.number().int(),
|
|
589
|
+
hash: import_zod42.z.string().nullable(),
|
|
590
|
+
source: import_zod42.z.string().nullable(),
|
|
591
|
+
model: import_zod42.z.string().nullable(),
|
|
592
|
+
environment: import_zod42.z.string(),
|
|
593
|
+
occurredAt: import_zod42.z.string(),
|
|
594
|
+
captured: import_zod42.z.boolean()
|
|
595
|
+
});
|
|
596
|
+
var PromptDetailSchema = import_zod42.z.object({
|
|
597
|
+
asset: PromptAssetSchema,
|
|
598
|
+
versions: import_zod42.z.array(PromptStoredVersionSchema),
|
|
599
|
+
versionsNextCursor: import_zod42.z.string().nullable().optional(),
|
|
600
|
+
draft: import_browser4.PromptContentSchema.nullable(),
|
|
601
|
+
draftBase: import_zod42.z.number().int().nullable(),
|
|
602
|
+
draftRevision: revision,
|
|
603
|
+
evidence: import_zod42.z.array(PromptEvidenceSchema),
|
|
604
|
+
usage: import_zod42.z.array(PromptUsageSchema),
|
|
605
|
+
activity: import_zod42.z.array(
|
|
606
|
+
import_zod42.z.object({
|
|
607
|
+
id: import_zod42.z.string(),
|
|
608
|
+
action: import_zod42.z.string(),
|
|
609
|
+
actor: import_zod42.z.string(),
|
|
610
|
+
details: import_zod42.z.record(import_zod42.z.string(), import_zod42.z.unknown()),
|
|
611
|
+
createdAt: import_zod42.z.string()
|
|
612
|
+
})
|
|
613
|
+
),
|
|
614
|
+
policies: import_zod42.z.array(PromptPolicySchema),
|
|
615
|
+
reviews: import_zod42.z.array(
|
|
616
|
+
import_zod42.z.object({
|
|
617
|
+
version,
|
|
618
|
+
hash: import_zod42.z.string(),
|
|
619
|
+
independent: import_zod42.z.boolean(),
|
|
620
|
+
reviewer: import_zod42.z.string(),
|
|
621
|
+
note: import_zod42.z.string(),
|
|
622
|
+
createdAt: import_zod42.z.string()
|
|
623
|
+
})
|
|
624
|
+
).optional()
|
|
625
|
+
});
|
|
626
|
+
var PromptMutationSchema = import_zod42.z.discriminatedUnion("action", [
|
|
627
|
+
import_zod42.z.object({
|
|
628
|
+
action: import_zod42.z.literal("tag-set"),
|
|
629
|
+
id: import_zod42.z.uuid(),
|
|
630
|
+
tag: import_browser4.PromptTagSchema.refine(
|
|
631
|
+
(tag) => tag !== "live",
|
|
632
|
+
"Use promotion to update live."
|
|
633
|
+
),
|
|
634
|
+
version,
|
|
635
|
+
expectedRevision: revision
|
|
636
|
+
}).strict(),
|
|
637
|
+
import_zod42.z.object({
|
|
638
|
+
action: import_zod42.z.literal("tag-remove"),
|
|
639
|
+
id: import_zod42.z.uuid(),
|
|
640
|
+
tag: import_browser4.PromptTagSchema.refine(
|
|
641
|
+
(tag) => tag !== "live",
|
|
642
|
+
"The live tag cannot be removed manually."
|
|
643
|
+
),
|
|
644
|
+
expectedRevision: revision
|
|
645
|
+
}).strict(),
|
|
646
|
+
import_zod42.z.object({
|
|
647
|
+
action: import_zod42.z.literal("bulk-update"),
|
|
648
|
+
assets: import_zod42.z.array(import_zod42.z.object({ id: import_zod42.z.uuid(), expectedRevision: revision })).min(1).max(100),
|
|
649
|
+
categoryId: import_zod42.z.uuid().nullable().optional(),
|
|
650
|
+
archived: import_zod42.z.boolean().optional()
|
|
651
|
+
}).strict().refine(
|
|
652
|
+
(value) => value.categoryId !== void 0 || value.archived !== void 0,
|
|
653
|
+
"Choose a category or archive state."
|
|
654
|
+
),
|
|
655
|
+
import_zod42.z.object({
|
|
656
|
+
action: import_zod42.z.literal("create"),
|
|
657
|
+
key: import_browser4.PromptKeySchema,
|
|
658
|
+
name,
|
|
659
|
+
categoryId: import_zod42.z.uuid().nullable().default(null),
|
|
660
|
+
content: import_browser4.PromptContentSchema,
|
|
661
|
+
note: import_zod42.z.string().trim().max(2e3).default(""),
|
|
662
|
+
idempotencyKey: import_zod42.z.uuid().optional()
|
|
663
|
+
}).strict(),
|
|
664
|
+
import_zod42.z.object({
|
|
665
|
+
action: import_zod42.z.literal("draft"),
|
|
666
|
+
id: import_zod42.z.uuid(),
|
|
667
|
+
content: import_browser4.PromptContentSchema,
|
|
668
|
+
baseVersion: version,
|
|
669
|
+
expectedRevision: revision
|
|
670
|
+
}).strict(),
|
|
671
|
+
import_zod42.z.object({
|
|
672
|
+
action: import_zod42.z.literal("discard-draft"),
|
|
673
|
+
id: import_zod42.z.uuid(),
|
|
674
|
+
expectedRevision: revision
|
|
675
|
+
}).strict(),
|
|
676
|
+
import_zod42.z.object({
|
|
677
|
+
action: import_zod42.z.literal("save"),
|
|
678
|
+
id: import_zod42.z.uuid(),
|
|
679
|
+
content: import_browser4.PromptContentSchema,
|
|
680
|
+
baseVersion: version,
|
|
681
|
+
expectedHash: import_zod42.z.string().regex(/^[a-f0-9]{64}$/),
|
|
682
|
+
expectedDraftRevision: revision.optional(),
|
|
683
|
+
note: import_zod42.z.string().trim().max(2e3).default(""),
|
|
684
|
+
idempotencyKey: import_zod42.z.uuid()
|
|
685
|
+
}).strict(),
|
|
686
|
+
import_zod42.z.object({
|
|
687
|
+
action: import_zod42.z.literal("update"),
|
|
688
|
+
id: import_zod42.z.uuid(),
|
|
689
|
+
expectedRevision: revision,
|
|
690
|
+
name: name.optional(),
|
|
691
|
+
categoryId: import_zod42.z.uuid().nullable().optional(),
|
|
692
|
+
archived: import_zod42.z.boolean().optional()
|
|
693
|
+
}).strict(),
|
|
694
|
+
import_zod42.z.object({
|
|
695
|
+
action: import_zod42.z.literal("category-create"),
|
|
696
|
+
path: import_zod42.z.string().trim().min(1).max(1e3)
|
|
697
|
+
}).strict(),
|
|
698
|
+
import_zod42.z.object({
|
|
699
|
+
action: import_zod42.z.literal("category-update"),
|
|
700
|
+
id: import_zod42.z.uuid(),
|
|
701
|
+
expectedRevision: revision,
|
|
702
|
+
name: name.optional(),
|
|
703
|
+
parentId: import_zod42.z.uuid().nullable().optional()
|
|
704
|
+
}).strict(),
|
|
705
|
+
import_zod42.z.object({
|
|
706
|
+
action: import_zod42.z.literal("category-delete"),
|
|
707
|
+
id: import_zod42.z.uuid(),
|
|
708
|
+
expectedRevision: revision,
|
|
709
|
+
destinationId: import_zod42.z.uuid().nullable()
|
|
710
|
+
}).strict(),
|
|
711
|
+
import_zod42.z.object({
|
|
712
|
+
action: import_zod42.z.literal("group-create"),
|
|
713
|
+
name,
|
|
714
|
+
members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
|
|
715
|
+
}).strict(),
|
|
716
|
+
import_zod42.z.object({
|
|
717
|
+
action: import_zod42.z.literal("group-update"),
|
|
718
|
+
id: import_zod42.z.uuid(),
|
|
719
|
+
expectedRevision: revision,
|
|
720
|
+
name: name.optional(),
|
|
721
|
+
members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
|
|
722
|
+
}).strict(),
|
|
723
|
+
import_zod42.z.object({
|
|
724
|
+
action: import_zod42.z.literal("group-delete"),
|
|
725
|
+
id: import_zod42.z.uuid(),
|
|
726
|
+
expectedRevision: revision
|
|
727
|
+
}).strict(),
|
|
728
|
+
import_zod42.z.object({
|
|
729
|
+
action: import_zod42.z.literal("promote"),
|
|
730
|
+
id: import_zod42.z.uuid(),
|
|
731
|
+
version,
|
|
732
|
+
expectedRevision: revision,
|
|
733
|
+
exceptionReason: import_zod42.z.string().trim().min(1).max(2e3).optional(),
|
|
734
|
+
rollback: import_zod42.z.boolean().default(false)
|
|
735
|
+
}).strict(),
|
|
736
|
+
import_zod42.z.object({
|
|
737
|
+
action: import_zod42.z.literal("review"),
|
|
738
|
+
id: import_zod42.z.uuid(),
|
|
739
|
+
version,
|
|
740
|
+
note: import_zod42.z.string().trim().min(1).max(2e3)
|
|
741
|
+
}).strict(),
|
|
742
|
+
import_zod42.z.object({
|
|
743
|
+
action: import_zod42.z.literal("policy"),
|
|
744
|
+
id: import_zod42.z.uuid(),
|
|
745
|
+
policy: PromptPolicySchema
|
|
746
|
+
}).strict()
|
|
747
|
+
]);
|
|
748
|
+
var PromptSelectionSchema = import_zod42.z.discriminatedUnion("type", [
|
|
749
|
+
import_zod42.z.object({ type: import_zod42.z.literal("live") }).strict(),
|
|
750
|
+
import_zod42.z.object({
|
|
751
|
+
type: import_zod42.z.literal("single"),
|
|
752
|
+
id: import_zod42.z.string().min(1),
|
|
753
|
+
version: import_zod42.z.number().int().positive()
|
|
754
|
+
}).strict(),
|
|
755
|
+
import_zod42.z.object({ type: import_zod42.z.literal("group"), groupId: import_zod42.z.uuid() }).strict()
|
|
756
|
+
]);
|
|
757
|
+
var PromptBundleSchema = import_zod42.z.object({
|
|
758
|
+
schemaVersion: import_zod42.z.literal(1),
|
|
759
|
+
origin: import_zod42.z.object({
|
|
760
|
+
apiUrl: import_zod42.z.url().refine((value) => {
|
|
761
|
+
const url = new URL(value);
|
|
762
|
+
return /^https?:$/.test(url.protocol) && !url.username && !url.password && !url.search;
|
|
763
|
+
}),
|
|
764
|
+
projectId: import_zod42.z.string().min(1)
|
|
765
|
+
}).strict(),
|
|
766
|
+
prompts: import_zod42.z.array(
|
|
767
|
+
import_zod42.z.object({
|
|
768
|
+
key: import_browser4.PromptKeySchema,
|
|
769
|
+
name,
|
|
770
|
+
categoryPath: import_zod42.z.string().optional(),
|
|
771
|
+
versions: import_zod42.z.array(import_browser4.PromptVersionSchema.extend({ note: import_zod42.z.string().max(2e3) })).min(1).max(200)
|
|
772
|
+
}).strict()
|
|
773
|
+
).min(1).max(200),
|
|
774
|
+
groups: import_zod42.z.array(
|
|
775
|
+
import_zod42.z.object({
|
|
776
|
+
name,
|
|
777
|
+
members: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version })).max(100)
|
|
778
|
+
}).strict()
|
|
779
|
+
).max(100).default([])
|
|
780
|
+
}).strict();
|
|
781
|
+
var PromptTransferRequestSchema = import_zod42.z.object({
|
|
782
|
+
bundle: PromptBundleSchema,
|
|
783
|
+
conflicts: import_zod42.z.enum(["error", "append"]).default("error"),
|
|
784
|
+
rename: import_zod42.z.record(import_browser4.PromptKeySchema, import_browser4.PromptKeySchema).default({}),
|
|
785
|
+
categories: import_zod42.z.boolean().default(true),
|
|
786
|
+
groups: import_zod42.z.boolean().default(false)
|
|
787
|
+
}).strict();
|
|
788
|
+
var import_zod52 = require("zod");
|
|
789
|
+
var StudioReleaseVersionSchema = import_zod52.z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/);
|
|
790
|
+
var StudioReleaseDeploymentSchema = import_zod52.z.object({
|
|
791
|
+
strategy: import_zod52.z.enum(["rolling", "recreate"])
|
|
474
792
|
});
|
|
475
|
-
var StudioReleaseSchema =
|
|
476
|
-
format:
|
|
477
|
-
installer:
|
|
793
|
+
var StudioReleaseSchema = import_zod52.z.object({
|
|
794
|
+
format: import_zod52.z.literal(1),
|
|
795
|
+
installer: import_zod52.z.literal(1),
|
|
478
796
|
version: StudioReleaseVersionSchema,
|
|
479
|
-
api:
|
|
480
|
-
studio:
|
|
797
|
+
api: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-api@sha256:[a-f0-9]{64}$/),
|
|
798
|
+
studio: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-studio@sha256:[a-f0-9]{64}$/),
|
|
481
799
|
deployment: StudioReleaseDeploymentSchema.default({
|
|
482
800
|
strategy: "recreate"
|
|
483
801
|
})
|
|
484
802
|
});
|
|
485
|
-
var StudioUpdateSettingsSchema =
|
|
486
|
-
automatic:
|
|
487
|
-
hourUtc:
|
|
803
|
+
var StudioUpdateSettingsSchema = import_zod52.z.object({
|
|
804
|
+
automatic: import_zod52.z.boolean(),
|
|
805
|
+
hourUtc: import_zod52.z.number().int().min(0).max(23)
|
|
488
806
|
});
|
|
489
|
-
var StudioUpdateOperationSchema =
|
|
490
|
-
id:
|
|
491
|
-
from:
|
|
807
|
+
var StudioUpdateOperationSchema = import_zod52.z.object({
|
|
808
|
+
id: import_zod52.z.uuid(),
|
|
809
|
+
from: import_zod52.z.string(),
|
|
492
810
|
release: StudioReleaseSchema,
|
|
493
|
-
phase:
|
|
811
|
+
phase: import_zod52.z.enum([
|
|
494
812
|
"starting",
|
|
495
813
|
"pulling",
|
|
496
814
|
"backup",
|
|
@@ -499,16 +817,16 @@ var require_dist = __commonJS({
|
|
|
499
817
|
"succeeded",
|
|
500
818
|
"failed"
|
|
501
819
|
]),
|
|
502
|
-
startedAt:
|
|
503
|
-
updatedAt:
|
|
504
|
-
backup:
|
|
505
|
-
messages:
|
|
820
|
+
startedAt: import_zod52.z.iso.datetime(),
|
|
821
|
+
updatedAt: import_zod52.z.iso.datetime(),
|
|
822
|
+
backup: import_zod52.z.string().nullable(),
|
|
823
|
+
messages: import_zod52.z.array(import_zod52.z.string()).max(100)
|
|
506
824
|
});
|
|
507
|
-
var StudioUpdateStatusSchema =
|
|
508
|
-
current:
|
|
825
|
+
var StudioUpdateStatusSchema = import_zod52.z.object({
|
|
826
|
+
current: import_zod52.z.string().nullable(),
|
|
509
827
|
available: StudioReleaseSchema.nullable(),
|
|
510
|
-
checkedAt:
|
|
511
|
-
checkError:
|
|
828
|
+
checkedAt: import_zod52.z.iso.datetime().nullable(),
|
|
829
|
+
checkError: import_zod52.z.string().nullable(),
|
|
512
830
|
settings: StudioUpdateSettingsSchema,
|
|
513
831
|
operation: StudioUpdateOperationSchema.nullable()
|
|
514
832
|
});
|
|
@@ -659,82 +977,82 @@ var require_dist = __commonJS({
|
|
|
659
977
|
"session.forked",
|
|
660
978
|
"session.rolled_back"
|
|
661
979
|
];
|
|
662
|
-
var TelemetryServiceSchema =
|
|
663
|
-
name:
|
|
664
|
-
deploymentRef:
|
|
980
|
+
var TelemetryServiceSchema = import_zod62.z.object({
|
|
981
|
+
name: import_zod62.z.string().min(1),
|
|
982
|
+
deploymentRef: import_zod62.z.string().min(1).optional()
|
|
665
983
|
}).strict();
|
|
666
|
-
var WorkflowTopologyNodeSchema =
|
|
667
|
-
id:
|
|
668
|
-
label:
|
|
669
|
-
type:
|
|
670
|
-
provider:
|
|
671
|
-
model:
|
|
672
|
-
metadata:
|
|
673
|
-
tools:
|
|
984
|
+
var WorkflowTopologyNodeSchema = import_zod62.z.object({
|
|
985
|
+
id: import_zod62.z.string().min(1),
|
|
986
|
+
label: import_zod62.z.string().optional(),
|
|
987
|
+
type: import_zod62.z.string().optional(),
|
|
988
|
+
provider: import_zod62.z.string().optional(),
|
|
989
|
+
model: import_zod62.z.string().optional(),
|
|
990
|
+
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional(),
|
|
991
|
+
tools: import_zod62.z.array(WorkflowToolSchema).optional(),
|
|
674
992
|
toolDiscovery: ToolDiscoverySchema.optional()
|
|
675
993
|
}).strict();
|
|
676
|
-
var WorkflowTopologyEdgeSchema =
|
|
677
|
-
sourceNodeId:
|
|
678
|
-
targetNodeId:
|
|
679
|
-
condition:
|
|
994
|
+
var WorkflowTopologyEdgeSchema = import_zod62.z.object({
|
|
995
|
+
sourceNodeId: import_zod62.z.string().min(1),
|
|
996
|
+
targetNodeId: import_zod62.z.string().min(1),
|
|
997
|
+
condition: import_zod62.z.string().optional()
|
|
680
998
|
}).strict();
|
|
681
|
-
var WorkflowTopologyCallSchema =
|
|
682
|
-
sourceNodeId:
|
|
683
|
-
targetWorkflowId:
|
|
999
|
+
var WorkflowTopologyCallSchema = import_zod62.z.object({
|
|
1000
|
+
sourceNodeId: import_zod62.z.string().min(1),
|
|
1001
|
+
targetWorkflowId: import_zod62.z.string().min(1)
|
|
684
1002
|
}).strict();
|
|
685
|
-
var WorkflowTopologyTransitionSchema =
|
|
686
|
-
sourceNodeId:
|
|
687
|
-
targetWorkflowId:
|
|
688
|
-
condition:
|
|
689
|
-
intent:
|
|
690
|
-
kind:
|
|
1003
|
+
var WorkflowTopologyTransitionSchema = import_zod62.z.object({
|
|
1004
|
+
sourceNodeId: import_zod62.z.string().min(1).optional(),
|
|
1005
|
+
targetWorkflowId: import_zod62.z.string().min(1),
|
|
1006
|
+
condition: import_zod62.z.string().optional(),
|
|
1007
|
+
intent: import_zod62.z.string().optional(),
|
|
1008
|
+
kind: import_zod62.z.enum(["call", "handoff"]).optional()
|
|
691
1009
|
}).strict();
|
|
692
|
-
var EnsureWorkflowTopologyRequestSchema =
|
|
693
|
-
schemaVersion:
|
|
694
|
-
environment:
|
|
1010
|
+
var EnsureWorkflowTopologyRequestSchema = import_zod62.z.object({
|
|
1011
|
+
schemaVersion: import_zod62.z.literal(1),
|
|
1012
|
+
environment: import_zod62.z.string().min(1),
|
|
695
1013
|
service: TelemetryServiceSchema,
|
|
696
|
-
workflow:
|
|
697
|
-
id:
|
|
698
|
-
declaredVersion:
|
|
699
|
-
description:
|
|
700
|
-
tags:
|
|
701
|
-
topologyHash:
|
|
702
|
-
nodes:
|
|
703
|
-
edges:
|
|
704
|
-
transitions:
|
|
705
|
-
calls:
|
|
1014
|
+
workflow: import_zod62.z.object({
|
|
1015
|
+
id: import_zod62.z.string().min(1),
|
|
1016
|
+
declaredVersion: import_zod62.z.string().min(1),
|
|
1017
|
+
description: import_zod62.z.string().optional(),
|
|
1018
|
+
tags: import_zod62.z.array(import_zod62.z.string()).optional(),
|
|
1019
|
+
topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/),
|
|
1020
|
+
nodes: import_zod62.z.array(WorkflowTopologyNodeSchema),
|
|
1021
|
+
edges: import_zod62.z.array(WorkflowTopologyEdgeSchema),
|
|
1022
|
+
transitions: import_zod62.z.array(WorkflowTopologyTransitionSchema).optional(),
|
|
1023
|
+
calls: import_zod62.z.array(WorkflowTopologyCallSchema).optional()
|
|
706
1024
|
}).strict()
|
|
707
1025
|
}).strict();
|
|
708
|
-
var EnsureWorkflowTopologyResponseSchema =
|
|
709
|
-
var TelemetryEventTypeSchema =
|
|
710
|
-
var TelemetryEventSchema =
|
|
711
|
-
schemaVersion:
|
|
712
|
-
eventId:
|
|
713
|
-
occurredAt:
|
|
714
|
-
environment:
|
|
1026
|
+
var EnsureWorkflowTopologyResponseSchema = import_zod62.z.object({ workflowRevisionId: import_zod62.z.string().min(1), created: import_zod62.z.boolean() }).strict();
|
|
1027
|
+
var TelemetryEventTypeSchema = import_zod62.z.enum(TELEMETRY_EVENT_TYPES);
|
|
1028
|
+
var TelemetryEventSchema = import_zod62.z.object({
|
|
1029
|
+
schemaVersion: import_zod62.z.literal(1),
|
|
1030
|
+
eventId: import_zod62.z.string().min(1),
|
|
1031
|
+
occurredAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1032
|
+
environment: import_zod62.z.string().min(1),
|
|
715
1033
|
service: TelemetryServiceSchema,
|
|
716
|
-
correlation:
|
|
717
|
-
invocationId:
|
|
718
|
-
parentInvocationId:
|
|
719
|
-
branchId:
|
|
720
|
-
traceId:
|
|
721
|
-
spanId:
|
|
722
|
-
parentSpanId:
|
|
723
|
-
runId:
|
|
724
|
-
sessionId:
|
|
725
|
-
workflowId:
|
|
726
|
-
workflowRevisionId:
|
|
727
|
-
topologyHash:
|
|
728
|
-
nodeId:
|
|
1034
|
+
correlation: import_zod62.z.object({
|
|
1035
|
+
invocationId: import_zod62.z.string().optional(),
|
|
1036
|
+
parentInvocationId: import_zod62.z.string().optional(),
|
|
1037
|
+
branchId: import_zod62.z.string().optional(),
|
|
1038
|
+
traceId: import_zod62.z.string().optional(),
|
|
1039
|
+
spanId: import_zod62.z.string().optional(),
|
|
1040
|
+
parentSpanId: import_zod62.z.string().optional(),
|
|
1041
|
+
runId: import_zod62.z.string().min(1),
|
|
1042
|
+
sessionId: import_zod62.z.string().optional(),
|
|
1043
|
+
workflowId: import_zod62.z.string().min(1),
|
|
1044
|
+
workflowRevisionId: import_zod62.z.string().optional(),
|
|
1045
|
+
topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
1046
|
+
nodeId: import_zod62.z.string().optional()
|
|
729
1047
|
}).strict(),
|
|
730
|
-
context:
|
|
731
|
-
userId:
|
|
732
|
-
tenantId:
|
|
733
|
-
tags:
|
|
734
|
-
metadata:
|
|
1048
|
+
context: import_zod62.z.object({
|
|
1049
|
+
userId: import_zod62.z.string().optional(),
|
|
1050
|
+
tenantId: import_zod62.z.string().optional(),
|
|
1051
|
+
tags: import_zod62.z.array(import_zod62.z.string()).optional(),
|
|
1052
|
+
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
|
|
735
1053
|
}).strict().optional(),
|
|
736
1054
|
type: TelemetryEventTypeSchema,
|
|
737
|
-
payload:
|
|
1055
|
+
payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
|
|
738
1056
|
}).strict().superRefine((event, ctx) => {
|
|
739
1057
|
if (!event.type.startsWith("tool.")) return;
|
|
740
1058
|
const canonical2 = event.payload.version === 1 || [
|
|
@@ -777,20 +1095,20 @@ var require_dist = __commonJS({
|
|
|
777
1095
|
});
|
|
778
1096
|
}
|
|
779
1097
|
});
|
|
780
|
-
var TelemetryEventBatchSchema =
|
|
781
|
-
var TelemetryEventBatchResponseSchema =
|
|
782
|
-
accepted:
|
|
783
|
-
inserted:
|
|
784
|
-
duplicates:
|
|
1098
|
+
var TelemetryEventBatchSchema = import_zod62.z.object({ events: import_zod62.z.array(TelemetryEventSchema).min(1) }).strict();
|
|
1099
|
+
var TelemetryEventBatchResponseSchema = import_zod62.z.object({
|
|
1100
|
+
accepted: import_zod62.z.number().int().nonnegative(),
|
|
1101
|
+
inserted: import_zod62.z.number().int().nonnegative(),
|
|
1102
|
+
duplicates: import_zod62.z.number().int().nonnegative()
|
|
785
1103
|
}).strict();
|
|
786
|
-
var TelemetryPricingSourceSchema =
|
|
1104
|
+
var TelemetryPricingSourceSchema = import_zod62.z.enum([
|
|
787
1105
|
"sdk",
|
|
788
1106
|
"provider",
|
|
789
1107
|
"custom",
|
|
790
1108
|
"project-rate-card",
|
|
791
1109
|
"default-rate-card"
|
|
792
1110
|
]);
|
|
793
|
-
var TelemetryPricingUnitSchema =
|
|
1111
|
+
var TelemetryPricingUnitSchema = import_zod62.z.enum([
|
|
794
1112
|
"token",
|
|
795
1113
|
"character",
|
|
796
1114
|
"request",
|
|
@@ -803,7 +1121,7 @@ var require_dist = __commonJS({
|
|
|
803
1121
|
"minute",
|
|
804
1122
|
"custom"
|
|
805
1123
|
]);
|
|
806
|
-
var TelemetryPricingUsageTypeSchema =
|
|
1124
|
+
var TelemetryPricingUsageTypeSchema = import_zod62.z.enum([
|
|
807
1125
|
"input",
|
|
808
1126
|
"output",
|
|
809
1127
|
"reasoning",
|
|
@@ -819,42 +1137,42 @@ var require_dist = __commonJS({
|
|
|
819
1137
|
"request",
|
|
820
1138
|
"custom"
|
|
821
1139
|
]);
|
|
822
|
-
var TelemetryUsageItemSchema =
|
|
1140
|
+
var TelemetryUsageItemSchema = import_zod62.z.object({
|
|
823
1141
|
usageType: TelemetryPricingUsageTypeSchema,
|
|
824
|
-
quantity:
|
|
1142
|
+
quantity: import_zod62.z.number().nonnegative(),
|
|
825
1143
|
unit: TelemetryPricingUnitSchema,
|
|
826
|
-
label:
|
|
1144
|
+
label: import_zod62.z.string().optional()
|
|
827
1145
|
}).strict();
|
|
828
|
-
var TelemetryUnitPriceSchema =
|
|
1146
|
+
var TelemetryUnitPriceSchema = import_zod62.z.object({
|
|
829
1147
|
usageType: TelemetryPricingUsageTypeSchema,
|
|
830
1148
|
unit: TelemetryPricingUnitSchema,
|
|
831
|
-
unitQuantity:
|
|
832
|
-
priceMicros:
|
|
833
|
-
label:
|
|
1149
|
+
unitQuantity: import_zod62.z.number().positive().default(1),
|
|
1150
|
+
priceMicros: import_zod62.z.number().int().nonnegative(),
|
|
1151
|
+
label: import_zod62.z.string().optional()
|
|
834
1152
|
}).strict();
|
|
835
|
-
var TelemetryPricingLineItemSchema =
|
|
1153
|
+
var TelemetryPricingLineItemSchema = import_zod62.z.object({
|
|
836
1154
|
usageType: TelemetryPricingUsageTypeSchema,
|
|
837
|
-
quantity:
|
|
1155
|
+
quantity: import_zod62.z.number().nonnegative(),
|
|
838
1156
|
unit: TelemetryPricingUnitSchema,
|
|
839
|
-
unitQuantity:
|
|
840
|
-
unitPriceMicros:
|
|
841
|
-
totalCostMicros:
|
|
842
|
-
label:
|
|
1157
|
+
unitQuantity: import_zod62.z.number().positive().default(1),
|
|
1158
|
+
unitPriceMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1159
|
+
totalCostMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1160
|
+
label: import_zod62.z.string().optional()
|
|
843
1161
|
}).strict().refine(
|
|
844
1162
|
(value) => value.totalCostMicros !== void 0 || value.unitPriceMicros !== void 0,
|
|
845
1163
|
"line item requires totalCostMicros or unitPriceMicros"
|
|
846
1164
|
);
|
|
847
|
-
var TelemetryPricingHintSchema =
|
|
1165
|
+
var TelemetryPricingHintSchema = import_zod62.z.object({
|
|
848
1166
|
source: TelemetryPricingSourceSchema.extract(["sdk", "provider", "custom"]),
|
|
849
|
-
currency:
|
|
850
|
-
actualCostMicros:
|
|
851
|
-
lineItems:
|
|
852
|
-
unitPrices:
|
|
853
|
-
usageItems:
|
|
854
|
-
pricingRef:
|
|
855
|
-
metadata:
|
|
1167
|
+
currency: import_zod62.z.string().min(3).max(12).default("USD"),
|
|
1168
|
+
actualCostMicros: import_zod62.z.number().int().nonnegative().optional(),
|
|
1169
|
+
lineItems: import_zod62.z.array(TelemetryPricingLineItemSchema).optional(),
|
|
1170
|
+
unitPrices: import_zod62.z.array(TelemetryUnitPriceSchema).optional(),
|
|
1171
|
+
usageItems: import_zod62.z.array(TelemetryUsageItemSchema).optional(),
|
|
1172
|
+
pricingRef: import_zod62.z.string().min(1).optional(),
|
|
1173
|
+
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
|
|
856
1174
|
}).strict();
|
|
857
|
-
var StudioRunStatusSchema =
|
|
1175
|
+
var StudioRunStatusSchema = import_zod62.z.enum([
|
|
858
1176
|
"running",
|
|
859
1177
|
"completed",
|
|
860
1178
|
"interrupted",
|
|
@@ -870,16 +1188,16 @@ var require_dist = __commonJS({
|
|
|
870
1188
|
"All time",
|
|
871
1189
|
"Custom range"
|
|
872
1190
|
];
|
|
873
|
-
var StudioTimeRangeSchema =
|
|
874
|
-
var StudioTimeRangeQuerySchema =
|
|
1191
|
+
var StudioTimeRangeSchema = import_zod62.z.enum(STUDIO_TIME_RANGES);
|
|
1192
|
+
var StudioTimeRangeQuerySchema = import_zod62.z.object({
|
|
875
1193
|
range: StudioTimeRangeSchema.optional(),
|
|
876
|
-
startedAfter:
|
|
877
|
-
startedBefore:
|
|
1194
|
+
startedAfter: import_zod62.z.string().optional(),
|
|
1195
|
+
startedBefore: import_zod62.z.string().optional()
|
|
878
1196
|
}).strict();
|
|
879
|
-
var StudioTimeRangeContextSchema =
|
|
1197
|
+
var StudioTimeRangeContextSchema = import_zod62.z.object({
|
|
880
1198
|
range: StudioTimeRangeSchema,
|
|
881
|
-
startedAfter:
|
|
882
|
-
startedBefore:
|
|
1199
|
+
startedAfter: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1200
|
+
startedBefore: import_zod62.z.string().datetime({ offset: true }).nullable()
|
|
883
1201
|
}).strict();
|
|
884
1202
|
var STUDIO_TIME_RANGE_MILLISECONDS = {
|
|
885
1203
|
"Last hour": 60 * 60 * 1e3,
|
|
@@ -953,7 +1271,7 @@ var require_dist = __commonJS({
|
|
|
953
1271
|
}
|
|
954
1272
|
};
|
|
955
1273
|
};
|
|
956
|
-
var StudioInterruptStatusSchema =
|
|
1274
|
+
var StudioInterruptStatusSchema = import_zod62.z.enum([
|
|
957
1275
|
"pending",
|
|
958
1276
|
"resolved",
|
|
959
1277
|
"expired",
|
|
@@ -966,329 +1284,329 @@ var require_dist = __commonJS({
|
|
|
966
1284
|
const nowMs = typeof now === "number" ? now : now.getTime();
|
|
967
1285
|
return Number.isFinite(expiresAt) && Number.isFinite(nowMs) && expiresAt <= nowMs ? "expired" : input.status;
|
|
968
1286
|
};
|
|
969
|
-
var StudioInterruptTypeSchema =
|
|
1287
|
+
var StudioInterruptTypeSchema = import_zod62.z.enum([
|
|
970
1288
|
"choice",
|
|
971
1289
|
"multi-choice",
|
|
972
1290
|
"text",
|
|
973
1291
|
"structured",
|
|
974
1292
|
"unknown"
|
|
975
1293
|
]);
|
|
976
|
-
var StudioInterruptInteractionModeSchema =
|
|
1294
|
+
var StudioInterruptInteractionModeSchema = import_zod62.z.enum([
|
|
977
1295
|
"static-options",
|
|
978
1296
|
"dynamic-picker",
|
|
979
1297
|
"freeform",
|
|
980
1298
|
"unknown"
|
|
981
1299
|
]);
|
|
982
|
-
var StudioInterruptOptionSchema =
|
|
983
|
-
id:
|
|
984
|
-
label:
|
|
985
|
-
description:
|
|
1300
|
+
var StudioInterruptOptionSchema = import_zod62.z.object({
|
|
1301
|
+
id: import_zod62.z.string().min(1),
|
|
1302
|
+
label: import_zod62.z.string().min(1),
|
|
1303
|
+
description: import_zod62.z.string().nullable()
|
|
986
1304
|
}).strip();
|
|
987
|
-
var StudioInterruptValueSchema =
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1305
|
+
var StudioInterruptValueSchema = import_zod62.z.union([
|
|
1306
|
+
import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()),
|
|
1307
|
+
import_zod62.z.array(import_zod62.z.unknown()),
|
|
1308
|
+
import_zod62.z.string(),
|
|
1309
|
+
import_zod62.z.number(),
|
|
1310
|
+
import_zod62.z.boolean()
|
|
993
1311
|
]);
|
|
994
|
-
var StudioResumeOutcomeSchema =
|
|
1312
|
+
var StudioResumeOutcomeSchema = import_zod62.z.enum([
|
|
995
1313
|
"resumed",
|
|
996
1314
|
"resume failed",
|
|
997
1315
|
"expired before resume",
|
|
998
1316
|
"cancelled"
|
|
999
1317
|
]);
|
|
1000
|
-
var StudioWorkflowHealthSchema =
|
|
1318
|
+
var StudioWorkflowHealthSchema = import_zod62.z.enum([
|
|
1001
1319
|
"unknown",
|
|
1002
1320
|
"healthy",
|
|
1003
1321
|
"degraded",
|
|
1004
1322
|
"failing",
|
|
1005
1323
|
"idle"
|
|
1006
1324
|
]);
|
|
1007
|
-
var StudioPricingStatusSchema =
|
|
1325
|
+
var StudioPricingStatusSchema = import_zod62.z.enum([
|
|
1008
1326
|
"priced",
|
|
1009
1327
|
"unpriced",
|
|
1010
1328
|
"unknown"
|
|
1011
1329
|
]);
|
|
1012
1330
|
var StudioPricingSourceSchema = TelemetryPricingSourceSchema.nullable();
|
|
1013
|
-
var StudioChangeResourceSchema =
|
|
1331
|
+
var StudioChangeResourceSchema = import_zod62.z.enum([
|
|
1014
1332
|
"evals",
|
|
1015
1333
|
"workflows",
|
|
1016
1334
|
"runs",
|
|
1017
1335
|
"sessions",
|
|
1018
1336
|
"interrupts"
|
|
1019
1337
|
]);
|
|
1020
|
-
var StudioChangeSchema =
|
|
1021
|
-
schemaVersion:
|
|
1022
|
-
changeId:
|
|
1023
|
-
emittedAt:
|
|
1024
|
-
organizationId:
|
|
1025
|
-
projectId:
|
|
1026
|
-
environment:
|
|
1027
|
-
resources:
|
|
1338
|
+
var StudioChangeSchema = import_zod62.z.object({
|
|
1339
|
+
schemaVersion: import_zod62.z.literal(1),
|
|
1340
|
+
changeId: import_zod62.z.string().min(1),
|
|
1341
|
+
emittedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1342
|
+
organizationId: import_zod62.z.string().min(1),
|
|
1343
|
+
projectId: import_zod62.z.string().min(1),
|
|
1344
|
+
environment: import_zod62.z.string().min(1).optional(),
|
|
1345
|
+
resources: import_zod62.z.array(StudioChangeResourceSchema).min(1)
|
|
1028
1346
|
}).strict();
|
|
1029
|
-
var StudioMetricSchema =
|
|
1030
|
-
runCount:
|
|
1031
|
-
successRate:
|
|
1032
|
-
errorRate:
|
|
1033
|
-
retryCount:
|
|
1034
|
-
interruptRate:
|
|
1035
|
-
p50DurationMs:
|
|
1036
|
-
p95DurationMs:
|
|
1037
|
-
averageTokens:
|
|
1038
|
-
averageCost:
|
|
1039
|
-
currency:
|
|
1347
|
+
var StudioMetricSchema = import_zod62.z.object({
|
|
1348
|
+
runCount: import_zod62.z.number().int().nonnegative(),
|
|
1349
|
+
successRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1350
|
+
errorRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1351
|
+
retryCount: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1352
|
+
interruptRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1353
|
+
p50DurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1354
|
+
p95DurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1355
|
+
averageTokens: import_zod62.z.number().nonnegative().nullable(),
|
|
1356
|
+
averageCost: import_zod62.z.number().nonnegative().nullable(),
|
|
1357
|
+
currency: import_zod62.z.string().nullable()
|
|
1040
1358
|
}).strip();
|
|
1041
|
-
var StudioRunSchema =
|
|
1042
|
-
id:
|
|
1359
|
+
var StudioRunSchema = import_zod62.z.object({
|
|
1360
|
+
id: import_zod62.z.string().min(1),
|
|
1043
1361
|
feedback: FeedbackSummarySchema.optional(),
|
|
1044
|
-
callerNodeId:
|
|
1045
|
-
parentRunId:
|
|
1046
|
-
parentWorkflowId:
|
|
1047
|
-
invocationId:
|
|
1048
|
-
branchId:
|
|
1049
|
-
callId:
|
|
1362
|
+
callerNodeId: import_zod62.z.string().optional(),
|
|
1363
|
+
parentRunId: import_zod62.z.string().optional(),
|
|
1364
|
+
parentWorkflowId: import_zod62.z.string().optional(),
|
|
1365
|
+
invocationId: import_zod62.z.string().optional(),
|
|
1366
|
+
branchId: import_zod62.z.string().optional(),
|
|
1367
|
+
callId: import_zod62.z.string().optional(),
|
|
1050
1368
|
status: StudioRunStatusSchema,
|
|
1051
|
-
startedAt:
|
|
1052
|
-
endedAt:
|
|
1053
|
-
workflowId:
|
|
1054
|
-
workflowIds:
|
|
1055
|
-
workflowRefs:
|
|
1056
|
-
|
|
1057
|
-
workflowId:
|
|
1058
|
-
workflowRevisionId:
|
|
1059
|
-
declaredVersion:
|
|
1369
|
+
startedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1370
|
+
endedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1371
|
+
workflowId: import_zod62.z.string().min(1),
|
|
1372
|
+
workflowIds: import_zod62.z.array(import_zod62.z.string().min(1)),
|
|
1373
|
+
workflowRefs: import_zod62.z.array(
|
|
1374
|
+
import_zod62.z.object({
|
|
1375
|
+
workflowId: import_zod62.z.string().min(1),
|
|
1376
|
+
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1377
|
+
declaredVersion: import_zod62.z.string().nullable()
|
|
1060
1378
|
}).strip()
|
|
1061
1379
|
),
|
|
1062
|
-
workflowRevisionId:
|
|
1063
|
-
declaredVersion:
|
|
1064
|
-
transitionIds:
|
|
1065
|
-
path:
|
|
1066
|
-
sessionId:
|
|
1067
|
-
provider:
|
|
1068
|
-
model:
|
|
1069
|
-
models:
|
|
1070
|
-
durationMs:
|
|
1071
|
-
tokens:
|
|
1072
|
-
cost:
|
|
1380
|
+
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1381
|
+
declaredVersion: import_zod62.z.string().nullable(),
|
|
1382
|
+
transitionIds: import_zod62.z.array(import_zod62.z.string().min(1)),
|
|
1383
|
+
path: import_zod62.z.array(import_zod62.z.string()),
|
|
1384
|
+
sessionId: import_zod62.z.string().nullable(),
|
|
1385
|
+
provider: import_zod62.z.string().nullable(),
|
|
1386
|
+
model: import_zod62.z.string().nullable(),
|
|
1387
|
+
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1388
|
+
durationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1389
|
+
tokens: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1390
|
+
cost: import_zod62.z.number().nonnegative().nullable(),
|
|
1073
1391
|
pricingStatus: StudioPricingStatusSchema,
|
|
1074
1392
|
pricingSource: StudioPricingSourceSchema,
|
|
1075
|
-
currency:
|
|
1076
|
-
result:
|
|
1077
|
-
environment:
|
|
1078
|
-
userId:
|
|
1079
|
-
tenantId:
|
|
1080
|
-
hasTool:
|
|
1081
|
-
hasRetry:
|
|
1082
|
-
interruptNodeId:
|
|
1083
|
-
interruptId:
|
|
1393
|
+
currency: import_zod62.z.string().nullable(),
|
|
1394
|
+
result: import_zod62.z.string().nullable(),
|
|
1395
|
+
environment: import_zod62.z.string().min(1),
|
|
1396
|
+
userId: import_zod62.z.string().nullable(),
|
|
1397
|
+
tenantId: import_zod62.z.string().nullable(),
|
|
1398
|
+
hasTool: import_zod62.z.boolean(),
|
|
1399
|
+
hasRetry: import_zod62.z.boolean(),
|
|
1400
|
+
interruptNodeId: import_zod62.z.string().nullable(),
|
|
1401
|
+
interruptId: import_zod62.z.string().nullable().optional(),
|
|
1084
1402
|
interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
|
|
1085
|
-
interruptExpiresAt:
|
|
1403
|
+
interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional()
|
|
1086
1404
|
}).strip();
|
|
1087
|
-
var StudioSessionSchema =
|
|
1088
|
-
id:
|
|
1405
|
+
var StudioSessionSchema = import_zod62.z.object({
|
|
1406
|
+
id: import_zod62.z.string().min(1),
|
|
1089
1407
|
status: StudioRunStatusSchema,
|
|
1090
|
-
lastActivityAt:
|
|
1091
|
-
workflowIds:
|
|
1092
|
-
workflowCount:
|
|
1093
|
-
activeWorkflowId:
|
|
1094
|
-
activeVersion:
|
|
1095
|
-
userId:
|
|
1096
|
-
tenantId:
|
|
1097
|
-
runs:
|
|
1098
|
-
succeeded:
|
|
1099
|
-
failed:
|
|
1100
|
-
interrupted:
|
|
1101
|
-
checkpoints:
|
|
1102
|
-
hasFork:
|
|
1103
|
-
durationMs:
|
|
1104
|
-
tokens:
|
|
1105
|
-
cost:
|
|
1408
|
+
lastActivityAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1409
|
+
workflowIds: import_zod62.z.array(import_zod62.z.string()),
|
|
1410
|
+
workflowCount: import_zod62.z.number().int().nonnegative(),
|
|
1411
|
+
activeWorkflowId: import_zod62.z.string().nullable(),
|
|
1412
|
+
activeVersion: import_zod62.z.string().nullable(),
|
|
1413
|
+
userId: import_zod62.z.string().nullable(),
|
|
1414
|
+
tenantId: import_zod62.z.string().nullable(),
|
|
1415
|
+
runs: import_zod62.z.number().int().nonnegative(),
|
|
1416
|
+
succeeded: import_zod62.z.number().int().nonnegative(),
|
|
1417
|
+
failed: import_zod62.z.number().int().nonnegative(),
|
|
1418
|
+
interrupted: import_zod62.z.number().int().nonnegative(),
|
|
1419
|
+
checkpoints: import_zod62.z.number().int().nonnegative(),
|
|
1420
|
+
hasFork: import_zod62.z.boolean(),
|
|
1421
|
+
durationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1422
|
+
tokens: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1423
|
+
cost: import_zod62.z.number().nonnegative().nullable(),
|
|
1106
1424
|
pricingStatus: StudioPricingStatusSchema,
|
|
1107
1425
|
pricingSource: StudioPricingSourceSchema,
|
|
1108
|
-
currency:
|
|
1109
|
-
latestResult:
|
|
1110
|
-
latestError:
|
|
1111
|
-
pendingInterruptId:
|
|
1426
|
+
currency: import_zod62.z.string().nullable(),
|
|
1427
|
+
latestResult: import_zod62.z.string().nullable(),
|
|
1428
|
+
latestError: import_zod62.z.string().nullable(),
|
|
1429
|
+
pendingInterruptId: import_zod62.z.string().nullable(),
|
|
1112
1430
|
interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
|
|
1113
|
-
interruptExpiresAt:
|
|
1114
|
-
providers:
|
|
1115
|
-
models:
|
|
1116
|
-
tags:
|
|
1117
|
-
environment:
|
|
1431
|
+
interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional(),
|
|
1432
|
+
providers: import_zod62.z.array(import_zod62.z.string()),
|
|
1433
|
+
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1434
|
+
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1435
|
+
environment: import_zod62.z.string().min(1)
|
|
1118
1436
|
}).strip();
|
|
1119
|
-
var StudioInterruptSchema =
|
|
1120
|
-
afterResponseCompleted:
|
|
1121
|
-
id:
|
|
1437
|
+
var StudioInterruptSchema = import_zod62.z.object({
|
|
1438
|
+
afterResponseCompleted: import_zod62.z.boolean().optional(),
|
|
1439
|
+
id: import_zod62.z.string().min(1),
|
|
1122
1440
|
status: StudioInterruptStatusSchema,
|
|
1123
1441
|
type: StudioInterruptTypeSchema,
|
|
1124
1442
|
interactionMode: StudioInterruptInteractionModeSchema,
|
|
1125
|
-
contract:
|
|
1126
|
-
schemaId:
|
|
1127
|
-
schemaVersion:
|
|
1128
|
-
createdAt:
|
|
1129
|
-
resolvedAt:
|
|
1130
|
-
expiresAt:
|
|
1131
|
-
question:
|
|
1132
|
-
contentCaptured:
|
|
1133
|
-
request:
|
|
1134
|
-
requestCaptured:
|
|
1135
|
-
optionCount:
|
|
1136
|
-
options:
|
|
1137
|
-
workflowId:
|
|
1138
|
-
nodeId:
|
|
1139
|
-
sessionId:
|
|
1140
|
-
userId:
|
|
1141
|
-
tenantId:
|
|
1142
|
-
response:
|
|
1443
|
+
contract: import_zod62.z.string().nullable().default(null),
|
|
1444
|
+
schemaId: import_zod62.z.string().nullable(),
|
|
1445
|
+
schemaVersion: import_zod62.z.string().nullable(),
|
|
1446
|
+
createdAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1447
|
+
resolvedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1448
|
+
expiresAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1449
|
+
question: import_zod62.z.string().nullable(),
|
|
1450
|
+
contentCaptured: import_zod62.z.boolean(),
|
|
1451
|
+
request: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable().default(null),
|
|
1452
|
+
requestCaptured: import_zod62.z.boolean().default(false),
|
|
1453
|
+
optionCount: import_zod62.z.number().int().nonnegative().nullable(),
|
|
1454
|
+
options: import_zod62.z.array(StudioInterruptOptionSchema).nullable(),
|
|
1455
|
+
workflowId: import_zod62.z.string().min(1),
|
|
1456
|
+
nodeId: import_zod62.z.string().nullable(),
|
|
1457
|
+
sessionId: import_zod62.z.string().nullable(),
|
|
1458
|
+
userId: import_zod62.z.string().nullable(),
|
|
1459
|
+
tenantId: import_zod62.z.string().nullable(),
|
|
1460
|
+
response: import_zod62.z.string().nullable(),
|
|
1143
1461
|
responseValue: StudioInterruptValueSchema.nullable().default(null),
|
|
1144
|
-
responseCaptured:
|
|
1462
|
+
responseCaptured: import_zod62.z.boolean(),
|
|
1145
1463
|
resumeOutcome: StudioResumeOutcomeSchema.nullable(),
|
|
1146
|
-
resumeError:
|
|
1147
|
-
runId:
|
|
1148
|
-
resumeToken:
|
|
1149
|
-
resolvedBy:
|
|
1150
|
-
environment:
|
|
1464
|
+
resumeError: import_zod62.z.string().nullable(),
|
|
1465
|
+
runId: import_zod62.z.string().min(1),
|
|
1466
|
+
resumeToken: import_zod62.z.string().nullable(),
|
|
1467
|
+
resolvedBy: import_zod62.z.string().nullable(),
|
|
1468
|
+
environment: import_zod62.z.string().min(1)
|
|
1151
1469
|
}).strip();
|
|
1152
|
-
var StudioDetailEventSchema =
|
|
1153
|
-
id:
|
|
1470
|
+
var StudioDetailEventSchema = import_zod62.z.object({
|
|
1471
|
+
id: import_zod62.z.string().min(1),
|
|
1154
1472
|
// Studio readers must preserve future event kinds within the same API major.
|
|
1155
|
-
type:
|
|
1156
|
-
occurredAt:
|
|
1157
|
-
receivedAt:
|
|
1158
|
-
environment:
|
|
1159
|
-
serviceName:
|
|
1160
|
-
deploymentRef:
|
|
1161
|
-
traceId:
|
|
1162
|
-
spanId:
|
|
1163
|
-
parentSpanId:
|
|
1164
|
-
runId:
|
|
1165
|
-
sessionId:
|
|
1166
|
-
workflowId:
|
|
1167
|
-
workflowRevisionId:
|
|
1168
|
-
nodeId:
|
|
1169
|
-
userId:
|
|
1170
|
-
tenantId:
|
|
1171
|
-
tags:
|
|
1172
|
-
metadata:
|
|
1173
|
-
payload:
|
|
1473
|
+
type: import_zod62.z.string().min(1),
|
|
1474
|
+
occurredAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1475
|
+
receivedAt: import_zod62.z.string().datetime({ offset: true }),
|
|
1476
|
+
environment: import_zod62.z.string().min(1),
|
|
1477
|
+
serviceName: import_zod62.z.string().min(1),
|
|
1478
|
+
deploymentRef: import_zod62.z.string().nullable(),
|
|
1479
|
+
traceId: import_zod62.z.string().nullable(),
|
|
1480
|
+
spanId: import_zod62.z.string().nullable(),
|
|
1481
|
+
parentSpanId: import_zod62.z.string().nullable(),
|
|
1482
|
+
runId: import_zod62.z.string().min(1),
|
|
1483
|
+
sessionId: import_zod62.z.string().nullable(),
|
|
1484
|
+
workflowId: import_zod62.z.string().min(1),
|
|
1485
|
+
workflowRevisionId: import_zod62.z.string().nullable(),
|
|
1486
|
+
nodeId: import_zod62.z.string().nullable(),
|
|
1487
|
+
userId: import_zod62.z.string().nullable(),
|
|
1488
|
+
tenantId: import_zod62.z.string().nullable(),
|
|
1489
|
+
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1490
|
+
metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable(),
|
|
1491
|
+
payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
|
|
1174
1492
|
}).strip();
|
|
1175
|
-
var StudioWorkflowNodeSchema =
|
|
1176
|
-
tools:
|
|
1493
|
+
var StudioWorkflowNodeSchema = import_zod62.z.object({
|
|
1494
|
+
tools: import_zod62.z.array(StudioToolSchema).optional(),
|
|
1177
1495
|
toolDiscovery: ToolDiscoverySchema.optional(),
|
|
1178
|
-
id:
|
|
1179
|
-
label:
|
|
1180
|
-
type:
|
|
1181
|
-
state:
|
|
1182
|
-
provider:
|
|
1183
|
-
model:
|
|
1496
|
+
id: import_zod62.z.string().min(1),
|
|
1497
|
+
label: import_zod62.z.string(),
|
|
1498
|
+
type: import_zod62.z.string().nullable(),
|
|
1499
|
+
state: import_zod62.z.enum(["healthy", "warning", "failed", "interrupted", "retried"]).nullable(),
|
|
1500
|
+
provider: import_zod62.z.string().nullable(),
|
|
1501
|
+
model: import_zod62.z.string().nullable(),
|
|
1184
1502
|
metrics: StudioMetricSchema
|
|
1185
1503
|
}).strip();
|
|
1186
|
-
var StudioWorkflowInternalEdgeSchema =
|
|
1187
|
-
id:
|
|
1188
|
-
source:
|
|
1189
|
-
target:
|
|
1190
|
-
condition:
|
|
1504
|
+
var StudioWorkflowInternalEdgeSchema = import_zod62.z.object({
|
|
1505
|
+
id: import_zod62.z.string().min(1),
|
|
1506
|
+
source: import_zod62.z.string().min(1),
|
|
1507
|
+
target: import_zod62.z.string().min(1),
|
|
1508
|
+
condition: import_zod62.z.string().nullable()
|
|
1191
1509
|
}).strip();
|
|
1192
|
-
var StudioWorkflowTransitionSchema =
|
|
1193
|
-
kind:
|
|
1194
|
-
id:
|
|
1195
|
-
sourceWorkflowId:
|
|
1196
|
-
sourceNodeId:
|
|
1197
|
-
targetWorkflowId:
|
|
1198
|
-
condition:
|
|
1199
|
-
volume:
|
|
1200
|
-
successRate:
|
|
1201
|
-
medianDurationMs:
|
|
1202
|
-
errorRate:
|
|
1510
|
+
var StudioWorkflowTransitionSchema = import_zod62.z.object({
|
|
1511
|
+
kind: import_zod62.z.enum(["call", "handoff"]).optional(),
|
|
1512
|
+
id: import_zod62.z.string().min(1),
|
|
1513
|
+
sourceWorkflowId: import_zod62.z.string().min(1),
|
|
1514
|
+
sourceNodeId: import_zod62.z.string().nullable(),
|
|
1515
|
+
targetWorkflowId: import_zod62.z.string().min(1),
|
|
1516
|
+
condition: import_zod62.z.string().nullable(),
|
|
1517
|
+
volume: import_zod62.z.number().int().nonnegative(),
|
|
1518
|
+
successRate: import_zod62.z.number().min(0).max(100).nullable(),
|
|
1519
|
+
medianDurationMs: import_zod62.z.number().nonnegative().nullable(),
|
|
1520
|
+
errorRate: import_zod62.z.number().min(0).max(100).nullable()
|
|
1203
1521
|
}).strip();
|
|
1204
|
-
var StudioWorkflowSchema =
|
|
1205
|
-
id:
|
|
1206
|
-
name:
|
|
1207
|
-
description:
|
|
1208
|
-
versions:
|
|
1209
|
-
activeVersion:
|
|
1210
|
-
activeRevisionId:
|
|
1522
|
+
var StudioWorkflowSchema = import_zod62.z.object({
|
|
1523
|
+
id: import_zod62.z.string().min(1),
|
|
1524
|
+
name: import_zod62.z.string().min(1),
|
|
1525
|
+
description: import_zod62.z.string().nullable(),
|
|
1526
|
+
versions: import_zod62.z.array(import_zod62.z.string()),
|
|
1527
|
+
activeVersion: import_zod62.z.string().nullable(),
|
|
1528
|
+
activeRevisionId: import_zod62.z.string().nullable(),
|
|
1211
1529
|
health: StudioWorkflowHealthSchema,
|
|
1212
|
-
tags:
|
|
1213
|
-
lastActivityAt:
|
|
1530
|
+
tags: import_zod62.z.array(import_zod62.z.string()),
|
|
1531
|
+
lastActivityAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
|
|
1214
1532
|
metrics: StudioMetricSchema,
|
|
1215
|
-
nodes:
|
|
1216
|
-
internalEdges:
|
|
1533
|
+
nodes: import_zod62.z.array(StudioWorkflowNodeSchema),
|
|
1534
|
+
internalEdges: import_zod62.z.array(StudioWorkflowInternalEdgeSchema)
|
|
1217
1535
|
}).strip();
|
|
1218
|
-
var StudioRunsResponseSchema =
|
|
1219
|
-
runs:
|
|
1220
|
-
totalCount:
|
|
1536
|
+
var StudioRunsResponseSchema = import_zod62.z.object({
|
|
1537
|
+
runs: import_zod62.z.array(StudioRunSchema),
|
|
1538
|
+
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1221
1539
|
}).strip();
|
|
1222
|
-
var StudioSessionsResponseSchema =
|
|
1223
|
-
sessions:
|
|
1224
|
-
totalCount:
|
|
1540
|
+
var StudioSessionsResponseSchema = import_zod62.z.object({
|
|
1541
|
+
sessions: import_zod62.z.array(StudioSessionSchema),
|
|
1542
|
+
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1225
1543
|
}).strip();
|
|
1226
|
-
var StudioInterruptsResponseSchema =
|
|
1227
|
-
interrupts:
|
|
1228
|
-
totalCount:
|
|
1544
|
+
var StudioInterruptsResponseSchema = import_zod62.z.object({
|
|
1545
|
+
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1546
|
+
totalCount: import_zod62.z.number().int().nonnegative()
|
|
1229
1547
|
}).strip();
|
|
1230
|
-
var StudioRunDetailResponseSchema =
|
|
1548
|
+
var StudioRunDetailResponseSchema = import_zod62.z.object({
|
|
1231
1549
|
run: StudioRunSchema,
|
|
1232
|
-
scores:
|
|
1233
|
-
canReview:
|
|
1234
|
-
reviewActorId:
|
|
1235
|
-
events:
|
|
1550
|
+
scores: import_zod62.z.array(StudioScoreSchema).optional(),
|
|
1551
|
+
canReview: import_zod62.z.boolean().optional(),
|
|
1552
|
+
reviewActorId: import_zod62.z.string().optional(),
|
|
1553
|
+
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1236
1554
|
session: StudioSessionSchema.nullable(),
|
|
1237
|
-
interrupts:
|
|
1238
|
-
updatedAt:
|
|
1555
|
+
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1556
|
+
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1239
1557
|
}).strip();
|
|
1240
|
-
var StudioSessionDetailResponseSchema =
|
|
1558
|
+
var StudioSessionDetailResponseSchema = import_zod62.z.object({
|
|
1241
1559
|
session: StudioSessionSchema,
|
|
1242
|
-
runs:
|
|
1243
|
-
events:
|
|
1244
|
-
interrupts:
|
|
1245
|
-
updatedAt:
|
|
1560
|
+
runs: import_zod62.z.array(StudioRunSchema),
|
|
1561
|
+
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1562
|
+
interrupts: import_zod62.z.array(StudioInterruptSchema),
|
|
1563
|
+
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1246
1564
|
}).strip();
|
|
1247
|
-
var StudioInterruptDetailResponseSchema =
|
|
1565
|
+
var StudioInterruptDetailResponseSchema = import_zod62.z.object({
|
|
1248
1566
|
interrupt: StudioInterruptSchema,
|
|
1249
|
-
events:
|
|
1567
|
+
events: import_zod62.z.array(StudioDetailEventSchema),
|
|
1250
1568
|
run: StudioRunSchema.nullable(),
|
|
1251
1569
|
session: StudioSessionSchema.nullable(),
|
|
1252
|
-
updatedAt:
|
|
1570
|
+
updatedAt: import_zod62.z.string().datetime({ offset: true })
|
|
1253
1571
|
}).strip();
|
|
1254
|
-
var StudioWorkflowsResponseSchema =
|
|
1255
|
-
workflows:
|
|
1256
|
-
observedCalls:
|
|
1572
|
+
var StudioWorkflowsResponseSchema = import_zod62.z.object({
|
|
1573
|
+
workflows: import_zod62.z.array(StudioWorkflowSchema),
|
|
1574
|
+
observedCalls: import_zod62.z.array(
|
|
1257
1575
|
StudioWorkflowTransitionSchema.extend({
|
|
1258
|
-
runId:
|
|
1259
|
-
invocationId:
|
|
1260
|
-
branchId:
|
|
1576
|
+
runId: import_zod62.z.string(),
|
|
1577
|
+
invocationId: import_zod62.z.string(),
|
|
1578
|
+
branchId: import_zod62.z.string()
|
|
1261
1579
|
})
|
|
1262
1580
|
).optional(),
|
|
1263
|
-
transitions:
|
|
1581
|
+
transitions: import_zod62.z.array(StudioWorkflowTransitionSchema),
|
|
1264
1582
|
cohort: StudioTimeRangeContextSchema.extend({
|
|
1265
|
-
workflowId:
|
|
1266
|
-
version:
|
|
1583
|
+
workflowId: import_zod62.z.string().nullable(),
|
|
1584
|
+
version: import_zod62.z.string().nullable()
|
|
1267
1585
|
}).strip()
|
|
1268
1586
|
}).strip();
|
|
1269
|
-
var StudioCatalogsResponseSchema =
|
|
1270
|
-
environments:
|
|
1271
|
-
providers:
|
|
1272
|
-
models:
|
|
1273
|
-
workflows:
|
|
1274
|
-
tags:
|
|
1587
|
+
var StudioCatalogsResponseSchema = import_zod62.z.object({
|
|
1588
|
+
environments: import_zod62.z.array(import_zod62.z.string()),
|
|
1589
|
+
providers: import_zod62.z.array(import_zod62.z.string()),
|
|
1590
|
+
models: import_zod62.z.array(import_zod62.z.string()),
|
|
1591
|
+
workflows: import_zod62.z.array(import_zod62.z.string()),
|
|
1592
|
+
tags: import_zod62.z.array(import_zod62.z.string())
|
|
1275
1593
|
}).strip();
|
|
1276
|
-
var StudioContextResponseSchema =
|
|
1277
|
-
organization:
|
|
1278
|
-
name:
|
|
1594
|
+
var StudioContextResponseSchema = import_zod62.z.object({
|
|
1595
|
+
organization: import_zod62.z.object({
|
|
1596
|
+
name: import_zod62.z.string().min(1)
|
|
1279
1597
|
}).strip(),
|
|
1280
|
-
project:
|
|
1281
|
-
name:
|
|
1598
|
+
project: import_zod62.z.object({
|
|
1599
|
+
name: import_zod62.z.string().min(1)
|
|
1282
1600
|
}).strip(),
|
|
1283
|
-
environments:
|
|
1284
|
-
apiKey:
|
|
1285
|
-
mode:
|
|
1286
|
-
scopes:
|
|
1601
|
+
environments: import_zod62.z.array(import_zod62.z.string()),
|
|
1602
|
+
apiKey: import_zod62.z.object({
|
|
1603
|
+
mode: import_zod62.z.enum(["test", "live"]),
|
|
1604
|
+
scopes: import_zod62.z.array(import_zod62.z.string())
|
|
1287
1605
|
}).strip().nullable(),
|
|
1288
|
-
api:
|
|
1289
|
-
status:
|
|
1290
|
-
service:
|
|
1291
|
-
version:
|
|
1606
|
+
api: import_zod62.z.object({
|
|
1607
|
+
status: import_zod62.z.literal("ok"),
|
|
1608
|
+
service: import_zod62.z.literal("kortyx-api"),
|
|
1609
|
+
version: import_zod62.z.string().min(1)
|
|
1292
1610
|
}).strip()
|
|
1293
1611
|
}).strip();
|
|
1294
1612
|
}
|
|
@@ -1662,6 +1980,86 @@ var require_internal = __commonJS({
|
|
|
1662
1980
|
}
|
|
1663
1981
|
return execution ?? invoke(safeSpan(void 0));
|
|
1664
1982
|
}
|
|
1983
|
+
var import_node2 = require("@kortyx/usage-telemetry/node");
|
|
1984
|
+
var package_default2 = {
|
|
1985
|
+
name: "@kortyx/hooks",
|
|
1986
|
+
version: "0.34.0",
|
|
1987
|
+
private: false,
|
|
1988
|
+
description: "Lifecycle hooks and extension points for Kortyx.",
|
|
1989
|
+
keywords: [
|
|
1990
|
+
"ai",
|
|
1991
|
+
"agent",
|
|
1992
|
+
"agents",
|
|
1993
|
+
"orchestration",
|
|
1994
|
+
"framework",
|
|
1995
|
+
"hooks",
|
|
1996
|
+
"kortyx"
|
|
1997
|
+
],
|
|
1998
|
+
license: "Apache-2.0",
|
|
1999
|
+
homepage: "https://kortyx.io/",
|
|
2000
|
+
bugs: {
|
|
2001
|
+
url: "https://github.com/kortyx-io/kortyx/issues"
|
|
2002
|
+
},
|
|
2003
|
+
author: "Kortyx (https://github.com/kortyx-io)",
|
|
2004
|
+
main: "./dist/index.js",
|
|
2005
|
+
types: "./dist/index.d.ts",
|
|
2006
|
+
exports: {
|
|
2007
|
+
".": {
|
|
2008
|
+
types: "./dist/index.d.ts",
|
|
2009
|
+
default: "./dist/index.js"
|
|
2010
|
+
},
|
|
2011
|
+
"./internal": {
|
|
2012
|
+
types: "./dist/internal.d.ts",
|
|
2013
|
+
default: "./dist/internal.js"
|
|
2014
|
+
}
|
|
2015
|
+
},
|
|
2016
|
+
files: [
|
|
2017
|
+
"dist/**",
|
|
2018
|
+
"README.md",
|
|
2019
|
+
"CHANGELOG.md"
|
|
2020
|
+
],
|
|
2021
|
+
scripts: {
|
|
2022
|
+
build: "tsup",
|
|
2023
|
+
predev: "tsc -p tsconfig.build.json",
|
|
2024
|
+
dev: "tsc --watch",
|
|
2025
|
+
lint: "biome check src",
|
|
2026
|
+
test: "vitest run && pnpm -s type-check",
|
|
2027
|
+
coverage: "vitest run --coverage && pnpm -s type-check",
|
|
2028
|
+
"test:watch": "vitest",
|
|
2029
|
+
"type-check": "tsc --noEmit -p tsconfig.test.json"
|
|
2030
|
+
},
|
|
2031
|
+
dependencies: {
|
|
2032
|
+
"@kortyx/core": "workspace:*",
|
|
2033
|
+
"@kortyx/providers": "workspace:*",
|
|
2034
|
+
"@kortyx/telemetry-contracts": "workspace:*",
|
|
2035
|
+
zod: "catalog:",
|
|
2036
|
+
"@kortyx/prompts": "workspace:*",
|
|
2037
|
+
"@kortyx/usage-telemetry": "workspace:*"
|
|
2038
|
+
},
|
|
2039
|
+
devDependencies: {
|
|
2040
|
+
turbo: "catalog:",
|
|
2041
|
+
typescript: "catalog:",
|
|
2042
|
+
tsup: "catalog:",
|
|
2043
|
+
vitest: "catalog:",
|
|
2044
|
+
"@vitest/coverage-v8": "catalog:"
|
|
2045
|
+
},
|
|
2046
|
+
sideEffects: false,
|
|
2047
|
+
repository: {
|
|
2048
|
+
type: "git",
|
|
2049
|
+
url: "https://github.com/kortyx-io/kortyx.git",
|
|
2050
|
+
directory: "packages/hooks"
|
|
2051
|
+
},
|
|
2052
|
+
publishConfig: {
|
|
2053
|
+
access: "public"
|
|
2054
|
+
}
|
|
2055
|
+
};
|
|
2056
|
+
var client2 = () => (0, import_node2.getUsageTelemetry)("hooks", package_default2.version);
|
|
2057
|
+
var usage2 = {
|
|
2058
|
+
start: (...args) => client2().start(...args),
|
|
2059
|
+
feature: (...args) => client2().feature(...args),
|
|
2060
|
+
measure: (...args) => client2().measure(...args),
|
|
2061
|
+
flush: () => client2().flush()
|
|
2062
|
+
};
|
|
1665
2063
|
function assertModelFinish(result) {
|
|
1666
2064
|
if (result.finishReason?.unified !== "content-filter") return;
|
|
1667
2065
|
throw Object.assign(
|
|
@@ -1701,7 +2099,7 @@ var require_internal = __commonJS({
|
|
|
1701
2099
|
}
|
|
1702
2100
|
return providerOptions;
|
|
1703
2101
|
};
|
|
1704
|
-
async function
|
|
2102
|
+
async function runReasonEngineInternal(args) {
|
|
1705
2103
|
let stream = args.stream ?? args.model.options?.streaming ?? true;
|
|
1706
2104
|
const emit = args.emit ?? true;
|
|
1707
2105
|
const temperature = args.temperature ?? args.model.options?.temperature ?? args.defaultTemperature;
|
|
@@ -1728,6 +2126,11 @@ var require_internal = __commonJS({
|
|
|
1728
2126
|
...providerOptions !== void 0 ? { providerOptions } : {}
|
|
1729
2127
|
});
|
|
1730
2128
|
if (tools?.length && !model.supportsToolStreaming) stream = false;
|
|
2129
|
+
usage2.feature("sdk.model", {
|
|
2130
|
+
streaming: stream,
|
|
2131
|
+
tools: Boolean(tools?.length),
|
|
2132
|
+
provider: args.model.provider.id
|
|
2133
|
+
});
|
|
1731
2134
|
const messages = args.messages !== void 0 ? args.messages : [
|
|
1732
2135
|
...typeof args.system === "string" && args.system.length > 0 ? [{ role: "system", content: args.system }] : [],
|
|
1733
2136
|
{ role: "user", content: String(args.input ?? "") }
|
|
@@ -1807,7 +2210,7 @@ var require_internal = __commonJS({
|
|
|
1807
2210
|
let continuation;
|
|
1808
2211
|
let final = "";
|
|
1809
2212
|
let raw;
|
|
1810
|
-
let
|
|
2213
|
+
let usage22;
|
|
1811
2214
|
let finishReason;
|
|
1812
2215
|
let toolCalls;
|
|
1813
2216
|
let providerMetadata;
|
|
@@ -1848,7 +2251,7 @@ var require_internal = __commonJS({
|
|
|
1848
2251
|
raw = chunk.raw;
|
|
1849
2252
|
}
|
|
1850
2253
|
if (chunk.usage !== void 0) {
|
|
1851
|
-
|
|
2254
|
+
usage22 = chunk.usage;
|
|
1852
2255
|
}
|
|
1853
2256
|
if (chunk.finishReason !== void 0) {
|
|
1854
2257
|
finishReason = chunk.finishReason;
|
|
@@ -1886,7 +2289,7 @@ var require_internal = __commonJS({
|
|
|
1886
2289
|
}
|
|
1887
2290
|
}
|
|
1888
2291
|
(0, import_core6.throwIfExecutionAborted)(abortSignal);
|
|
1889
|
-
assertModelFinish({ finishReason, usage });
|
|
2292
|
+
assertModelFinish({ finishReason, usage: usage22 });
|
|
1890
2293
|
if (emit) {
|
|
1891
2294
|
emitNodeEvent(args.emitEvent, args.nodeId, "text-end", {
|
|
1892
2295
|
...commonMeta
|
|
@@ -1896,7 +2299,7 @@ var require_internal = __commonJS({
|
|
|
1896
2299
|
text: final,
|
|
1897
2300
|
...continuation ? { continuation } : {},
|
|
1898
2301
|
...raw !== void 0 ? { raw } : {},
|
|
1899
|
-
...
|
|
2302
|
+
...usage22 !== void 0 ? { usage: usage22 } : {},
|
|
1900
2303
|
...finishReason !== void 0 ? { finishReason } : {},
|
|
1901
2304
|
...toolCalls !== void 0 ? { toolCalls } : {},
|
|
1902
2305
|
...providerMetadata !== void 0 ? { providerMetadata } : {},
|
|
@@ -1942,6 +2345,14 @@ var require_internal = __commonJS({
|
|
|
1942
2345
|
throw failure2;
|
|
1943
2346
|
}
|
|
1944
2347
|
}
|
|
2348
|
+
function runReasonEngine(args) {
|
|
2349
|
+
return usage2.measure("sdk.model", () => runReasonEngineInternal(args), {
|
|
2350
|
+
provider: args.model.provider.id,
|
|
2351
|
+
requested_streaming: args.stream ?? args.model.options?.streaming ?? true,
|
|
2352
|
+
tools: Boolean((args.tools ?? args.model.options?.tools)?.length),
|
|
2353
|
+
format: args.responseFormat?.type ?? args.model.options?.responseFormat?.type ?? "text"
|
|
2354
|
+
});
|
|
2355
|
+
}
|
|
1945
2356
|
var privateField = /^(?:password|passwd|secret|clientsecret|privatekey|credential|credentials|authorization|bearer|bearertoken|jwt|token|accesstoken|refreshtoken|authtoken|apikey|apisecret|cookie|setcookie|resumehandle|resumetoken)$/;
|
|
1946
2357
|
function safeTelemetryMetadata2(value) {
|
|
1947
2358
|
const seen = /* @__PURE__ */ new WeakSet();
|
|
@@ -2003,6 +2414,7 @@ var require_dist2 = __commonJS({
|
|
|
2003
2414
|
useAbortSignal: () => useAbortSignal,
|
|
2004
2415
|
useInterrupt: () => useInterrupt,
|
|
2005
2416
|
useNodeState: () => useNodeState,
|
|
2417
|
+
usePrompt: () => usePrompt,
|
|
2006
2418
|
useReason: () => useReason2,
|
|
2007
2419
|
useRuntimeContext: () => useRuntimeContext,
|
|
2008
2420
|
useStructuredData: () => useStructuredData,
|
|
@@ -2013,8 +2425,8 @@ var require_dist2 = __commonJS({
|
|
|
2013
2425
|
});
|
|
2014
2426
|
module2.exports = __toCommonJS2(index_exports2);
|
|
2015
2427
|
var import_zod12 = require("zod");
|
|
2016
|
-
var
|
|
2017
|
-
var storage = new
|
|
2428
|
+
var import_node_async_hooks2 = require("async_hooks");
|
|
2429
|
+
var storage = new import_node_async_hooks2.AsyncLocalStorage();
|
|
2018
2430
|
var isRecord5 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
2019
2431
|
var normalizeNodeState = (value) => {
|
|
2020
2432
|
if (Array.isArray(value)) {
|
|
@@ -2104,11 +2516,11 @@ var require_dist2 = __commonJS({
|
|
|
2104
2516
|
}
|
|
2105
2517
|
if (changed) ctx.stateDirty = true;
|
|
2106
2518
|
};
|
|
2107
|
-
var toTokenUsageDelta = (
|
|
2108
|
-
if (!
|
|
2109
|
-
const input =
|
|
2110
|
-
const output =
|
|
2111
|
-
const total =
|
|
2519
|
+
var toTokenUsageDelta = (usage22) => {
|
|
2520
|
+
if (!usage22) return void 0;
|
|
2521
|
+
const input = usage22.input ?? 0;
|
|
2522
|
+
const output = usage22.output ?? 0;
|
|
2523
|
+
const total = usage22.total ?? input + output + (usage22.outputIncludesReasoning ? 0 : usage22.reasoning ?? 0);
|
|
2112
2524
|
if (input === 0 && output === 0 && total === 0) {
|
|
2113
2525
|
return void 0;
|
|
2114
2526
|
}
|
|
@@ -2118,8 +2530,8 @@ var require_dist2 = __commonJS({
|
|
|
2118
2530
|
total
|
|
2119
2531
|
};
|
|
2120
2532
|
};
|
|
2121
|
-
function accumulateTokenUsage(
|
|
2122
|
-
const delta = toTokenUsageDelta(
|
|
2533
|
+
function accumulateTokenUsage(usage22) {
|
|
2534
|
+
const delta = toTokenUsageDelta(usage22);
|
|
2123
2535
|
if (!delta) return;
|
|
2124
2536
|
const ctx = getHookContext();
|
|
2125
2537
|
const current = ctx.tokenUsage ?? {
|
|
@@ -2204,11 +2616,91 @@ var require_dist2 = __commonJS({
|
|
|
2204
2616
|
runtime: { ...ctx.state.runtime, ...buildRuntimeStateUpdates(ctx) }
|
|
2205
2617
|
};
|
|
2206
2618
|
}
|
|
2619
|
+
var import_node2 = require("@kortyx/usage-telemetry/node");
|
|
2620
|
+
var package_default2 = {
|
|
2621
|
+
name: "@kortyx/hooks",
|
|
2622
|
+
version: "0.34.0",
|
|
2623
|
+
private: false,
|
|
2624
|
+
description: "Lifecycle hooks and extension points for Kortyx.",
|
|
2625
|
+
keywords: [
|
|
2626
|
+
"ai",
|
|
2627
|
+
"agent",
|
|
2628
|
+
"agents",
|
|
2629
|
+
"orchestration",
|
|
2630
|
+
"framework",
|
|
2631
|
+
"hooks",
|
|
2632
|
+
"kortyx"
|
|
2633
|
+
],
|
|
2634
|
+
license: "Apache-2.0",
|
|
2635
|
+
homepage: "https://kortyx.io/",
|
|
2636
|
+
bugs: {
|
|
2637
|
+
url: "https://github.com/kortyx-io/kortyx/issues"
|
|
2638
|
+
},
|
|
2639
|
+
author: "Kortyx (https://github.com/kortyx-io)",
|
|
2640
|
+
main: "./dist/index.js",
|
|
2641
|
+
types: "./dist/index.d.ts",
|
|
2642
|
+
exports: {
|
|
2643
|
+
".": {
|
|
2644
|
+
types: "./dist/index.d.ts",
|
|
2645
|
+
default: "./dist/index.js"
|
|
2646
|
+
},
|
|
2647
|
+
"./internal": {
|
|
2648
|
+
types: "./dist/internal.d.ts",
|
|
2649
|
+
default: "./dist/internal.js"
|
|
2650
|
+
}
|
|
2651
|
+
},
|
|
2652
|
+
files: [
|
|
2653
|
+
"dist/**",
|
|
2654
|
+
"README.md",
|
|
2655
|
+
"CHANGELOG.md"
|
|
2656
|
+
],
|
|
2657
|
+
scripts: {
|
|
2658
|
+
build: "tsup",
|
|
2659
|
+
predev: "tsc -p tsconfig.build.json",
|
|
2660
|
+
dev: "tsc --watch",
|
|
2661
|
+
lint: "biome check src",
|
|
2662
|
+
test: "vitest run && pnpm -s type-check",
|
|
2663
|
+
coverage: "vitest run --coverage && pnpm -s type-check",
|
|
2664
|
+
"test:watch": "vitest",
|
|
2665
|
+
"type-check": "tsc --noEmit -p tsconfig.test.json"
|
|
2666
|
+
},
|
|
2667
|
+
dependencies: {
|
|
2668
|
+
"@kortyx/core": "workspace:*",
|
|
2669
|
+
"@kortyx/providers": "workspace:*",
|
|
2670
|
+
"@kortyx/telemetry-contracts": "workspace:*",
|
|
2671
|
+
zod: "catalog:",
|
|
2672
|
+
"@kortyx/prompts": "workspace:*",
|
|
2673
|
+
"@kortyx/usage-telemetry": "workspace:*"
|
|
2674
|
+
},
|
|
2675
|
+
devDependencies: {
|
|
2676
|
+
turbo: "catalog:",
|
|
2677
|
+
typescript: "catalog:",
|
|
2678
|
+
tsup: "catalog:",
|
|
2679
|
+
vitest: "catalog:",
|
|
2680
|
+
"@vitest/coverage-v8": "catalog:"
|
|
2681
|
+
},
|
|
2682
|
+
sideEffects: false,
|
|
2683
|
+
repository: {
|
|
2684
|
+
type: "git",
|
|
2685
|
+
url: "https://github.com/kortyx-io/kortyx.git",
|
|
2686
|
+
directory: "packages/hooks"
|
|
2687
|
+
},
|
|
2688
|
+
publishConfig: {
|
|
2689
|
+
access: "public"
|
|
2690
|
+
}
|
|
2691
|
+
};
|
|
2692
|
+
var client2 = () => (0, import_node2.getUsageTelemetry)("hooks", package_default2.version);
|
|
2693
|
+
var usage2 = {
|
|
2694
|
+
start: (...args) => client2().start(...args),
|
|
2695
|
+
feature: (...args) => client2().feature(...args),
|
|
2696
|
+
measure: (...args) => client2().measure(...args),
|
|
2697
|
+
flush: () => client2().flush()
|
|
2698
|
+
};
|
|
2207
2699
|
var optionsSchema = import_zod12.z.object({
|
|
2208
2700
|
message: import_zod12.z.string().optional(),
|
|
2209
2701
|
data: import_zod12.z.json().optional()
|
|
2210
2702
|
}).strict();
|
|
2211
|
-
async function
|
|
2703
|
+
async function completeResponseInternal(options = {}) {
|
|
2212
2704
|
const ctx = getHookContext();
|
|
2213
2705
|
const parsed = optionsSchema.parse(options);
|
|
2214
2706
|
if (!ctx.node.completeResponse) {
|
|
@@ -2218,6 +2710,12 @@ var require_dist2 = __commonJS({
|
|
|
2218
2710
|
}
|
|
2219
2711
|
await ctx.node.completeResponse(parsed, snapshotHookState());
|
|
2220
2712
|
}
|
|
2713
|
+
function completeResponse(options = {}) {
|
|
2714
|
+
return usage2.measure(
|
|
2715
|
+
"sdk.complete_response",
|
|
2716
|
+
() => completeResponseInternal(options)
|
|
2717
|
+
);
|
|
2718
|
+
}
|
|
2221
2719
|
var import_core6 = require("@kortyx/core");
|
|
2222
2720
|
var resolveReasonCheckpointKey = (args) => typeof args.id === "string" && args.id.length > 0 ? `__useReason:${args.id}` : `__useReason:auto:${args.autoIndex}`;
|
|
2223
2721
|
var readReasonCheckpoint = (raw) => {
|
|
@@ -2594,7 +3092,7 @@ var require_dist2 = __commonJS({
|
|
|
2594
3092
|
return `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
2595
3093
|
}
|
|
2596
3094
|
};
|
|
2597
|
-
var
|
|
3095
|
+
var emitStructuredDataInternal = (args) => {
|
|
2598
3096
|
const ctx = getHookContext();
|
|
2599
3097
|
const streamId = typeof args.streamId === "string" && args.streamId.length > 0 ? args.streamId : createStructuredStreamId();
|
|
2600
3098
|
const dataType = typeof args.dataType === "string" && args.dataType.length > 0 ? args.dataType : "generic";
|
|
@@ -2657,7 +3155,17 @@ var require_dist2 = __commonJS({
|
|
|
2657
3155
|
};
|
|
2658
3156
|
var shouldEmitStructured = (cfg) => Boolean(cfg);
|
|
2659
3157
|
var shouldStreamStructured = (cfg) => Boolean(cfg) && (cfg?.stream ?? true);
|
|
2660
|
-
var
|
|
3158
|
+
var emitStructuredData = (args) => {
|
|
3159
|
+
const scope = usage2.start("sdk.structured", { format: args.kind });
|
|
3160
|
+
try {
|
|
3161
|
+
emitStructuredDataInternal(args);
|
|
3162
|
+
scope.end();
|
|
3163
|
+
} catch (error) {
|
|
3164
|
+
scope.fail(error);
|
|
3165
|
+
throw error;
|
|
3166
|
+
}
|
|
3167
|
+
};
|
|
3168
|
+
var import_node_async_hooks22 = require("async_hooks");
|
|
2661
3169
|
var import_node_crypto32 = require("crypto");
|
|
2662
3170
|
var import_core22 = require("@kortyx/core");
|
|
2663
3171
|
var import_errors32 = require("@kortyx/core/errors");
|
|
@@ -2947,7 +3455,7 @@ var require_dist2 = __commonJS({
|
|
|
2947
3455
|
const serialized = JSON.stringify(value);
|
|
2948
3456
|
return serialized && Buffer.byteLength(serialized, "utf8") <= 16384 ? { [side]: JSON.parse(serialized) } : { [`${side}Omitted`]: "Content exceeds 16 KiB" };
|
|
2949
3457
|
}
|
|
2950
|
-
var tools = new
|
|
3458
|
+
var tools = new import_node_async_hooks22.AsyncLocalStorage();
|
|
2951
3459
|
function classification(tool, value, thrown, policy = tool.outcomes) {
|
|
2952
3460
|
const fallback = {
|
|
2953
3461
|
outcome: thrown ? "fault" : "success"
|
|
@@ -3122,6 +3630,11 @@ var require_dist2 = __commonJS({
|
|
|
3122
3630
|
}
|
|
3123
3631
|
ctx.node.consumeExecution?.("maxToolCalls");
|
|
3124
3632
|
observation.executed = true;
|
|
3633
|
+
const operationUsage = usage2.start("sdk.tool", {
|
|
3634
|
+
calling_mode: args.callingMode,
|
|
3635
|
+
source: args.tool.source === "mcp" ? "mcp" : "native",
|
|
3636
|
+
executed: true
|
|
3637
|
+
});
|
|
3125
3638
|
const started = performance.now();
|
|
3126
3639
|
return withSafeTraceSpan2(
|
|
3127
3640
|
toolTrace(observation),
|
|
@@ -3171,9 +3684,13 @@ var require_dist2 = __commonJS({
|
|
|
3171
3684
|
scope.telemetry ?? args.tool.telemetry
|
|
3172
3685
|
);
|
|
3173
3686
|
}
|
|
3687
|
+
if (observation.outcome === "fault") operationUsage.fail(error);
|
|
3174
3688
|
throw error;
|
|
3175
3689
|
} finally {
|
|
3176
3690
|
observation.durationMs = Math.max(0, performance.now() - started);
|
|
3691
|
+
operationUsage.end(
|
|
3692
|
+
observation.outcome === "fault" ? "failed" : observation.outcome ?? "suspended"
|
|
3693
|
+
);
|
|
3177
3694
|
span.end?.({
|
|
3178
3695
|
attributes: {
|
|
3179
3696
|
...observation,
|
|
@@ -3283,6 +3800,7 @@ var require_dist2 = __commonJS({
|
|
|
3283
3800
|
});
|
|
3284
3801
|
var import_node_crypto52 = require("crypto");
|
|
3285
3802
|
var import_core42 = require("@kortyx/core");
|
|
3803
|
+
var import_prompts3 = require("@kortyx/prompts");
|
|
3286
3804
|
var import_node_crypto42 = require("crypto");
|
|
3287
3805
|
var import_core32 = require("@kortyx/core");
|
|
3288
3806
|
var import_errors42 = require("@kortyx/core/errors");
|
|
@@ -3325,7 +3843,7 @@ var require_dist2 = __commonJS({
|
|
|
3325
3843
|
}
|
|
3326
3844
|
return providerOptions;
|
|
3327
3845
|
};
|
|
3328
|
-
async function
|
|
3846
|
+
async function runReasonEngineInternal(args) {
|
|
3329
3847
|
let stream = args.stream ?? args.model.options?.streaming ?? true;
|
|
3330
3848
|
const emit = args.emit ?? true;
|
|
3331
3849
|
const temperature = args.temperature ?? args.model.options?.temperature ?? args.defaultTemperature;
|
|
@@ -3352,6 +3870,11 @@ var require_dist2 = __commonJS({
|
|
|
3352
3870
|
...providerOptions !== void 0 ? { providerOptions } : {}
|
|
3353
3871
|
});
|
|
3354
3872
|
if (tools2?.length && !model.supportsToolStreaming) stream = false;
|
|
3873
|
+
usage2.feature("sdk.model", {
|
|
3874
|
+
streaming: stream,
|
|
3875
|
+
tools: Boolean(tools2?.length),
|
|
3876
|
+
provider: args.model.provider.id
|
|
3877
|
+
});
|
|
3355
3878
|
const messages = args.messages !== void 0 ? args.messages : [
|
|
3356
3879
|
...typeof args.system === "string" && args.system.length > 0 ? [{ role: "system", content: args.system }] : [],
|
|
3357
3880
|
{ role: "user", content: String(args.input ?? "") }
|
|
@@ -3431,7 +3954,7 @@ var require_dist2 = __commonJS({
|
|
|
3431
3954
|
let continuation;
|
|
3432
3955
|
let final = "";
|
|
3433
3956
|
let raw;
|
|
3434
|
-
let
|
|
3957
|
+
let usage22;
|
|
3435
3958
|
let finishReason;
|
|
3436
3959
|
let toolCalls;
|
|
3437
3960
|
let providerMetadata;
|
|
@@ -3472,7 +3995,7 @@ var require_dist2 = __commonJS({
|
|
|
3472
3995
|
raw = chunk.raw;
|
|
3473
3996
|
}
|
|
3474
3997
|
if (chunk.usage !== void 0) {
|
|
3475
|
-
|
|
3998
|
+
usage22 = chunk.usage;
|
|
3476
3999
|
}
|
|
3477
4000
|
if (chunk.finishReason !== void 0) {
|
|
3478
4001
|
finishReason = chunk.finishReason;
|
|
@@ -3510,7 +4033,7 @@ var require_dist2 = __commonJS({
|
|
|
3510
4033
|
}
|
|
3511
4034
|
}
|
|
3512
4035
|
(0, import_core32.throwIfExecutionAborted)(abortSignal);
|
|
3513
|
-
assertModelFinish({ finishReason, usage });
|
|
4036
|
+
assertModelFinish({ finishReason, usage: usage22 });
|
|
3514
4037
|
if (emit) {
|
|
3515
4038
|
emitNodeEvent(args.emitEvent, args.nodeId, "text-end", {
|
|
3516
4039
|
...commonMeta
|
|
@@ -3520,7 +4043,7 @@ var require_dist2 = __commonJS({
|
|
|
3520
4043
|
text: final,
|
|
3521
4044
|
...continuation ? { continuation } : {},
|
|
3522
4045
|
...raw !== void 0 ? { raw } : {},
|
|
3523
|
-
...
|
|
4046
|
+
...usage22 !== void 0 ? { usage: usage22 } : {},
|
|
3524
4047
|
...finishReason !== void 0 ? { finishReason } : {},
|
|
3525
4048
|
...toolCalls !== void 0 ? { toolCalls } : {},
|
|
3526
4049
|
...providerMetadata !== void 0 ? { providerMetadata } : {},
|
|
@@ -3566,6 +4089,14 @@ var require_dist2 = __commonJS({
|
|
|
3566
4089
|
throw failure2;
|
|
3567
4090
|
}
|
|
3568
4091
|
}
|
|
4092
|
+
function runReasonEngine(args) {
|
|
4093
|
+
return usage2.measure("sdk.model", () => runReasonEngineInternal(args), {
|
|
4094
|
+
provider: args.model.provider.id,
|
|
4095
|
+
requested_streaming: args.stream ?? args.model.options?.streaming ?? true,
|
|
4096
|
+
tools: Boolean((args.tools ?? args.model.options?.tools)?.length),
|
|
4097
|
+
format: args.responseFormat?.type ?? args.model.options?.responseFormat?.type ?? "text"
|
|
4098
|
+
});
|
|
4099
|
+
}
|
|
3569
4100
|
var createRuntimeId2 = () => {
|
|
3570
4101
|
try {
|
|
3571
4102
|
return (0, import_node_crypto52.randomUUID)();
|
|
@@ -3582,6 +4113,19 @@ var require_dist2 = __commonJS({
|
|
|
3582
4113
|
);
|
|
3583
4114
|
(0, import_core42.throwIfExecutionAborted)(abortSignal);
|
|
3584
4115
|
ctx.node.consumeExecution?.("maxModelPasses");
|
|
4116
|
+
const prompt = args.telemetry?.prompt;
|
|
4117
|
+
const identity = prompt?.metadata;
|
|
4118
|
+
if ((0, import_prompts3.isPromptUsageMetadata)(identity)) {
|
|
4119
|
+
const { dependencies, ...parent } = identity;
|
|
4120
|
+
ctx.node.prompts?.recordUsage({ ...parent, model: args.model.modelId });
|
|
4121
|
+
for (const dependency of dependencies ?? [])
|
|
4122
|
+
ctx.node.prompts?.recordUsage({
|
|
4123
|
+
...dependency,
|
|
4124
|
+
environment: identity.environment,
|
|
4125
|
+
snapshotRevision: identity.snapshotRevision,
|
|
4126
|
+
model: args.model.modelId
|
|
4127
|
+
});
|
|
4128
|
+
}
|
|
3585
4129
|
try {
|
|
3586
4130
|
return await runReasonEngine({
|
|
3587
4131
|
model: args.model,
|
|
@@ -3595,7 +4139,12 @@ var require_dist2 = __commonJS({
|
|
|
3595
4139
|
responseFormat: args.responseFormat,
|
|
3596
4140
|
providerOptions: args.providerOptions,
|
|
3597
4141
|
tools: args.tools,
|
|
3598
|
-
messages: args.messages
|
|
4142
|
+
messages: args.messages && inputOverride !== void 0 ? args.messages.map(
|
|
4143
|
+
(message, index) => index === args.messages.reduce(
|
|
4144
|
+
(last, item, position) => item.role === "user" ? position : last,
|
|
4145
|
+
-1
|
|
4146
|
+
) ? { ...message, content: inputOverride } : message
|
|
4147
|
+
) : args.messages,
|
|
3599
4148
|
defaultTemperature: ctx.node.config?.model?.temperature,
|
|
3600
4149
|
stream: args.stream,
|
|
3601
4150
|
emit: args.emit,
|
|
@@ -3608,8 +4157,8 @@ var require_dist2 = __commonJS({
|
|
|
3608
4157
|
opId: meta.opId
|
|
3609
4158
|
});
|
|
3610
4159
|
} catch (error) {
|
|
3611
|
-
const
|
|
3612
|
-
if (
|
|
4160
|
+
const usage22 = error?.usage;
|
|
4161
|
+
if (usage22) accumulateTokenUsage(usage22);
|
|
3613
4162
|
throw error;
|
|
3614
4163
|
}
|
|
3615
4164
|
}
|
|
@@ -4490,6 +5039,36 @@ var require_dist2 = __commonJS({
|
|
|
4490
5039
|
throw error;
|
|
4491
5040
|
}
|
|
4492
5041
|
};
|
|
5042
|
+
var import_prompts22 = require("@kortyx/prompts");
|
|
5043
|
+
function normalizePromptReasonArgs(args) {
|
|
5044
|
+
if (!("prompt" in args)) return args;
|
|
5045
|
+
if (args.input !== void 0 || args.system !== void 0 || "messages" in args)
|
|
5046
|
+
throw new import_prompts22.PromptError(
|
|
5047
|
+
"PROMPT_REASON_CONFLICT",
|
|
5048
|
+
"Use either prompt or input/system/messages in useReason."
|
|
5049
|
+
);
|
|
5050
|
+
const { prompt, ...rest } = args;
|
|
5051
|
+
if (!(0, import_prompts22.isCompiledPrompt)(prompt))
|
|
5052
|
+
throw new import_prompts22.PromptError(
|
|
5053
|
+
"PROMPT_REASON_INVALID",
|
|
5054
|
+
"useReason requires a compiled prompt returned by usePrompt."
|
|
5055
|
+
);
|
|
5056
|
+
return {
|
|
5057
|
+
...rest,
|
|
5058
|
+
input: prompt.format === "system-user" ? prompt.user ?? "" : prompt.messages.filter((message) => message.role === "user").at(-1)?.content ?? "",
|
|
5059
|
+
...prompt.format === "system-user" ? { system: prompt.system } : { messages: structuredClone(prompt.messages) },
|
|
5060
|
+
telemetry: {
|
|
5061
|
+
...args.telemetry,
|
|
5062
|
+
prompt: {
|
|
5063
|
+
name: prompt.ref.id,
|
|
5064
|
+
version: prompt.ref.version,
|
|
5065
|
+
type: prompt.format === "chat" ? "chat" : "text",
|
|
5066
|
+
source: prompt.ref.source,
|
|
5067
|
+
metadata: (0, import_prompts22.promptUsageMetadata)(prompt)
|
|
5068
|
+
}
|
|
5069
|
+
}
|
|
5070
|
+
};
|
|
5071
|
+
}
|
|
4493
5072
|
var import_zod42 = require("zod");
|
|
4494
5073
|
var tryStringify2 = (value) => {
|
|
4495
5074
|
if (typeof value === "string") return value;
|
|
@@ -4734,19 +5313,34 @@ ${tryStringify2(args.response)}`
|
|
|
4734
5313
|
...value !== void 0 && value !== null && typeof value === "object" && !Array.isArray(value) ? { structuredContent: value } : {}
|
|
4735
5314
|
};
|
|
4736
5315
|
};
|
|
4737
|
-
var createInitialMessages = (args) =>
|
|
4738
|
-
|
|
4739
|
-
{
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
]
|
|
4746
|
-
|
|
4747
|
-
|
|
4748
|
-
|
|
4749
|
-
|
|
5316
|
+
var createInitialMessages = (args) => {
|
|
5317
|
+
if (args.messages) {
|
|
5318
|
+
const ordered = args.messages.map((message) => ({ ...message }));
|
|
5319
|
+
const lastUser = ordered.reduce(
|
|
5320
|
+
(last, message, position) => message.role === "user" ? position : last,
|
|
5321
|
+
-1
|
|
5322
|
+
);
|
|
5323
|
+
if (args.inputOverride && lastUser >= 0)
|
|
5324
|
+
ordered[lastUser] = { ...ordered[lastUser], content: args.input };
|
|
5325
|
+
const instructions = [args.interruptInstructions, args.outputInstructions].filter(Boolean).join("\n\n");
|
|
5326
|
+
if (instructions)
|
|
5327
|
+
ordered.unshift({ role: "system", content: instructions });
|
|
5328
|
+
return ordered;
|
|
5329
|
+
}
|
|
5330
|
+
return [
|
|
5331
|
+
...typeof args.system === "string" || args.interruptInstructions || args.outputInstructions ? [
|
|
5332
|
+
{
|
|
5333
|
+
role: "system",
|
|
5334
|
+
content: [
|
|
5335
|
+
args.system,
|
|
5336
|
+
args.interruptInstructions,
|
|
5337
|
+
args.outputInstructions
|
|
5338
|
+
].filter((value) => Boolean(value)).join("\n\n")
|
|
5339
|
+
}
|
|
5340
|
+
] : [],
|
|
5341
|
+
{ role: "user", content: String(args.input ?? "") }
|
|
5342
|
+
];
|
|
5343
|
+
};
|
|
4750
5344
|
var runReasonToolLoop = async (args) => {
|
|
4751
5345
|
const { useReasonArgs, id: id3, traceSpan, checkpointKey } = args;
|
|
4752
5346
|
const ctx = getHookContext();
|
|
@@ -4888,6 +5482,8 @@ ${tryStringify2(args.response)}`
|
|
|
4888
5482
|
let completed = false;
|
|
4889
5483
|
const maxSteps = Math.max(1, useReasonArgs.toolExecution?.maxSteps ?? 3);
|
|
4890
5484
|
const messages = checkpoint?.messages ?? createInitialMessages({
|
|
5485
|
+
messages: useReasonArgs.messages,
|
|
5486
|
+
inputOverride: Boolean(useReasonArgs.outputSchema),
|
|
4891
5487
|
system: useReasonArgs.system,
|
|
4892
5488
|
input: useReasonArgs.outputSchema ? withOutputGuardrails(useReasonArgs.input, useReasonArgs.outputSchema) : useReasonArgs.input,
|
|
4893
5489
|
...interruptConfig ? {
|
|
@@ -4922,7 +5518,7 @@ ${tryStringify2(args.response)}`
|
|
|
4922
5518
|
let finalRaw;
|
|
4923
5519
|
let finalOutput;
|
|
4924
5520
|
let aggregatedUsage = steps.reduce(
|
|
4925
|
-
(
|
|
5521
|
+
(usage22, step) => mergeUsage(usage22, step.usage),
|
|
4926
5522
|
void 0
|
|
4927
5523
|
);
|
|
4928
5524
|
let finalFinishReason;
|
|
@@ -6023,7 +6619,10 @@ ${tryStringify2(args.response)}`
|
|
|
6023
6619
|
}
|
|
6024
6620
|
);
|
|
6025
6621
|
}
|
|
6026
|
-
async function useReason(
|
|
6622
|
+
async function useReason(request) {
|
|
6623
|
+
const args = normalizePromptReasonArgs(
|
|
6624
|
+
request
|
|
6625
|
+
);
|
|
6027
6626
|
try {
|
|
6028
6627
|
return await runReason(args);
|
|
6029
6628
|
} finally {
|
|
@@ -6321,6 +6920,36 @@ ${tryStringify2(args.response)}`
|
|
|
6321
6920
|
} catch {
|
|
6322
6921
|
}
|
|
6323
6922
|
}
|
|
6923
|
+
var import_prompts32 = require("@kortyx/prompts");
|
|
6924
|
+
async function usePrompt(reference, options) {
|
|
6925
|
+
const ctx = getHookContext();
|
|
6926
|
+
if (!ctx.node.prompts)
|
|
6927
|
+
throw new import_prompts32.PromptError(
|
|
6928
|
+
"PROMPT_SOURCE_NOT_CONFIGURED",
|
|
6929
|
+
"Configure createAgent({ prompts: createPrompts(...) }) before calling usePrompt."
|
|
6930
|
+
);
|
|
6931
|
+
const stored = ctx.workflowState.__promptSnapshot;
|
|
6932
|
+
const snapshot = await ctx.node.prompts.snapshot(stored);
|
|
6933
|
+
if (!stored) {
|
|
6934
|
+
ctx.workflowState.__promptSnapshot = snapshot;
|
|
6935
|
+
ctx.stateDirty = true;
|
|
6936
|
+
}
|
|
6937
|
+
const key = `__promptPin:${reference.id}:${options.version ?? "assignment"}`;
|
|
6938
|
+
const pin = options.version !== void 0 ? await ctx.node.prompts.pin(
|
|
6939
|
+
reference.id,
|
|
6940
|
+
options.version,
|
|
6941
|
+
ctx.workflowState[key]
|
|
6942
|
+
) : void 0;
|
|
6943
|
+
if (pin && !ctx.workflowState[key]) {
|
|
6944
|
+
ctx.workflowState[key] = pin;
|
|
6945
|
+
ctx.stateDirty = true;
|
|
6946
|
+
}
|
|
6947
|
+
return ctx.node.prompts.resolve(reference, {
|
|
6948
|
+
...options,
|
|
6949
|
+
stored: snapshot,
|
|
6950
|
+
...pin ? { pin } : {}
|
|
6951
|
+
});
|
|
6952
|
+
}
|
|
6324
6953
|
var import_node_crypto62 = require("crypto");
|
|
6325
6954
|
var import_core8 = require("@kortyx/core");
|
|
6326
6955
|
var import_errors82 = require("@kortyx/core/errors");
|
|
@@ -6475,14 +7104,14 @@ ${tryStringify2(args.response)}`
|
|
|
6475
7104
|
...content
|
|
6476
7105
|
});
|
|
6477
7106
|
};
|
|
6478
|
-
const recordUsage = (
|
|
6479
|
-
if (!
|
|
7107
|
+
const recordUsage = (usage22) => {
|
|
7108
|
+
if (!usage22) return;
|
|
6480
7109
|
accumulateTokenUsage({
|
|
6481
|
-
input:
|
|
6482
|
-
output:
|
|
6483
|
-
total:
|
|
7110
|
+
input: usage22.input - (record.usage?.input ?? 0),
|
|
7111
|
+
output: usage22.output - (record.usage?.output ?? 0),
|
|
7112
|
+
total: usage22.total - (record.usage?.total ?? 0)
|
|
6484
7113
|
});
|
|
6485
|
-
record.usage =
|
|
7114
|
+
record.usage = usage22;
|
|
6486
7115
|
};
|
|
6487
7116
|
const cached = record.status === "completed";
|
|
6488
7117
|
try {
|
|
@@ -6938,6 +7567,7 @@ var emitTelemetryEvent = (args) => {
|
|
|
6938
7567
|
|
|
6939
7568
|
// src/telemetry/topology.ts
|
|
6940
7569
|
var import_node_crypto2 = require("crypto");
|
|
7570
|
+
var import_eval_attribution = require("@kortyx/core/eval-attribution");
|
|
6941
7571
|
var isRecord2 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
6942
7572
|
var stableValue = (value) => {
|
|
6943
7573
|
if (Array.isArray(value)) return value.map(stableValue);
|
|
@@ -7090,6 +7720,8 @@ var projectWorkflowTopology = (args) => {
|
|
|
7090
7720
|
};
|
|
7091
7721
|
};
|
|
7092
7722
|
var prepareWorkflowTelemetry = (args) => {
|
|
7723
|
+
if (args.sessionId)
|
|
7724
|
+
(0, import_eval_attribution.associateEvalExecution)({ sessionId: args.sessionId, runId: args.runId });
|
|
7093
7725
|
const telemetry = isRecord2(args.config.telemetry) ? args.config.telemetry : void 0;
|
|
7094
7726
|
if (!telemetry?.reporter || !telemetry.environment || !telemetry.service) {
|
|
7095
7727
|
return args.config;
|
|
@@ -7141,6 +7773,100 @@ var prepareWorkflowTelemetry = (args) => {
|
|
|
7141
7773
|
};
|
|
7142
7774
|
};
|
|
7143
7775
|
|
|
7776
|
+
// src/usage.ts
|
|
7777
|
+
var import_node = require("@kortyx/usage-telemetry/node");
|
|
7778
|
+
|
|
7779
|
+
// package.json
|
|
7780
|
+
var package_default = {
|
|
7781
|
+
name: "@kortyx/agent",
|
|
7782
|
+
version: "0.37.0",
|
|
7783
|
+
private: false,
|
|
7784
|
+
description: "Agent composition utilities for Kortyx.",
|
|
7785
|
+
keywords: [
|
|
7786
|
+
"ai",
|
|
7787
|
+
"agent",
|
|
7788
|
+
"agents",
|
|
7789
|
+
"orchestration",
|
|
7790
|
+
"framework",
|
|
7791
|
+
"kortyx"
|
|
7792
|
+
],
|
|
7793
|
+
license: "Apache-2.0",
|
|
7794
|
+
homepage: "https://kortyx.io/",
|
|
7795
|
+
bugs: {
|
|
7796
|
+
url: "https://github.com/kortyx-io/kortyx/issues"
|
|
7797
|
+
},
|
|
7798
|
+
author: "Kortyx (https://github.com/kortyx-io)",
|
|
7799
|
+
main: "./dist/index.js",
|
|
7800
|
+
types: "./dist/index.d.ts",
|
|
7801
|
+
exports: {
|
|
7802
|
+
".": {
|
|
7803
|
+
types: "./dist/index.d.ts",
|
|
7804
|
+
default: "./dist/index.js"
|
|
7805
|
+
},
|
|
7806
|
+
"./browser": {
|
|
7807
|
+
types: "./dist/browser.d.ts",
|
|
7808
|
+
default: "./dist/browser.js"
|
|
7809
|
+
},
|
|
7810
|
+
"./evals": {
|
|
7811
|
+
types: "./dist/evals.d.ts",
|
|
7812
|
+
default: "./dist/evals.js"
|
|
7813
|
+
}
|
|
7814
|
+
},
|
|
7815
|
+
files: [
|
|
7816
|
+
"dist/**",
|
|
7817
|
+
"README.md",
|
|
7818
|
+
"CHANGELOG.md"
|
|
7819
|
+
],
|
|
7820
|
+
scripts: {
|
|
7821
|
+
build: "tsup",
|
|
7822
|
+
predev: "tsc -p tsconfig.build.json",
|
|
7823
|
+
dev: "tsc --watch",
|
|
7824
|
+
lint: "biome check src",
|
|
7825
|
+
test: "vitest run && pnpm -s type-check",
|
|
7826
|
+
coverage: "vitest run --coverage && pnpm -s type-check",
|
|
7827
|
+
"test:watch": "vitest",
|
|
7828
|
+
"type-check": "tsc --noEmit -p tsconfig.test.json"
|
|
7829
|
+
},
|
|
7830
|
+
dependencies: {
|
|
7831
|
+
"@kortyx/core": "workspace:*",
|
|
7832
|
+
"@kortyx/providers": "workspace:*",
|
|
7833
|
+
"@kortyx/runtime": "workspace:*",
|
|
7834
|
+
"@kortyx/stream": "workspace:*",
|
|
7835
|
+
"@kortyx/utils": "workspace:*",
|
|
7836
|
+
"@langchain/core": "catalog:",
|
|
7837
|
+
"@langchain/langgraph": "catalog:",
|
|
7838
|
+
zod: "catalog:",
|
|
7839
|
+
"@kortyx/prompts": "workspace:*",
|
|
7840
|
+
"@kortyx/usage-telemetry": "workspace:*"
|
|
7841
|
+
},
|
|
7842
|
+
devDependencies: {
|
|
7843
|
+
"@kortyx/hooks": "workspace:*",
|
|
7844
|
+
turbo: "catalog:",
|
|
7845
|
+
typescript: "catalog:",
|
|
7846
|
+
tsup: "catalog:",
|
|
7847
|
+
vitest: "catalog:",
|
|
7848
|
+
"@vitest/coverage-v8": "catalog:",
|
|
7849
|
+
"@kortyx/openai": "workspace:*",
|
|
7850
|
+
postgres: "^3.4.7"
|
|
7851
|
+
},
|
|
7852
|
+
sideEffects: false,
|
|
7853
|
+
repository: {
|
|
7854
|
+
type: "git",
|
|
7855
|
+
url: "https://github.com/kortyx-io/kortyx.git",
|
|
7856
|
+
directory: "packages/agent"
|
|
7857
|
+
},
|
|
7858
|
+
publishConfig: {
|
|
7859
|
+
access: "public"
|
|
7860
|
+
}
|
|
7861
|
+
};
|
|
7862
|
+
|
|
7863
|
+
// src/usage.ts
|
|
7864
|
+
var client = () => (0, import_node.getUsageTelemetry)("agent", package_default.version);
|
|
7865
|
+
var usage = {
|
|
7866
|
+
start: (...args) => client().start(...args),
|
|
7867
|
+
measure: (...args) => client().measure(...args)
|
|
7868
|
+
};
|
|
7869
|
+
|
|
7144
7870
|
// src/orchestrator.ts
|
|
7145
7871
|
var isRecord3 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
7146
7872
|
var isGraphState = (value) => isRecord3(value) && typeof value.currentWorkflow === "string";
|
|
@@ -7175,6 +7901,13 @@ async function orchestrateGraphStream({
|
|
|
7175
7901
|
frameworkAdapter,
|
|
7176
7902
|
executionLeaseRelease
|
|
7177
7903
|
}) {
|
|
7904
|
+
let usageLimitReached = false;
|
|
7905
|
+
const executionUsage = usage.start("sdk.execution", {
|
|
7906
|
+
continue_on_disconnect: continueOnDisconnect
|
|
7907
|
+
});
|
|
7908
|
+
const responseUsage = emitOutput ? usage.start("sdk.response", { phase: "delivery" }) : { end() {
|
|
7909
|
+
}, fail(_error) {
|
|
7910
|
+
} };
|
|
7178
7911
|
const out = new import_node_stream.PassThrough({ objectMode: true });
|
|
7179
7912
|
const visibleMessage = emitOutput && clientTurnId && onResponseFinalized && initialConfig.responseCompleted !== true ? (0, import_stream.createFinalizedChatMessageAccumulator)(clientTurnId) : void 0;
|
|
7180
7913
|
let responseFinalized = false;
|
|
@@ -7182,6 +7915,9 @@ async function orchestrateGraphStream({
|
|
|
7182
7915
|
if (!visibleMessage || !onResponseFinalized || !clientTurnId || !sessionId || responseFinalized)
|
|
7183
7916
|
return;
|
|
7184
7917
|
responseFinalized = true;
|
|
7918
|
+
responseUsage.end(
|
|
7919
|
+
status2 === "completed" ? "success" : status2 === "interrupted" ? "suspended" : status2
|
|
7920
|
+
);
|
|
7185
7921
|
try {
|
|
7186
7922
|
await onResponseFinalized({
|
|
7187
7923
|
sessionId,
|
|
@@ -7198,6 +7934,16 @@ async function orchestrateGraphStream({
|
|
|
7198
7934
|
const write = (chunk) => {
|
|
7199
7935
|
if (isRecord3(chunk) && chunk.type === "done" && isGraphState(chunk.data)) {
|
|
7200
7936
|
const { __kortyxParallelGraph: _journal, ...runtime } = chunk.data.runtime;
|
|
7937
|
+
if (isRecord3(runtime.__kortyx) && isRecord3(runtime.__kortyx.workflowState)) {
|
|
7938
|
+
runtime.__kortyx = {
|
|
7939
|
+
...runtime.__kortyx,
|
|
7940
|
+
workflowState: Object.fromEntries(
|
|
7941
|
+
Object.entries(runtime.__kortyx.workflowState).filter(
|
|
7942
|
+
([key]) => key !== "__promptSnapshot" && !key.startsWith("__promptPin:")
|
|
7943
|
+
)
|
|
7944
|
+
)
|
|
7945
|
+
};
|
|
7946
|
+
}
|
|
7201
7947
|
chunk = { ...chunk, data: { ...chunk.data, runtime } };
|
|
7202
7948
|
}
|
|
7203
7949
|
if (emitOutput && !response.closed) {
|
|
@@ -7221,6 +7967,7 @@ async function orchestrateGraphStream({
|
|
|
7221
7967
|
requestSignal: continueOnDisconnect ? void 0 : requestSignal,
|
|
7222
7968
|
executionSignal: executionSignal ? AbortSignal.any([executionSignal, leaseAbort.signal]) : leaseAbort.signal,
|
|
7223
7969
|
onClosed: () => {
|
|
7970
|
+
responseUsage.end();
|
|
7224
7971
|
out.write({ type: "done" });
|
|
7225
7972
|
out.end();
|
|
7226
7973
|
},
|
|
@@ -8207,6 +8954,7 @@ async function orchestrateGraphStream({
|
|
|
8207
8954
|
return;
|
|
8208
8955
|
}
|
|
8209
8956
|
if ((0, import_core2.isExecutionLimitReached)(err)) {
|
|
8957
|
+
usageLimitReached = true;
|
|
8210
8958
|
try {
|
|
8211
8959
|
const snapshot = frameworkAdapter && await (0, import_runtime.captureGraphSnapshot)(frameworkAdapter.checkpointer, runId);
|
|
8212
8960
|
(0, import_core2.throwIfExecutionAborted)(abortSignal);
|
|
@@ -8340,6 +9088,16 @@ async function orchestrateGraphStream({
|
|
|
8340
9088
|
await finalizeVisibleResponse(
|
|
8341
9089
|
outcomeError ? (0, import_core2.isExecutionCancelled)(outcomeError) ? "cancelled" : "failed" : pendingRecordToken || outcomeState.awaitingHumanInput ? "interrupted" : "completed"
|
|
8342
9090
|
);
|
|
9091
|
+
if (outcomeError) executionUsage.fail(outcomeError);
|
|
9092
|
+
else
|
|
9093
|
+
executionUsage.end(
|
|
9094
|
+
usageLimitReached ? "limit_reached" : pendingRecordToken || outcomeState.awaitingHumanInput ? "suspended" : "success"
|
|
9095
|
+
);
|
|
9096
|
+
if (outcomeError) responseUsage.fail(outcomeError);
|
|
9097
|
+
else
|
|
9098
|
+
responseUsage.end(
|
|
9099
|
+
pendingRecordToken || outcomeState.awaitingHumanInput ? "suspended" : "success"
|
|
9100
|
+
);
|
|
8343
9101
|
onOutcome?.({
|
|
8344
9102
|
state: outcomeState,
|
|
8345
9103
|
pending: pendingRecordToken ? activePendingRequests.get(pendingRecordToken) : void 0,
|
|
@@ -8954,6 +9712,10 @@ var import_providers = require("@kortyx/providers");
|
|
|
8954
9712
|
var import_runtime5 = require("@kortyx/runtime");
|
|
8955
9713
|
var import_zod5 = require("zod");
|
|
8956
9714
|
|
|
9715
|
+
// src/evals/prompt-context.ts
|
|
9716
|
+
var import_node_async_hooks = require("async_hooks");
|
|
9717
|
+
var promptEvaluationContext = new import_node_async_hooks.AsyncLocalStorage();
|
|
9718
|
+
|
|
8957
9719
|
// src/execution/execute.ts
|
|
8958
9720
|
var import_core4 = require("@kortyx/core");
|
|
8959
9721
|
var import_errors6 = require("@kortyx/core/errors");
|
|
@@ -8975,12 +9737,12 @@ var responseSchema = import_zod3.z.discriminatedUnion("type", [
|
|
|
8975
9737
|
import_zod3.z.object({ type: import_zod3.z.literal("cancel") }).strict()
|
|
8976
9738
|
]);
|
|
8977
9739
|
function resultFromOutcome(outcome, runId, sessionId) {
|
|
8978
|
-
const
|
|
9740
|
+
const usage2 = outcome.pending?.schema.meta?.__kortyxResumeStatePatch?.tokenUsage ?? outcome.error?.__kortyxHookStatePatch?.tokenUsage ?? outcome.state.runtime?.tokenUsage;
|
|
8979
9741
|
const info = {
|
|
8980
9742
|
runId,
|
|
8981
9743
|
sessionId,
|
|
8982
9744
|
...outcome.checkpointId ? { checkpointId: outcome.checkpointId } : {},
|
|
8983
|
-
...
|
|
9745
|
+
...usage2 ? { usage: usage2 } : {}
|
|
8984
9746
|
};
|
|
8985
9747
|
if ((0, import_core4.isExecutionCancelled)(outcome.error))
|
|
8986
9748
|
return { ...info, status: "cancelled", reason: "Execution cancelled." };
|
|
@@ -9034,6 +9796,7 @@ function resultFromOutcome(outcome, runId, sessionId) {
|
|
|
9034
9796
|
function runtimeConfig(services, sessionId, context) {
|
|
9035
9797
|
return {
|
|
9036
9798
|
session: { id: sessionId },
|
|
9799
|
+
...services.prompts ? { prompts: services.prompts.start() } : {},
|
|
9037
9800
|
...context ? { context } : {},
|
|
9038
9801
|
getProvider: services.getProvider,
|
|
9039
9802
|
checkpointer: services.frameworkAdapter.checkpointer,
|
|
@@ -9572,7 +10335,8 @@ var createAgentArgsBaseSchema = import_zod5.z.object({
|
|
|
9572
10335
|
workflowRegistry: import_zod5.z.unknown().optional(),
|
|
9573
10336
|
defaultWorkflowId: import_zod5.z.string().optional(),
|
|
9574
10337
|
frameworkAdapter: import_zod5.z.unknown().optional(),
|
|
9575
|
-
telemetry: import_zod5.z.unknown().optional()
|
|
10338
|
+
telemetry: import_zod5.z.unknown().optional(),
|
|
10339
|
+
prompts: import_zod5.z.unknown().optional()
|
|
9576
10340
|
}).strict();
|
|
9577
10341
|
var createAgentArgsSchema = createAgentArgsBaseSchema.superRefine(
|
|
9578
10342
|
(value, ctx) => {
|
|
@@ -9633,7 +10397,7 @@ var clearPendingGraphWrites = async (frameworkAdapter, request) => {
|
|
|
9633
10397
|
)
|
|
9634
10398
|
]);
|
|
9635
10399
|
};
|
|
9636
|
-
function
|
|
10400
|
+
function createAgentInternal(args) {
|
|
9637
10401
|
const parsedArgs = parseCreateAgentArgs(args);
|
|
9638
10402
|
const {
|
|
9639
10403
|
limits,
|
|
@@ -9643,7 +10407,8 @@ function createAgent(args) {
|
|
|
9643
10407
|
workflowRegistry,
|
|
9644
10408
|
defaultWorkflowId,
|
|
9645
10409
|
frameworkAdapter,
|
|
9646
|
-
telemetry
|
|
10410
|
+
telemetry,
|
|
10411
|
+
prompts
|
|
9647
10412
|
} = parsedArgs;
|
|
9648
10413
|
const resolvedDefaultWorkflowId = defaultWorkflowId;
|
|
9649
10414
|
const resolvedFrameworkAdapter = frameworkAdapter ?? (0, import_runtime5.createFrameworkAdapterFromEnv)();
|
|
@@ -9668,9 +10433,14 @@ function createAgent(args) {
|
|
|
9668
10433
|
fallbackId: resolvedDefaultWorkflowId ?? "general-chat"
|
|
9669
10434
|
});
|
|
9670
10435
|
})();
|
|
10436
|
+
const registryUsage = usage.start("sdk.registry");
|
|
9671
10437
|
const registeredWorkflowsPromise = registryPromise.then(
|
|
9672
10438
|
(registry) => registry?.list?.()
|
|
9673
10439
|
);
|
|
10440
|
+
void registeredWorkflowsPromise.then(
|
|
10441
|
+
() => registryUsage.end(),
|
|
10442
|
+
(error) => registryUsage.fail(error)
|
|
10443
|
+
);
|
|
9674
10444
|
const projectTopology = async (options) => {
|
|
9675
10445
|
const registeredWorkflows = await registeredWorkflowsPromise;
|
|
9676
10446
|
if (!registeredWorkflows?.length) return [];
|
|
@@ -9684,8 +10454,11 @@ function createAgent(args) {
|
|
|
9684
10454
|
})
|
|
9685
10455
|
);
|
|
9686
10456
|
};
|
|
9687
|
-
const
|
|
10457
|
+
const streamChatInternal = async (messages, options) => {
|
|
9688
10458
|
const parsedOptions = parseAgentProcessOptions(options);
|
|
10459
|
+
const promptEvaluation = promptEvaluationContext.getStore();
|
|
10460
|
+
if (promptEvaluation && !prompts)
|
|
10461
|
+
throw new Error("This application has no prompt manager configured.");
|
|
9689
10462
|
const registry = await registryPromise;
|
|
9690
10463
|
if (!registry) {
|
|
9691
10464
|
throw new Error(
|
|
@@ -9725,12 +10498,35 @@ function createAgent(args) {
|
|
|
9725
10498
|
}
|
|
9726
10499
|
} : {},
|
|
9727
10500
|
...runtimeOptions?.context ? { context: runtimeOptions.context } : {},
|
|
9728
|
-
...telemetry ? { telemetry } : {}
|
|
10501
|
+
...telemetry ? { telemetry } : {},
|
|
10502
|
+
...prompts ? {
|
|
10503
|
+
prompts: prompts.start({
|
|
10504
|
+
...runtimeOptions?.abortSignal ? { signal: runtimeOptions.abortSignal } : {},
|
|
10505
|
+
...promptEvaluation ? {
|
|
10506
|
+
snapshot: promptEvaluation.snapshot,
|
|
10507
|
+
onUsage: promptEvaluation.onUsage
|
|
10508
|
+
} : {}
|
|
10509
|
+
})
|
|
10510
|
+
} : {}
|
|
9729
10511
|
})
|
|
9730
10512
|
});
|
|
9731
10513
|
};
|
|
9732
|
-
const
|
|
9733
|
-
|
|
10514
|
+
const streamChat2 = (messages, options) => usage.measure(
|
|
10515
|
+
"sdk.chat.admission",
|
|
10516
|
+
() => streamChatInternal(messages, options),
|
|
10517
|
+
{
|
|
10518
|
+
phase: "admission",
|
|
10519
|
+
continue_on_disconnect: options?.continueOnDisconnect === true
|
|
10520
|
+
}
|
|
10521
|
+
);
|
|
10522
|
+
const listCheckpoints = (sessionId) => usage.measure(
|
|
10523
|
+
"sdk.checkpoint.list",
|
|
10524
|
+
() => resolvedFrameworkAdapter.sessionCheckpoints.list(sessionId)
|
|
10525
|
+
);
|
|
10526
|
+
const getCheckpoint = (id3) => usage.measure(
|
|
10527
|
+
"sdk.checkpoint.read",
|
|
10528
|
+
() => resolvedFrameworkAdapter.sessionCheckpoints.get(id3)
|
|
10529
|
+
);
|
|
9734
10530
|
const rollbackTo = async (id3) => {
|
|
9735
10531
|
const target = await resolvedFrameworkAdapter.sessionCheckpoints.get(id3);
|
|
9736
10532
|
const sourceRunId = target?.runId ?? "unknown";
|
|
@@ -9842,27 +10638,76 @@ function createAgent(args) {
|
|
|
9842
10638
|
frameworkAdapter: resolvedFrameworkAdapter,
|
|
9843
10639
|
getProvider: resolvedGetProvider,
|
|
9844
10640
|
telemetry,
|
|
10641
|
+
prompts,
|
|
9845
10642
|
knownWorkflowIds: (await registeredWorkflowsPromise)?.map(
|
|
9846
10643
|
(workflow) => workflow.id
|
|
9847
10644
|
)
|
|
9848
10645
|
};
|
|
9849
10646
|
};
|
|
9850
|
-
const
|
|
9851
|
-
|
|
10647
|
+
const observe = async (operation, run) => {
|
|
10648
|
+
const scope = usage.start(operation);
|
|
10649
|
+
try {
|
|
10650
|
+
const result = await run();
|
|
10651
|
+
if (result.status === "failed") scope.fail(result.error);
|
|
10652
|
+
else
|
|
10653
|
+
scope.end(
|
|
10654
|
+
result.status === "completed" ? "success" : result.status === "suspended" && result.reason === "limit_reached" ? "limit_reached" : result.status
|
|
10655
|
+
);
|
|
10656
|
+
return result;
|
|
10657
|
+
} catch (error) {
|
|
10658
|
+
scope.fail(error, { phase: "admission" });
|
|
10659
|
+
throw error;
|
|
10660
|
+
}
|
|
10661
|
+
};
|
|
10662
|
+
const execute = ((options) => observe(
|
|
10663
|
+
"sdk.execute",
|
|
10664
|
+
async () => executeWorkflow(await services(), options)
|
|
10665
|
+
));
|
|
10666
|
+
const resume = ((options) => observe(
|
|
10667
|
+
"sdk.resume",
|
|
10668
|
+
async () => resumeWorkflow(await services(), options)
|
|
10669
|
+
));
|
|
9852
10670
|
return {
|
|
9853
10671
|
...createInterruptDiscovery(resolvedFrameworkAdapter.pendingRequests),
|
|
9854
10672
|
execute,
|
|
9855
10673
|
resume,
|
|
9856
10674
|
projectTopology,
|
|
9857
10675
|
streamChat: streamChat2,
|
|
10676
|
+
describePromptContracts: () => (prompts?.definitions ?? []).map((ref) => ({
|
|
10677
|
+
id: ref.id,
|
|
10678
|
+
format: ref.format,
|
|
10679
|
+
variablesSchema: import_zod5.z.toJSONSchema(ref.variables, { io: "input" }),
|
|
10680
|
+
configSchema: import_zod5.z.toJSONSchema(ref.config, { io: "input" })
|
|
10681
|
+
})),
|
|
9858
10682
|
listCheckpoints,
|
|
9859
10683
|
getCheckpoint,
|
|
9860
|
-
rollbackTo,
|
|
9861
|
-
fork
|
|
10684
|
+
rollbackTo: (id3) => usage.measure("sdk.checkpoint.rollback", () => rollbackTo(id3)),
|
|
10685
|
+
fork: (id3, options) => usage.measure("sdk.checkpoint.fork", () => fork(id3, options))
|
|
9862
10686
|
};
|
|
9863
10687
|
}
|
|
10688
|
+
function createAgent(args) {
|
|
10689
|
+
const scope = usage.start("sdk.initialize");
|
|
10690
|
+
try {
|
|
10691
|
+
const agent = createAgentInternal(args);
|
|
10692
|
+
try {
|
|
10693
|
+
scope.end("success", {
|
|
10694
|
+
source: args.workflowRegistry ? "custom" : args.workflows ? "memory" : "file",
|
|
10695
|
+
storage: args.frameworkAdapter?.kind ?? "custom",
|
|
10696
|
+
prompts: Boolean(args.prompts),
|
|
10697
|
+
observability: Boolean(args.telemetry)
|
|
10698
|
+
});
|
|
10699
|
+
} catch {
|
|
10700
|
+
scope.end();
|
|
10701
|
+
}
|
|
10702
|
+
return agent;
|
|
10703
|
+
} catch (error) {
|
|
10704
|
+
scope.fail(error, { phase: "configuration" });
|
|
10705
|
+
throw error;
|
|
10706
|
+
}
|
|
10707
|
+
}
|
|
9864
10708
|
|
|
9865
10709
|
// src/evals/contracts.ts
|
|
10710
|
+
var import_browser2 = require("@kortyx/prompts/browser");
|
|
9866
10711
|
var import_zod7 = require("zod");
|
|
9867
10712
|
|
|
9868
10713
|
// src/evals/evidence-policy.ts
|
|
@@ -9967,6 +10812,16 @@ var EvalVerdictSchema = import_zod7.z.object({
|
|
|
9967
10812
|
evidence: import_zod7.z.array(import_zod7.z.string().min(1))
|
|
9968
10813
|
}).strict();
|
|
9969
10814
|
var EvalObservationSchema = import_zod7.z.object({
|
|
10815
|
+
promptUsage: import_zod7.z.array(
|
|
10816
|
+
import_zod7.z.object({
|
|
10817
|
+
id: import_zod7.z.string(),
|
|
10818
|
+
version: import_zod7.z.number().int().positive(),
|
|
10819
|
+
hash: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
|
|
10820
|
+
environment: import_zod7.z.string(),
|
|
10821
|
+
snapshotRevision: import_zod7.z.string(),
|
|
10822
|
+
model: import_zod7.z.string().optional()
|
|
10823
|
+
}).strict()
|
|
10824
|
+
).optional(),
|
|
9970
10825
|
type: import_zod7.z.enum(["answer", "interrupt", "error", "cancelled"]),
|
|
9971
10826
|
text: import_zod7.z.string(),
|
|
9972
10827
|
structured: import_zod7.z.array(import_zod7.z.json()),
|
|
@@ -10059,9 +10914,12 @@ var EvalStepResultSchema = import_zod7.z.object({
|
|
|
10059
10914
|
EvalVerdictSchema.extend({ id: import_zod7.z.string(), text: import_zod7.z.string() })
|
|
10060
10915
|
)
|
|
10061
10916
|
}).strict();
|
|
10917
|
+
var runtimeExecution = import_zod7.z.object({ sessionId: import_zod7.z.string().min(1), runId: import_zod7.z.string().min(1) }).strict();
|
|
10062
10918
|
var EvalCaseResultSchema = import_zod7.z.object({
|
|
10063
10919
|
caseId: id2,
|
|
10064
10920
|
repetition: import_zod7.z.number().int().positive(),
|
|
10921
|
+
attemptId: import_zod7.z.uuid().optional(),
|
|
10922
|
+
runtimeExecutions: import_zod7.z.array(runtimeExecution).optional(),
|
|
10065
10923
|
sessionId: import_zod7.z.string(),
|
|
10066
10924
|
status,
|
|
10067
10925
|
durationMs: import_zod7.z.number().nonnegative(),
|
|
@@ -10110,6 +10968,15 @@ var EvalRunResultSchema = import_zod7.z.object({
|
|
|
10110
10968
|
var EvalManifestSchema = import_zod7.z.object({
|
|
10111
10969
|
schemaVersion: import_zod7.z.literal(1),
|
|
10112
10970
|
studioJudging: import_zod7.z.literal(true).optional(),
|
|
10971
|
+
promptContracts: import_zod7.z.array(
|
|
10972
|
+
import_zod7.z.object({
|
|
10973
|
+
id: import_zod7.z.string(),
|
|
10974
|
+
format: import_zod7.z.enum(["system-user", "chat"]),
|
|
10975
|
+
variablesSchema: import_zod7.z.record(import_zod7.z.string(), import_zod7.z.unknown()),
|
|
10976
|
+
configSchema: import_zod7.z.record(import_zod7.z.string(), import_zod7.z.unknown())
|
|
10977
|
+
}).strict()
|
|
10978
|
+
).optional(),
|
|
10979
|
+
attemptScheduling: import_zod7.z.literal(true).optional(),
|
|
10113
10980
|
suites: import_zod7.z.array(EvalSuiteSchema),
|
|
10114
10981
|
responders: import_zod7.z.array(
|
|
10115
10982
|
import_zod7.z.object({
|
|
@@ -10123,6 +10990,13 @@ var EvalManifestSchema = import_zod7.z.object({
|
|
|
10123
10990
|
judge: judge.optional()
|
|
10124
10991
|
}).strict();
|
|
10125
10992
|
var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
|
|
10993
|
+
import_zod7.z.object({
|
|
10994
|
+
type: import_zod7.z.literal("case-runtime-associated"),
|
|
10995
|
+
caseId: id2,
|
|
10996
|
+
repetition: import_zod7.z.number().int().positive(),
|
|
10997
|
+
attemptId: import_zod7.z.uuid(),
|
|
10998
|
+
execution: runtimeExecution
|
|
10999
|
+
}).strict(),
|
|
10126
11000
|
import_zod7.z.object({
|
|
10127
11001
|
type: import_zod7.z.literal("run-started"),
|
|
10128
11002
|
caseIds: import_zod7.z.array(id2).min(1),
|
|
@@ -10138,6 +11012,7 @@ var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
|
|
|
10138
11012
|
}).strict(),
|
|
10139
11013
|
import_zod7.z.object({
|
|
10140
11014
|
type: import_zod7.z.literal("case-started"),
|
|
11015
|
+
attemptId: import_zod7.z.uuid().optional(),
|
|
10141
11016
|
caseId: id2,
|
|
10142
11017
|
repetition: import_zod7.z.number().int().positive(),
|
|
10143
11018
|
sessionId: import_zod7.z.string()
|
|
@@ -10151,18 +11026,37 @@ var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
|
|
|
10151
11026
|
import_zod7.z.object({ type: import_zod7.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
|
|
10152
11027
|
]);
|
|
10153
11028
|
var EvalRemoteRunRequestSchema = import_zod7.z.object({
|
|
11029
|
+
promptSnapshot: import_browser2.PromptSnapshotSchema.optional(),
|
|
11030
|
+
promptGroupName: import_zod7.z.string().optional(),
|
|
10154
11031
|
suiteId: id2,
|
|
10155
11032
|
suiteRevision: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
|
|
10156
11033
|
grading: import_zod7.z.enum(["app", "studio"]).optional(),
|
|
10157
11034
|
judge: EvalJudgeIdentitySchema.optional(),
|
|
10158
11035
|
caseIds: import_zod7.z.array(id2).min(1).max(100).optional(),
|
|
10159
11036
|
repetitions: import_zod7.z.number().int().min(1).max(20).default(1),
|
|
10160
|
-
concurrency: import_zod7.z.number().int().min(1).max(20).default(1)
|
|
11037
|
+
concurrency: import_zod7.z.number().int().min(1).max(20).default(1),
|
|
11038
|
+
attempt: import_zod7.z.object({
|
|
11039
|
+
caseId: id2,
|
|
11040
|
+
repetition: import_zod7.z.number().int().min(1).max(20),
|
|
11041
|
+
evaluationId: import_zod7.z.uuid()
|
|
11042
|
+
}).strict().optional()
|
|
10161
11043
|
}).strict();
|
|
10162
11044
|
var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
|
|
10163
11045
|
grading: true,
|
|
10164
|
-
judge: true
|
|
11046
|
+
judge: true,
|
|
11047
|
+
promptSnapshot: true,
|
|
11048
|
+
promptGroupName: true,
|
|
11049
|
+
attempt: true
|
|
10165
11050
|
}).extend({
|
|
11051
|
+
promptSelection: import_zod7.z.discriminatedUnion("type", [
|
|
11052
|
+
import_zod7.z.object({ type: import_zod7.z.literal("live") }).strict(),
|
|
11053
|
+
import_zod7.z.object({
|
|
11054
|
+
type: import_zod7.z.literal("single"),
|
|
11055
|
+
id: import_zod7.z.string(),
|
|
11056
|
+
version: import_zod7.z.number().int().positive()
|
|
11057
|
+
}).strict(),
|
|
11058
|
+
import_zod7.z.object({ type: import_zod7.z.literal("group"), groupId: import_zod7.z.uuid() }).strict()
|
|
11059
|
+
]).optional(),
|
|
10166
11060
|
targetId: import_zod7.z.string().min(1).max(128),
|
|
10167
11061
|
judge: import_zod7.z.enum(["studio", "app"]).default("studio")
|
|
10168
11062
|
});
|
|
@@ -10198,6 +11092,8 @@ function parseEvalSuite(value) {
|
|
|
10198
11092
|
|
|
10199
11093
|
// src/evals/create-evals.ts
|
|
10200
11094
|
var import_node_crypto5 = require("crypto");
|
|
11095
|
+
var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
|
|
11096
|
+
var import_usage_telemetry = require("@kortyx/usage-telemetry");
|
|
10201
11097
|
var import_zod8 = require("zod");
|
|
10202
11098
|
|
|
10203
11099
|
// src/evals/evidence.ts
|
|
@@ -10325,6 +11221,7 @@ function canonical(value) {
|
|
|
10325
11221
|
var getEvalSuiteRevision = (suite) => (0, import_node_crypto4.createHash)("sha256").update(canonical(suite)).digest("hex");
|
|
10326
11222
|
|
|
10327
11223
|
// src/evals/stream.ts
|
|
11224
|
+
var import_prompts = require("@kortyx/prompts");
|
|
10328
11225
|
var import_stream3 = require("@kortyx/stream");
|
|
10329
11226
|
|
|
10330
11227
|
// src/evals/stream-evidence.ts
|
|
@@ -10462,6 +11359,17 @@ function captureEvalEvent(chunk) {
|
|
|
10462
11359
|
|
|
10463
11360
|
// src/evals/stream.ts
|
|
10464
11361
|
async function executeEvalChat(args) {
|
|
11362
|
+
if (args.promptSnapshot) {
|
|
11363
|
+
const snapshot = await (0, import_prompts.verifyPromptSnapshot)(args.promptSnapshot);
|
|
11364
|
+
const receipts = [];
|
|
11365
|
+
const { promptSnapshot: _, ...executionArgs } = args;
|
|
11366
|
+
const result = await promptEvaluationContext.run(
|
|
11367
|
+
{ snapshot, onUsage: (receipt) => receipts.push(receipt) },
|
|
11368
|
+
() => executeEvalChat(executionArgs)
|
|
11369
|
+
);
|
|
11370
|
+
result.observation.promptUsage = receipts;
|
|
11371
|
+
return result;
|
|
11372
|
+
}
|
|
10465
11373
|
const waiting = args.continuation;
|
|
10466
11374
|
if (args.command.type === "cancel" && waiting && waiting.length > 1) {
|
|
10467
11375
|
let failed = false;
|
|
@@ -10683,6 +11591,8 @@ function createEvals(options) {
|
|
|
10683
11591
|
return {
|
|
10684
11592
|
schemaVersion: 1,
|
|
10685
11593
|
studioJudging: true,
|
|
11594
|
+
...options.agent.describePromptContracts ? { promptContracts: options.agent.describePromptContracts() } : {},
|
|
11595
|
+
attemptScheduling: true,
|
|
10686
11596
|
suites: clone(suites),
|
|
10687
11597
|
responders: Object.entries(responders).map(([name, value]) => ({
|
|
10688
11598
|
name,
|
|
@@ -10727,6 +11637,8 @@ function createEvals(options) {
|
|
|
10727
11637
|
);
|
|
10728
11638
|
if (!Number.isSafeInteger(selected.length * repetitions))
|
|
10729
11639
|
throw new EvalConfigurationError("Too many case repetitions.");
|
|
11640
|
+
if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
|
|
11641
|
+
throw new EvalConfigurationError("Invalid scheduled attempt.");
|
|
10730
11642
|
const studioGrading = args.grading === "studio";
|
|
10731
11643
|
if (!studioGrading && !options.judge && selected.some(
|
|
10732
11644
|
(item) => item.steps.some((step) => step.expect.criteria?.length)
|
|
@@ -10739,6 +11651,9 @@ function createEvals(options) {
|
|
|
10739
11651
|
throw new EvalConfigurationError(
|
|
10740
11652
|
"App judge changed. Refresh before running."
|
|
10741
11653
|
);
|
|
11654
|
+
const evalUsage = usage.start("sdk.eval", {
|
|
11655
|
+
scale: (0, import_usage_telemetry.scale)(selected.length * repetitions)
|
|
11656
|
+
});
|
|
10742
11657
|
const start = Date.now();
|
|
10743
11658
|
const result = {
|
|
10744
11659
|
id: (0, import_node_crypto5.randomUUID)(),
|
|
@@ -10763,7 +11678,10 @@ function createEvals(options) {
|
|
|
10763
11678
|
const emit = async (event) => {
|
|
10764
11679
|
if (!args.onProgress || !reporting) return;
|
|
10765
11680
|
try {
|
|
10766
|
-
await
|
|
11681
|
+
await (0, import_eval_attribution2.withEvalAttribution)(
|
|
11682
|
+
void 0,
|
|
11683
|
+
() => args.onProgress(clone(event))
|
|
11684
|
+
);
|
|
10767
11685
|
} catch {
|
|
10768
11686
|
reporting = false;
|
|
10769
11687
|
result.errors.push(failure("reporting"));
|
|
@@ -10778,138 +11696,221 @@ function createEvals(options) {
|
|
|
10778
11696
|
controller.abort();
|
|
10779
11697
|
}, defaults.caseTimeoutMs);
|
|
10780
11698
|
const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
|
|
11699
|
+
const attemptId = (0, import_node_crypto5.randomUUID)();
|
|
11700
|
+
const runtimeExecutions = [];
|
|
10781
11701
|
const attemptResult = {
|
|
10782
11702
|
caseId: item.id,
|
|
10783
11703
|
repetition,
|
|
11704
|
+
attemptId,
|
|
11705
|
+
runtimeExecutions,
|
|
10784
11706
|
sessionId: `eval-${(0, import_node_crypto5.randomUUID)()}`,
|
|
10785
11707
|
status: "passed",
|
|
10786
11708
|
durationMs: 0,
|
|
10787
11709
|
steps: [],
|
|
10788
11710
|
errors: []
|
|
10789
11711
|
};
|
|
10790
|
-
let
|
|
10791
|
-
let
|
|
10792
|
-
const
|
|
10793
|
-
phase =
|
|
10794
|
-
|
|
11712
|
+
let associationProgress = Promise.resolve();
|
|
11713
|
+
let acceptingExecutions = true;
|
|
11714
|
+
const executeAttempt = async () => {
|
|
11715
|
+
let phase = "params";
|
|
11716
|
+
let stepIndex;
|
|
11717
|
+
const reportPhase = async (next) => {
|
|
11718
|
+
phase = next;
|
|
11719
|
+
if (args.includeActivity)
|
|
11720
|
+
await emit({
|
|
11721
|
+
type: "case-progress",
|
|
11722
|
+
caseId: item.id,
|
|
11723
|
+
repetition,
|
|
11724
|
+
phase,
|
|
11725
|
+
...stepIndex !== void 0 ? { stepIndex } : {}
|
|
11726
|
+
});
|
|
11727
|
+
};
|
|
11728
|
+
let context;
|
|
11729
|
+
let continuation;
|
|
11730
|
+
let activeStep;
|
|
11731
|
+
let activeStepReported = false;
|
|
11732
|
+
const history = [];
|
|
11733
|
+
const call = async (readyContext, command, activeSignal) => {
|
|
11734
|
+
const run = (overrides) => executeEvalChat({
|
|
11735
|
+
agent: options.agent,
|
|
11736
|
+
...args.promptSnapshot ? { promptSnapshot: args.promptSnapshot } : {},
|
|
11737
|
+
command,
|
|
11738
|
+
continuation,
|
|
11739
|
+
history: clone(history),
|
|
11740
|
+
sessionId: attemptResult.sessionId,
|
|
11741
|
+
clientTurnId: (0, import_node_crypto5.randomUUID)(),
|
|
11742
|
+
signal: activeSignal,
|
|
11743
|
+
...item.workflowId ? { workflowId: item.workflowId } : {},
|
|
11744
|
+
...overrides
|
|
11745
|
+
});
|
|
11746
|
+
const execution = options.execute ? await options.execute({
|
|
11747
|
+
...readyContext,
|
|
11748
|
+
signal: activeSignal,
|
|
11749
|
+
command,
|
|
11750
|
+
continuation,
|
|
11751
|
+
history: clone(history),
|
|
11752
|
+
run
|
|
11753
|
+
}) : await run();
|
|
11754
|
+
continuation = execution.continuation;
|
|
11755
|
+
const observation = EvalObservationSchema.parse(
|
|
11756
|
+
execution.observation
|
|
11757
|
+
);
|
|
11758
|
+
if (observation.type === "interrupt" && execution.continuation === void 0)
|
|
11759
|
+
throw new Error(
|
|
11760
|
+
"A waiting interrupt requires a private continuation."
|
|
11761
|
+
);
|
|
11762
|
+
return {
|
|
11763
|
+
observation,
|
|
11764
|
+
...continuation !== void 0 ? { continuation } : {}
|
|
11765
|
+
};
|
|
11766
|
+
};
|
|
11767
|
+
try {
|
|
10795
11768
|
await emit({
|
|
10796
|
-
type: "case-
|
|
11769
|
+
type: "case-started",
|
|
11770
|
+
attemptId,
|
|
10797
11771
|
caseId: item.id,
|
|
10798
11772
|
repetition,
|
|
10799
|
-
|
|
10800
|
-
...stepIndex !== void 0 ? { stepIndex } : {}
|
|
11773
|
+
sessionId: attemptResult.sessionId
|
|
10801
11774
|
});
|
|
10802
|
-
};
|
|
10803
|
-
let context;
|
|
10804
|
-
let continuation;
|
|
10805
|
-
let activeStep;
|
|
10806
|
-
let activeStepReported = false;
|
|
10807
|
-
const history = [];
|
|
10808
|
-
const call = async (readyContext, command, activeSignal) => {
|
|
10809
|
-
const run = (overrides) => executeEvalChat({
|
|
10810
|
-
agent: options.agent,
|
|
10811
|
-
command,
|
|
10812
|
-
continuation,
|
|
10813
|
-
history: clone(history),
|
|
10814
|
-
sessionId: attemptResult.sessionId,
|
|
10815
|
-
clientTurnId: (0, import_node_crypto5.randomUUID)(),
|
|
10816
|
-
signal: activeSignal,
|
|
10817
|
-
...item.workflowId ? { workflowId: item.workflowId } : {},
|
|
10818
|
-
...overrides
|
|
10819
|
-
});
|
|
10820
|
-
const execution = options.execute ? await options.execute({
|
|
10821
|
-
...readyContext,
|
|
10822
|
-
signal: activeSignal,
|
|
10823
|
-
command,
|
|
10824
|
-
continuation,
|
|
10825
|
-
history: clone(history),
|
|
10826
|
-
run
|
|
10827
|
-
}) : await run();
|
|
10828
|
-
continuation = execution.continuation;
|
|
10829
|
-
const observation = EvalObservationSchema.parse(
|
|
10830
|
-
execution.observation
|
|
10831
|
-
);
|
|
10832
|
-
if (observation.type === "interrupt" && execution.continuation === void 0)
|
|
10833
|
-
throw new Error(
|
|
10834
|
-
"A waiting interrupt requires a private continuation."
|
|
10835
|
-
);
|
|
10836
|
-
return {
|
|
10837
|
-
observation,
|
|
10838
|
-
...continuation !== void 0 ? { continuation } : {}
|
|
10839
|
-
};
|
|
10840
|
-
};
|
|
10841
|
-
try {
|
|
10842
|
-
await emit({
|
|
10843
|
-
type: "case-started",
|
|
10844
|
-
caseId: item.id,
|
|
10845
|
-
repetition,
|
|
10846
|
-
sessionId: attemptResult.sessionId
|
|
10847
|
-
});
|
|
10848
|
-
signal.throwIfAborted();
|
|
10849
|
-
await reportPhase("params");
|
|
10850
|
-
const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
|
|
10851
|
-
const base = {
|
|
10852
|
-
suiteId: suite.id,
|
|
10853
|
-
case: item,
|
|
10854
|
-
repetition,
|
|
10855
|
-
sessionId: attemptResult.sessionId,
|
|
10856
|
-
params,
|
|
10857
|
-
signal
|
|
10858
|
-
};
|
|
10859
|
-
await reportPhase("setup");
|
|
10860
|
-
const prepared = options.setup ? await options.setup(base) : void 0;
|
|
10861
|
-
context = { ...base, prepared };
|
|
10862
|
-
signal.throwIfAborted();
|
|
10863
|
-
for (const [index, step] of item.steps.entries()) {
|
|
10864
|
-
stepIndex = index;
|
|
10865
|
-
activeStep = void 0;
|
|
10866
|
-
activeStepReported = false;
|
|
10867
11775
|
signal.throwIfAborted();
|
|
10868
|
-
|
|
10869
|
-
|
|
10870
|
-
|
|
10871
|
-
|
|
10872
|
-
|
|
10873
|
-
|
|
10874
|
-
|
|
10875
|
-
|
|
10876
|
-
|
|
10877
|
-
if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
|
|
10878
|
-
throw new Error("Incompatible responder.");
|
|
10879
|
-
response = await (typeof responder === "function" ? responder : responder.respond)({
|
|
10880
|
-
...context,
|
|
10881
|
-
interrupt: clone(interrupt),
|
|
10882
|
-
...response.params !== void 0 ? { args: response.params } : {}
|
|
10883
|
-
});
|
|
10884
|
-
}
|
|
10885
|
-
command = {
|
|
10886
|
-
type: "resume",
|
|
10887
|
-
// z.json() is inferred as optional under the package's relaxed
|
|
10888
|
-
// declaration build. Runtime validation still requires value.
|
|
10889
|
-
response: EvalResumeResponseSchema.parse(
|
|
10890
|
-
response
|
|
10891
|
-
)
|
|
10892
|
-
};
|
|
10893
|
-
}
|
|
10894
|
-
signal.throwIfAborted();
|
|
10895
|
-
await reportPhase("execute");
|
|
10896
|
-
const { observation } = await call(context, command, signal);
|
|
10897
|
-
const input = command.type === "message" ? { message: command.message } : { resume: command.response };
|
|
10898
|
-
const evaluated = {
|
|
10899
|
-
index,
|
|
10900
|
-
input,
|
|
10901
|
-
expectation: clone(step.expect),
|
|
10902
|
-
observation,
|
|
10903
|
-
status: "passed",
|
|
10904
|
-
criteria: []
|
|
11776
|
+
await reportPhase("params");
|
|
11777
|
+
const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
|
|
11778
|
+
const base = {
|
|
11779
|
+
suiteId: suite.id,
|
|
11780
|
+
case: item,
|
|
11781
|
+
repetition,
|
|
11782
|
+
sessionId: attemptResult.sessionId,
|
|
11783
|
+
params,
|
|
11784
|
+
signal
|
|
10905
11785
|
};
|
|
10906
|
-
|
|
10907
|
-
|
|
11786
|
+
await reportPhase("setup");
|
|
11787
|
+
const prepared = options.setup ? await options.setup(base) : void 0;
|
|
11788
|
+
context = { ...base, prepared };
|
|
10908
11789
|
signal.throwIfAborted();
|
|
10909
|
-
|
|
10910
|
-
|
|
10911
|
-
|
|
10912
|
-
|
|
11790
|
+
for (const [index, step] of item.steps.entries()) {
|
|
11791
|
+
stepIndex = index;
|
|
11792
|
+
activeStep = void 0;
|
|
11793
|
+
activeStepReported = false;
|
|
11794
|
+
signal.throwIfAborted();
|
|
11795
|
+
let command;
|
|
11796
|
+
if (step.message !== void 0)
|
|
11797
|
+
command = { type: "message", message: step.message };
|
|
11798
|
+
else {
|
|
11799
|
+
await reportPhase("responder");
|
|
11800
|
+
const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
|
|
11801
|
+
let response = step.resume;
|
|
11802
|
+
if (handlerRef(response)) {
|
|
11803
|
+
const responder = responders[response.using];
|
|
11804
|
+
if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
|
|
11805
|
+
throw new Error("Incompatible responder.");
|
|
11806
|
+
response = await (typeof responder === "function" ? responder : responder.respond)({
|
|
11807
|
+
...context,
|
|
11808
|
+
interrupt: clone(interrupt),
|
|
11809
|
+
...response.params !== void 0 ? { args: response.params } : {}
|
|
11810
|
+
});
|
|
11811
|
+
}
|
|
11812
|
+
command = {
|
|
11813
|
+
type: "resume",
|
|
11814
|
+
// z.json() is inferred as optional under the package's relaxed
|
|
11815
|
+
// declaration build. Runtime validation still requires value.
|
|
11816
|
+
response: EvalResumeResponseSchema.parse(
|
|
11817
|
+
response
|
|
11818
|
+
)
|
|
11819
|
+
};
|
|
11820
|
+
}
|
|
11821
|
+
signal.throwIfAborted();
|
|
11822
|
+
await reportPhase("execute");
|
|
11823
|
+
const { observation } = await call(context, command, signal);
|
|
11824
|
+
const input = command.type === "message" ? { message: command.message } : { resume: command.response };
|
|
11825
|
+
const evaluated = {
|
|
11826
|
+
index,
|
|
11827
|
+
input,
|
|
11828
|
+
expectation: clone(step.expect),
|
|
11829
|
+
observation,
|
|
11830
|
+
status: "passed",
|
|
11831
|
+
criteria: []
|
|
11832
|
+
};
|
|
11833
|
+
attemptResult.steps.push(evaluated);
|
|
11834
|
+
activeStep = evaluated;
|
|
11835
|
+
signal.throwIfAborted();
|
|
11836
|
+
if (observation.type === "error")
|
|
11837
|
+
throw new Error("Agent failed.");
|
|
11838
|
+
if (observation.type === "cancelled") {
|
|
11839
|
+
evaluated.status = "cancelled";
|
|
11840
|
+
attemptResult.status = "cancelled";
|
|
11841
|
+
await emit({
|
|
11842
|
+
type: "step-completed",
|
|
11843
|
+
caseId: item.id,
|
|
11844
|
+
repetition,
|
|
11845
|
+
step: evaluated
|
|
11846
|
+
});
|
|
11847
|
+
activeStepReported = true;
|
|
11848
|
+
break;
|
|
11849
|
+
}
|
|
11850
|
+
if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
|
|
11851
|
+
evaluated.status = "failed";
|
|
11852
|
+
evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
|
|
11853
|
+
} else {
|
|
11854
|
+
const outputReason = missingOutputReason(
|
|
11855
|
+
step.expect.outputs,
|
|
11856
|
+
observation
|
|
11857
|
+
);
|
|
11858
|
+
if (outputReason) {
|
|
11859
|
+
evaluated.status = "failed";
|
|
11860
|
+
evaluated.reason = outputReason;
|
|
11861
|
+
} else {
|
|
11862
|
+
const reference = step.expect.reference;
|
|
11863
|
+
if (reference !== void 0) {
|
|
11864
|
+
await reportPhase("reference");
|
|
11865
|
+
evaluated.reference = handlerRef(reference) ? import_zod8.z.json().parse(
|
|
11866
|
+
await references[reference.using]({
|
|
11867
|
+
...context,
|
|
11868
|
+
observation: clone(observation),
|
|
11869
|
+
...reference.params !== void 0 ? { args: reference.params } : {}
|
|
11870
|
+
})
|
|
11871
|
+
) : clone(reference);
|
|
11872
|
+
}
|
|
11873
|
+
await reportPhase("grading");
|
|
11874
|
+
evaluated.evidence = createEvalJudgeEvidence(
|
|
11875
|
+
observation,
|
|
11876
|
+
suite.evidence,
|
|
11877
|
+
evidenceFilters
|
|
11878
|
+
);
|
|
11879
|
+
if (studioGrading && step.expect.criteria?.length) {
|
|
11880
|
+
evaluated.status = "ungraded";
|
|
11881
|
+
attemptResult.status = "ungraded";
|
|
11882
|
+
}
|
|
11883
|
+
for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
|
|
11884
|
+
signal.throwIfAborted();
|
|
11885
|
+
const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
|
|
11886
|
+
evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
|
|
11887
|
+
const verdict = EvalVerdictSchema.parse(
|
|
11888
|
+
await (0, import_eval_attribution2.withEvalAttribution)(
|
|
11889
|
+
void 0,
|
|
11890
|
+
() => options.judge.grade({
|
|
11891
|
+
criterion,
|
|
11892
|
+
onUsage: (usage2) => {
|
|
11893
|
+
if (!signal.aborted) {
|
|
11894
|
+
evaluated.judgeUsage ??= [];
|
|
11895
|
+
evaluated.judgeUsage.push(clone(usage2));
|
|
11896
|
+
}
|
|
11897
|
+
},
|
|
11898
|
+
input: clone(input),
|
|
11899
|
+
...getEvalGradeEvidence(
|
|
11900
|
+
evaluated,
|
|
11901
|
+
attemptResult.steps.slice(0, -1)
|
|
11902
|
+
),
|
|
11903
|
+
signal,
|
|
11904
|
+
...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
|
|
11905
|
+
})
|
|
11906
|
+
)
|
|
11907
|
+
);
|
|
11908
|
+
signal.throwIfAborted();
|
|
11909
|
+
evaluated.criteria.push({ ...criterion, ...verdict });
|
|
11910
|
+
if (!verdict.passed) evaluated.status = "failed";
|
|
11911
|
+
}
|
|
11912
|
+
}
|
|
11913
|
+
}
|
|
10913
11914
|
await emit({
|
|
10914
11915
|
type: "step-completed",
|
|
10915
11916
|
caseId: item.id,
|
|
@@ -10917,161 +11918,116 @@ function createEvals(options) {
|
|
|
10917
11918
|
step: evaluated
|
|
10918
11919
|
});
|
|
10919
11920
|
activeStepReported = true;
|
|
10920
|
-
|
|
10921
|
-
|
|
10922
|
-
|
|
10923
|
-
evaluated.status = "failed";
|
|
10924
|
-
evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
|
|
10925
|
-
} else {
|
|
10926
|
-
const outputReason = missingOutputReason(
|
|
10927
|
-
step.expect.outputs,
|
|
10928
|
-
observation
|
|
10929
|
-
);
|
|
10930
|
-
if (outputReason) {
|
|
10931
|
-
evaluated.status = "failed";
|
|
10932
|
-
evaluated.reason = outputReason;
|
|
10933
|
-
} else {
|
|
10934
|
-
const reference = step.expect.reference;
|
|
10935
|
-
if (reference !== void 0) {
|
|
10936
|
-
await reportPhase("reference");
|
|
10937
|
-
evaluated.reference = handlerRef(reference) ? import_zod8.z.json().parse(
|
|
10938
|
-
await references[reference.using]({
|
|
10939
|
-
...context,
|
|
10940
|
-
observation: clone(observation),
|
|
10941
|
-
...reference.params !== void 0 ? { args: reference.params } : {}
|
|
10942
|
-
})
|
|
10943
|
-
) : clone(reference);
|
|
10944
|
-
}
|
|
10945
|
-
await reportPhase("grading");
|
|
10946
|
-
evaluated.evidence = createEvalJudgeEvidence(
|
|
10947
|
-
observation,
|
|
10948
|
-
suite.evidence,
|
|
10949
|
-
evidenceFilters
|
|
10950
|
-
);
|
|
10951
|
-
if (studioGrading && step.expect.criteria?.length) {
|
|
10952
|
-
evaluated.status = "ungraded";
|
|
10953
|
-
attemptResult.status = "ungraded";
|
|
10954
|
-
}
|
|
10955
|
-
for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
|
|
10956
|
-
signal.throwIfAborted();
|
|
10957
|
-
const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
|
|
10958
|
-
evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
|
|
10959
|
-
const verdict = EvalVerdictSchema.parse(
|
|
10960
|
-
await options.judge.grade({
|
|
10961
|
-
criterion,
|
|
10962
|
-
onUsage: (usage) => {
|
|
10963
|
-
if (!signal.aborted) {
|
|
10964
|
-
evaluated.judgeUsage ??= [];
|
|
10965
|
-
evaluated.judgeUsage.push(clone(usage));
|
|
10966
|
-
}
|
|
10967
|
-
},
|
|
10968
|
-
input: clone(input),
|
|
10969
|
-
...getEvalGradeEvidence(
|
|
10970
|
-
evaluated,
|
|
10971
|
-
attemptResult.steps.slice(0, -1)
|
|
10972
|
-
),
|
|
10973
|
-
signal,
|
|
10974
|
-
...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
|
|
10975
|
-
})
|
|
10976
|
-
);
|
|
10977
|
-
signal.throwIfAborted();
|
|
10978
|
-
evaluated.criteria.push({ ...criterion, ...verdict });
|
|
10979
|
-
if (!verdict.passed) evaluated.status = "failed";
|
|
10980
|
-
}
|
|
11921
|
+
if (evaluated.status === "failed") {
|
|
11922
|
+
attemptResult.status = "failed";
|
|
11923
|
+
break;
|
|
10981
11924
|
}
|
|
11925
|
+
history.push(
|
|
11926
|
+
{
|
|
11927
|
+
role: "user",
|
|
11928
|
+
content: command.type === "message" ? command.message : JSON.stringify(command.response)
|
|
11929
|
+
},
|
|
11930
|
+
{ role: "assistant", content: observation.text }
|
|
11931
|
+
);
|
|
10982
11932
|
}
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
|
|
10986
|
-
|
|
10987
|
-
|
|
10988
|
-
|
|
10989
|
-
|
|
10990
|
-
|
|
10991
|
-
|
|
10992
|
-
|
|
11933
|
+
} catch {
|
|
11934
|
+
if (signal.aborted) {
|
|
11935
|
+
attemptResult.status = timedOut ? "error" : "cancelled";
|
|
11936
|
+
if (timedOut)
|
|
11937
|
+
attemptResult.errors.push({
|
|
11938
|
+
phase,
|
|
11939
|
+
code: "EVAL_TIMEOUT",
|
|
11940
|
+
message: "Case exceeded its configured time limit."
|
|
11941
|
+
});
|
|
11942
|
+
} else {
|
|
11943
|
+
attemptResult.status = "error";
|
|
11944
|
+
attemptResult.errors.push(failure(phase));
|
|
10993
11945
|
}
|
|
10994
|
-
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
10998
|
-
|
|
10999
|
-
|
|
11000
|
-
|
|
11001
|
-
|
|
11002
|
-
} catch {
|
|
11003
|
-
if (signal.aborted) {
|
|
11004
|
-
attemptResult.status = timedOut ? "error" : "cancelled";
|
|
11005
|
-
if (timedOut)
|
|
11006
|
-
attemptResult.errors.push({
|
|
11007
|
-
phase,
|
|
11008
|
-
code: "EVAL_TIMEOUT",
|
|
11009
|
-
message: "Case exceeded its configured time limit."
|
|
11946
|
+
if (activeStep && !activeStepReported) {
|
|
11947
|
+
activeStep.status = attemptResult.status;
|
|
11948
|
+
activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
|
|
11949
|
+
await emit({
|
|
11950
|
+
type: "step-completed",
|
|
11951
|
+
caseId: item.id,
|
|
11952
|
+
repetition,
|
|
11953
|
+
step: activeStep
|
|
11010
11954
|
});
|
|
11011
|
-
|
|
11012
|
-
|
|
11013
|
-
|
|
11014
|
-
|
|
11015
|
-
|
|
11016
|
-
|
|
11017
|
-
|
|
11018
|
-
|
|
11019
|
-
|
|
11020
|
-
|
|
11021
|
-
|
|
11022
|
-
|
|
11023
|
-
|
|
11024
|
-
|
|
11025
|
-
|
|
11026
|
-
|
|
11027
|
-
|
|
11028
|
-
|
|
11029
|
-
|
|
11030
|
-
|
|
11031
|
-
|
|
11032
|
-
|
|
11033
|
-
|
|
11034
|
-
|
|
11035
|
-
if (continuation !== void 0) {
|
|
11036
|
-
try {
|
|
11037
|
-
const cancelled = await call(
|
|
11038
|
-
context,
|
|
11039
|
-
{ type: "cancel" },
|
|
11040
|
-
cleanup.signal
|
|
11041
|
-
);
|
|
11042
|
-
if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
|
|
11955
|
+
}
|
|
11956
|
+
} finally {
|
|
11957
|
+
clearTimeout(timer);
|
|
11958
|
+
if (context) {
|
|
11959
|
+
await reportPhase("cleanup");
|
|
11960
|
+
const cleanup = new AbortController();
|
|
11961
|
+
const cleanupTimer = setTimeout(
|
|
11962
|
+
() => cleanup.abort(),
|
|
11963
|
+
defaults.cleanupTimeoutMs
|
|
11964
|
+
);
|
|
11965
|
+
try {
|
|
11966
|
+
if (continuation !== void 0) {
|
|
11967
|
+
try {
|
|
11968
|
+
const cancelled = await call(
|
|
11969
|
+
context,
|
|
11970
|
+
{ type: "cancel" },
|
|
11971
|
+
cleanup.signal
|
|
11972
|
+
);
|
|
11973
|
+
if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
|
|
11974
|
+
attemptResult.errors.push(
|
|
11975
|
+
failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
|
|
11976
|
+
);
|
|
11977
|
+
cleanup.signal.throwIfAborted();
|
|
11978
|
+
} catch {
|
|
11043
11979
|
attemptResult.errors.push(
|
|
11044
11980
|
failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
|
|
11045
11981
|
);
|
|
11982
|
+
}
|
|
11983
|
+
}
|
|
11984
|
+
try {
|
|
11985
|
+
await options.teardown?.({
|
|
11986
|
+
...context,
|
|
11987
|
+
signal: cleanup.signal,
|
|
11988
|
+
result: clone(attemptResult)
|
|
11989
|
+
});
|
|
11046
11990
|
cleanup.signal.throwIfAborted();
|
|
11047
11991
|
} catch {
|
|
11048
|
-
attemptResult.errors.push(
|
|
11049
|
-
failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
|
|
11050
|
-
);
|
|
11992
|
+
attemptResult.errors.push(failure("cleanup"));
|
|
11051
11993
|
}
|
|
11994
|
+
} finally {
|
|
11995
|
+
clearTimeout(cleanupTimer);
|
|
11052
11996
|
}
|
|
11053
|
-
|
|
11054
|
-
|
|
11055
|
-
...context,
|
|
11056
|
-
signal: cleanup.signal,
|
|
11057
|
-
result: clone(attemptResult)
|
|
11058
|
-
});
|
|
11059
|
-
cleanup.signal.throwIfAborted();
|
|
11060
|
-
} catch {
|
|
11061
|
-
attemptResult.errors.push(failure("cleanup"));
|
|
11062
|
-
}
|
|
11063
|
-
} finally {
|
|
11064
|
-
clearTimeout(cleanupTimer);
|
|
11997
|
+
if (attemptResult.errors.some((error) => error.phase === "cleanup"))
|
|
11998
|
+
attemptResult.status = "error";
|
|
11065
11999
|
}
|
|
11066
|
-
|
|
11067
|
-
attemptResult.status = "error";
|
|
12000
|
+
attemptResult.durationMs = Date.now() - started;
|
|
11068
12001
|
}
|
|
11069
|
-
|
|
11070
|
-
|
|
11071
|
-
|
|
11072
|
-
|
|
12002
|
+
acceptingExecutions = false;
|
|
12003
|
+
await associationProgress;
|
|
12004
|
+
await emit({ type: "case-completed", result: attemptResult });
|
|
12005
|
+
return attemptResult;
|
|
12006
|
+
};
|
|
12007
|
+
return (0, import_eval_attribution2.withEvalAttribution)(
|
|
12008
|
+
{
|
|
12009
|
+
attemptId,
|
|
12010
|
+
onExecution: (execution) => {
|
|
12011
|
+
if (!acceptingExecutions || runtimeExecutions.some(
|
|
12012
|
+
(entry) => entry.runId === execution.runId && entry.sessionId === execution.sessionId
|
|
12013
|
+
))
|
|
12014
|
+
return;
|
|
12015
|
+
runtimeExecutions.push(execution);
|
|
12016
|
+
associationProgress = associationProgress.then(
|
|
12017
|
+
() => emit({
|
|
12018
|
+
type: "case-runtime-associated",
|
|
12019
|
+
caseId: item.id,
|
|
12020
|
+
repetition,
|
|
12021
|
+
attemptId,
|
|
12022
|
+
execution
|
|
12023
|
+
})
|
|
12024
|
+
);
|
|
12025
|
+
}
|
|
12026
|
+
},
|
|
12027
|
+
executeAttempt
|
|
12028
|
+
);
|
|
11073
12029
|
};
|
|
11074
|
-
const total = selected.length * repetitions;
|
|
12030
|
+
const total = args.attempt ? 1 : selected.length * repetitions;
|
|
11075
12031
|
if (args.includeActivity)
|
|
11076
12032
|
await emit({
|
|
11077
12033
|
type: "run-started",
|
|
@@ -11085,8 +12041,11 @@ function createEvals(options) {
|
|
|
11085
12041
|
Array.from({ length: Math.min(concurrency, total) }, async () => {
|
|
11086
12042
|
while (cursor < total) {
|
|
11087
12043
|
const index = cursor++;
|
|
11088
|
-
const item = selected[Math.floor(index / repetitions)];
|
|
11089
|
-
completed[index] = await attempt(
|
|
12044
|
+
const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
|
|
12045
|
+
completed[index] = await attempt(
|
|
12046
|
+
item,
|
|
12047
|
+
args.attempt?.repetition ?? index % repetitions + 1
|
|
12048
|
+
);
|
|
11090
12049
|
}
|
|
11091
12050
|
})
|
|
11092
12051
|
);
|
|
@@ -11095,6 +12054,9 @@ function createEvals(options) {
|
|
|
11095
12054
|
result.counts[item.status] = (result.counts[item.status] ?? 0) + 1;
|
|
11096
12055
|
result.status = statusFor(result.counts, result.errors);
|
|
11097
12056
|
result.durationMs = Date.now() - start;
|
|
12057
|
+
evalUsage.end(
|
|
12058
|
+
result.status === "passed" ? "success" : result.status === "failed" ? "quality_failed" : result.status === "error" ? "failed" : result.status
|
|
12059
|
+
);
|
|
11098
12060
|
return result;
|
|
11099
12061
|
}
|
|
11100
12062
|
};
|
|
@@ -11109,6 +12071,7 @@ function defineSuite(suite) {
|
|
|
11109
12071
|
var import_zod10 = require("zod");
|
|
11110
12072
|
|
|
11111
12073
|
// src/evals/studio-contracts.ts
|
|
12074
|
+
var import_browser3 = require("@kortyx/prompts/browser");
|
|
11112
12075
|
var import_zod9 = require("zod");
|
|
11113
12076
|
var StudioEvalTargetsResponseSchema = import_zod9.z.object({
|
|
11114
12077
|
canRun: import_zod9.z.boolean(),
|
|
@@ -11194,7 +12157,9 @@ var StudioEvalDetailSchema = import_zod9.z.object({
|
|
|
11194
12157
|
grading: import_zod9.z.enum(["app", "studio"]).optional(),
|
|
11195
12158
|
judge: EvalJudgeIdentitySchema.optional(),
|
|
11196
12159
|
repetitions: import_zod9.z.number().int().positive(),
|
|
11197
|
-
concurrency: import_zod9.z.number().optional()
|
|
12160
|
+
concurrency: import_zod9.z.number().optional(),
|
|
12161
|
+
promptSnapshot: import_browser3.PromptSnapshotSchema.optional(),
|
|
12162
|
+
promptGroupName: import_zod9.z.string().optional()
|
|
11198
12163
|
}).optional(),
|
|
11199
12164
|
suite: EvalSuiteSchema,
|
|
11200
12165
|
result: EvalRunResultSchema.nullable(),
|
|
@@ -11218,6 +12183,7 @@ var EvaluationMetadataSchema = import_zod10.z.object({
|
|
|
11218
12183
|
var StudioEvaluationStartRequestSchema = import_zod10.z.object({
|
|
11219
12184
|
targetId: import_zod10.z.string().trim().min(1),
|
|
11220
12185
|
name: import_zod10.z.string().trim().min(1).max(200).optional(),
|
|
12186
|
+
promptSelection: StudioEvalStartRequestSchema.shape.promptSelection,
|
|
11221
12187
|
selection: import_zod10.z.enum(["all", "selected"]),
|
|
11222
12188
|
suites: import_zod10.z.array(
|
|
11223
12189
|
import_zod10.z.object({
|
|
@@ -11340,7 +12306,7 @@ function createEvalJudge({
|
|
|
11340
12306
|
"inputIncludesCacheRead",
|
|
11341
12307
|
"inputIncludesCacheWrite"
|
|
11342
12308
|
];
|
|
11343
|
-
const
|
|
12309
|
+
const usage2 = result.usage && Object.fromEntries(
|
|
11344
12310
|
Object.entries(result.usage).filter(
|
|
11345
12311
|
([key, value]) => numericKeys.includes(key) ? typeof value === "number" && Number.isFinite(value) && value >= 0 : booleanKeys.includes(key) && typeof value === "boolean"
|
|
11346
12312
|
)
|
|
@@ -11357,7 +12323,7 @@ function createEvalJudge({
|
|
|
11357
12323
|
provider: model.provider.id,
|
|
11358
12324
|
model: model.modelId,
|
|
11359
12325
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11360
|
-
...
|
|
12326
|
+
...usage2 ? { usage: usage2 } : {},
|
|
11361
12327
|
...pricingContext && Object.keys(pricingContext).length ? { pricingContext } : {},
|
|
11362
12328
|
...model.provider.id === "openrouter" && typeof cost === "number" && Number.isFinite(cost) && cost >= 0 && Number.isSafeInteger(Math.round(cost * 1e6)) ? {
|
|
11363
12329
|
pricing: {
|
|
@@ -11379,6 +12345,7 @@ function createEvalJudge({
|
|
|
11379
12345
|
|
|
11380
12346
|
// src/evals/route-handler.ts
|
|
11381
12347
|
var import_node_crypto6 = require("crypto");
|
|
12348
|
+
var import_prompts2 = require("@kortyx/prompts");
|
|
11382
12349
|
function createEvalRouteHandler({
|
|
11383
12350
|
evals,
|
|
11384
12351
|
serviceKey,
|
|
@@ -11396,6 +12363,7 @@ function createEvalRouteHandler({
|
|
|
11396
12363
|
const digest = (value) => (0, import_node_crypto6.createHash)("sha256").update(value).digest();
|
|
11397
12364
|
const expected = digest(`Bearer ${serviceKey}`);
|
|
11398
12365
|
let active = 0;
|
|
12366
|
+
const evaluations = /* @__PURE__ */ new Map();
|
|
11399
12367
|
return async (request) => {
|
|
11400
12368
|
if (!(0, import_node_crypto6.timingSafeEqual)(
|
|
11401
12369
|
expected,
|
|
@@ -11417,7 +12385,7 @@ function createEvalRouteHandler({
|
|
|
11417
12385
|
let body;
|
|
11418
12386
|
try {
|
|
11419
12387
|
const text = await request.text();
|
|
11420
|
-
if (text.length >
|
|
12388
|
+
if (text.length > 1048576)
|
|
11421
12389
|
return Response.json({ error: "Request too large." }, { status: 413 });
|
|
11422
12390
|
body = EvalRemoteRunRequestSchema.parse(JSON.parse(text));
|
|
11423
12391
|
} catch {
|
|
@@ -11427,6 +12395,28 @@ function createEvalRouteHandler({
|
|
|
11427
12395
|
);
|
|
11428
12396
|
}
|
|
11429
12397
|
const suite = manifest.suites.find((item) => item.id === body.suiteId);
|
|
12398
|
+
if (body.promptSnapshot) {
|
|
12399
|
+
try {
|
|
12400
|
+
const snapshot = await (0, import_prompts2.verifyPromptSnapshot)(body.promptSnapshot);
|
|
12401
|
+
if (snapshot.source !== "eval" || !manifest.promptContracts?.length)
|
|
12402
|
+
throw new Error("Unsupported prompt evaluation.");
|
|
12403
|
+
for (const contract of manifest.promptContracts) {
|
|
12404
|
+
const version = snapshot.versions[contract.id];
|
|
12405
|
+
if (!version || version.content.format !== contract.format)
|
|
12406
|
+
throw new Error("Prompt contract mismatch.");
|
|
12407
|
+
(0, import_prompts2.validatePromptValue)(
|
|
12408
|
+
contract.configSchema,
|
|
12409
|
+
version.content.config,
|
|
12410
|
+
"Application configuration"
|
|
12411
|
+
);
|
|
12412
|
+
}
|
|
12413
|
+
} catch {
|
|
12414
|
+
return Response.json(
|
|
12415
|
+
{ error: "Prompt snapshot does not match the application contract." },
|
|
12416
|
+
{ status: 409 }
|
|
12417
|
+
);
|
|
12418
|
+
}
|
|
12419
|
+
}
|
|
11430
12420
|
if (!suite || body.caseIds?.some((id3) => !suite.cases.some((item) => item.id === id3)) || body.caseIds && new Set(body.caseIds).size !== body.caseIds.length)
|
|
11431
12421
|
return Response.json(
|
|
11432
12422
|
{ error: "Unknown suite or case selection." },
|
|
@@ -11452,14 +12442,32 @@ function createEvalRouteHandler({
|
|
|
11452
12442
|
{ error: "App judge changed. Refresh before running." },
|
|
11453
12443
|
{ status: 409 }
|
|
11454
12444
|
);
|
|
11455
|
-
if (
|
|
12445
|
+
if (body.attempt && (!manifest.attemptScheduling || !suite.cases.some((item) => item.id === body.attempt?.caseId) || body.caseIds && !body.caseIds.includes(body.attempt.caseId) || body.attempt.repetition > body.repetitions))
|
|
12446
|
+
return Response.json(
|
|
12447
|
+
{ error: "Unsupported or invalid scheduled attempt." },
|
|
12448
|
+
{ status: 409 }
|
|
12449
|
+
);
|
|
12450
|
+
const group = body.attempt && evaluations.get(body.attempt.evaluationId);
|
|
12451
|
+
if (group && (group.concurrency !== body.concurrency || group.active >= group.concurrency))
|
|
12452
|
+
return Response.json(
|
|
12453
|
+
{ error: "Evaluation concurrency exhausted or changed." },
|
|
12454
|
+
{ status: 429 }
|
|
12455
|
+
);
|
|
12456
|
+
if (!group && active >= maxActiveRuns)
|
|
11456
12457
|
return Response.json(
|
|
11457
12458
|
{ error: "Eval executor is busy." },
|
|
11458
12459
|
{ status: 429 }
|
|
11459
12460
|
);
|
|
11460
12461
|
if (request.signal.aborted)
|
|
11461
12462
|
return Response.json({ error: "Request cancelled." }, { status: 409 });
|
|
11462
|
-
|
|
12463
|
+
if (body.attempt) {
|
|
12464
|
+
const state = group ?? { active: 0, concurrency: body.concurrency };
|
|
12465
|
+
if (!group) {
|
|
12466
|
+
active++;
|
|
12467
|
+
evaluations.set(body.attempt.evaluationId, state);
|
|
12468
|
+
}
|
|
12469
|
+
state.active++;
|
|
12470
|
+
} else active++;
|
|
11463
12471
|
const controller = new AbortController();
|
|
11464
12472
|
const requestSignal = request.signal;
|
|
11465
12473
|
const abortRequest = () => controller.abort();
|
|
@@ -11475,6 +12483,8 @@ function createEvalRouteHandler({
|
|
|
11475
12483
|
};
|
|
11476
12484
|
void evals.run({
|
|
11477
12485
|
suiteId: body.suiteId,
|
|
12486
|
+
...body.promptSnapshot ? { promptSnapshot: body.promptSnapshot } : {},
|
|
12487
|
+
...body.attempt ? { attempt: body.attempt } : {},
|
|
11478
12488
|
...body.grading ? { grading: body.grading } : {},
|
|
11479
12489
|
...body.judge ? { judgeIdentity: body.judge } : {},
|
|
11480
12490
|
repetitions: body.repetitions,
|
|
@@ -11487,7 +12497,14 @@ function createEvalRouteHandler({
|
|
|
11487
12497
|
() => send({ type: "error", message: "Eval runner failed." })
|
|
11488
12498
|
).finally(() => {
|
|
11489
12499
|
requestSignal.removeEventListener("abort", abortRequest);
|
|
11490
|
-
|
|
12500
|
+
if (body.attempt) {
|
|
12501
|
+
const state = evaluations.get(body.attempt.evaluationId);
|
|
12502
|
+
state.active--;
|
|
12503
|
+
if (!state.active) {
|
|
12504
|
+
evaluations.delete(body.attempt.evaluationId);
|
|
12505
|
+
active--;
|
|
12506
|
+
}
|
|
12507
|
+
} else active--;
|
|
11491
12508
|
if (!controller.signal.aborted) output.close();
|
|
11492
12509
|
});
|
|
11493
12510
|
},
|
|
@@ -11607,7 +12624,7 @@ async function createStudioEvalJudge(options) {
|
|
|
11607
12624
|
throw new Error(
|
|
11608
12625
|
"Studio judge identity changed. Refresh before running."
|
|
11609
12626
|
);
|
|
11610
|
-
for (const
|
|
12627
|
+
for (const usage2 of response.usage ?? []) input.onUsage?.(usage2);
|
|
11611
12628
|
return response.verdict;
|
|
11612
12629
|
}
|
|
11613
12630
|
};
|