@assemble-dev/evals 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.cts +74 -0
- package/dist/index.d.ts +74 -0
- package/package.json +5 -5
package/dist/index.d.cts
CHANGED
|
@@ -412,6 +412,7 @@ declare const EvalRunFileSchema: z.ZodObject<{
|
|
|
412
412
|
agentName: z.ZodString;
|
|
413
413
|
output: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
414
414
|
latencyMs: z.ZodNumber;
|
|
415
|
+
modelName: z.ZodOptional<z.ZodString>;
|
|
415
416
|
}, z.core.$strip>, z.ZodObject<{
|
|
416
417
|
id: z.ZodString;
|
|
417
418
|
runId: z.ZodString;
|
|
@@ -701,6 +702,79 @@ declare const EvalRunFileSchema: z.ZodObject<{
|
|
|
701
702
|
type: z.ZodLiteral<"replay.completed">;
|
|
702
703
|
sourceRunId: z.ZodString;
|
|
703
704
|
latencyMs: z.ZodNumber;
|
|
705
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
706
|
+
id: z.ZodString;
|
|
707
|
+
runId: z.ZodString;
|
|
708
|
+
workflowName: z.ZodString;
|
|
709
|
+
timestamp: z.ZodISODateTime;
|
|
710
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
711
|
+
redacted: z.ZodBoolean;
|
|
712
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
713
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
714
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
715
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
716
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
717
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
718
|
+
}, z.core.$strip>>;
|
|
719
|
+
type: z.ZodLiteral<"batch.submitted">;
|
|
720
|
+
batchId: z.ZodString;
|
|
721
|
+
requestCount: z.ZodNumber;
|
|
722
|
+
models: z.ZodArray<z.ZodString>;
|
|
723
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
724
|
+
id: z.ZodString;
|
|
725
|
+
runId: z.ZodString;
|
|
726
|
+
workflowName: z.ZodString;
|
|
727
|
+
timestamp: z.ZodISODateTime;
|
|
728
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
729
|
+
redacted: z.ZodBoolean;
|
|
730
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
731
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
732
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
733
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
734
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
735
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
736
|
+
}, z.core.$strip>>;
|
|
737
|
+
type: z.ZodLiteral<"batch.completed">;
|
|
738
|
+
batchId: z.ZodString;
|
|
739
|
+
requestCounts: z.ZodObject<{
|
|
740
|
+
processing: z.ZodNumber;
|
|
741
|
+
succeeded: z.ZodNumber;
|
|
742
|
+
errored: z.ZodNumber;
|
|
743
|
+
canceled: z.ZodNumber;
|
|
744
|
+
expired: z.ZodNumber;
|
|
745
|
+
}, z.core.$strip>;
|
|
746
|
+
latencyMs: z.ZodNumber;
|
|
747
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
748
|
+
id: z.ZodString;
|
|
749
|
+
runId: z.ZodString;
|
|
750
|
+
workflowName: z.ZodString;
|
|
751
|
+
timestamp: z.ZodISODateTime;
|
|
752
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
753
|
+
redacted: z.ZodBoolean;
|
|
754
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
755
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
756
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
757
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
758
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
759
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
760
|
+
}, z.core.$strip>>;
|
|
761
|
+
type: z.ZodLiteral<"batch.result">;
|
|
762
|
+
batchId: z.ZodString;
|
|
763
|
+
customId: z.ZodString;
|
|
764
|
+
outcomeType: z.ZodEnum<{
|
|
765
|
+
canceled: "canceled";
|
|
766
|
+
succeeded: "succeeded";
|
|
767
|
+
errored: "errored";
|
|
768
|
+
expired: "expired";
|
|
769
|
+
}>;
|
|
770
|
+
tokenUsage: z.ZodOptional<z.ZodObject<{
|
|
771
|
+
inputTokens: z.ZodNumber;
|
|
772
|
+
outputTokens: z.ZodNumber;
|
|
773
|
+
totalTokens: z.ZodNumber;
|
|
774
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
775
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
776
|
+
}, z.core.$strip>>;
|
|
777
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
704
778
|
}, z.core.$strip>], "type">>;
|
|
705
779
|
}, z.core.$strip>;
|
|
706
780
|
type EvalCaseRunResult = z.infer<typeof EvalCaseResultSchema>;
|
package/dist/index.d.ts
CHANGED
|
@@ -412,6 +412,7 @@ declare const EvalRunFileSchema: z.ZodObject<{
|
|
|
412
412
|
agentName: z.ZodString;
|
|
413
413
|
output: z.ZodType<_assemble_dev_shared_types.JsonValue, unknown, z.core.$ZodTypeInternals<_assemble_dev_shared_types.JsonValue, unknown>>;
|
|
414
414
|
latencyMs: z.ZodNumber;
|
|
415
|
+
modelName: z.ZodOptional<z.ZodString>;
|
|
415
416
|
}, z.core.$strip>, z.ZodObject<{
|
|
416
417
|
id: z.ZodString;
|
|
417
418
|
runId: z.ZodString;
|
|
@@ -701,6 +702,79 @@ declare const EvalRunFileSchema: z.ZodObject<{
|
|
|
701
702
|
type: z.ZodLiteral<"replay.completed">;
|
|
702
703
|
sourceRunId: z.ZodString;
|
|
703
704
|
latencyMs: z.ZodNumber;
|
|
705
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
706
|
+
id: z.ZodString;
|
|
707
|
+
runId: z.ZodString;
|
|
708
|
+
workflowName: z.ZodString;
|
|
709
|
+
timestamp: z.ZodISODateTime;
|
|
710
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
711
|
+
redacted: z.ZodBoolean;
|
|
712
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
713
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
714
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
715
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
716
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
717
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
718
|
+
}, z.core.$strip>>;
|
|
719
|
+
type: z.ZodLiteral<"batch.submitted">;
|
|
720
|
+
batchId: z.ZodString;
|
|
721
|
+
requestCount: z.ZodNumber;
|
|
722
|
+
models: z.ZodArray<z.ZodString>;
|
|
723
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
724
|
+
id: z.ZodString;
|
|
725
|
+
runId: z.ZodString;
|
|
726
|
+
workflowName: z.ZodString;
|
|
727
|
+
timestamp: z.ZodISODateTime;
|
|
728
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
729
|
+
redacted: z.ZodBoolean;
|
|
730
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
731
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
732
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
733
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
734
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
735
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
736
|
+
}, z.core.$strip>>;
|
|
737
|
+
type: z.ZodLiteral<"batch.completed">;
|
|
738
|
+
batchId: z.ZodString;
|
|
739
|
+
requestCounts: z.ZodObject<{
|
|
740
|
+
processing: z.ZodNumber;
|
|
741
|
+
succeeded: z.ZodNumber;
|
|
742
|
+
errored: z.ZodNumber;
|
|
743
|
+
canceled: z.ZodNumber;
|
|
744
|
+
expired: z.ZodNumber;
|
|
745
|
+
}, z.core.$strip>;
|
|
746
|
+
latencyMs: z.ZodNumber;
|
|
747
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
748
|
+
id: z.ZodString;
|
|
749
|
+
runId: z.ZodString;
|
|
750
|
+
workflowName: z.ZodString;
|
|
751
|
+
timestamp: z.ZodISODateTime;
|
|
752
|
+
parentEventId: z.ZodNullable<z.ZodString>;
|
|
753
|
+
redacted: z.ZodBoolean;
|
|
754
|
+
metadata: z.ZodOptional<z.ZodObject<{
|
|
755
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
756
|
+
userId: z.ZodOptional<z.ZodString>;
|
|
757
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
758
|
+
parentTraceId: z.ZodOptional<z.ZodString>;
|
|
759
|
+
parentSpanId: z.ZodOptional<z.ZodString>;
|
|
760
|
+
}, z.core.$strip>>;
|
|
761
|
+
type: z.ZodLiteral<"batch.result">;
|
|
762
|
+
batchId: z.ZodString;
|
|
763
|
+
customId: z.ZodString;
|
|
764
|
+
outcomeType: z.ZodEnum<{
|
|
765
|
+
canceled: "canceled";
|
|
766
|
+
succeeded: "succeeded";
|
|
767
|
+
errored: "errored";
|
|
768
|
+
expired: "expired";
|
|
769
|
+
}>;
|
|
770
|
+
tokenUsage: z.ZodOptional<z.ZodObject<{
|
|
771
|
+
inputTokens: z.ZodNumber;
|
|
772
|
+
outputTokens: z.ZodNumber;
|
|
773
|
+
totalTokens: z.ZodNumber;
|
|
774
|
+
cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
775
|
+
cacheReadInputTokens: z.ZodOptional<z.ZodNumber>;
|
|
776
|
+
}, z.core.$strip>>;
|
|
777
|
+
costUsd: z.ZodNullable<z.ZodNumber>;
|
|
704
778
|
}, z.core.$strip>], "type">>;
|
|
705
779
|
}, z.core.$strip>;
|
|
706
780
|
type EvalCaseRunResult = z.infer<typeof EvalCaseResultSchema>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@assemble-dev/evals",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"description": "Evaluation harness for Assemble agents and workflows",
|
|
6
6
|
"repository": {
|
|
@@ -27,10 +27,10 @@
|
|
|
27
27
|
],
|
|
28
28
|
"dependencies": {
|
|
29
29
|
"zod": "^4.1.13",
|
|
30
|
-
"@assemble-dev/providers": "0.
|
|
31
|
-
"@assemble-dev/sdk": "0.
|
|
32
|
-
"@assemble-dev/shared-types": "0.
|
|
33
|
-
"@assemble-dev/shared-utils": "0.
|
|
30
|
+
"@assemble-dev/providers": "0.2.0",
|
|
31
|
+
"@assemble-dev/sdk": "0.2.0",
|
|
32
|
+
"@assemble-dev/shared-types": "0.2.0",
|
|
33
|
+
"@assemble-dev/shared-utils": "0.2.0"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@types/node": "^20.19.0",
|