@wix/evalforge-types 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/build/index.js +1047 -0
  2. package/build/index.js.map +7 -0
  3. package/build/index.mjs +928 -0
  4. package/build/index.mjs.map +7 -0
  5. package/build/types/common/base-entity.d.ts +26 -0
  6. package/build/types/common/index.d.ts +3 -0
  7. package/build/types/common/mcp.d.ts +17 -0
  8. package/build/types/common/models.d.ts +55 -0
  9. package/build/types/evaluation/eval-result.d.ts +239 -0
  10. package/build/types/evaluation/eval-run.d.ts +581 -0
  11. package/build/types/evaluation/index.d.ts +4 -0
  12. package/build/types/evaluation/live-trace.d.ts +47 -0
  13. package/build/types/evaluation/metrics.d.ts +157 -0
  14. package/build/types/improvement/improvement.d.ts +140 -0
  15. package/build/types/improvement/index.d.ts +1 -0
  16. package/build/types/index.d.ts +24 -0
  17. package/build/types/project/index.d.ts +1 -0
  18. package/build/types/project/project.d.ts +41 -0
  19. package/build/types/scenario/environment.d.ts +58 -0
  20. package/build/types/scenario/index.d.ts +2 -0
  21. package/build/types/scenario/test-scenario.d.ts +50 -0
  22. package/build/types/suite/index.d.ts +1 -0
  23. package/build/types/suite/test-suite.d.ts +37 -0
  24. package/build/types/target/agent.d.ts +53 -0
  25. package/build/types/target/index.d.ts +4 -0
  26. package/build/types/target/skill.d.ts +78 -0
  27. package/build/types/target/skills-group.d.ts +37 -0
  28. package/build/types/target/target.d.ts +17 -0
  29. package/build/types/template/index.d.ts +1 -0
  30. package/build/types/template/template.d.ts +38 -0
  31. package/build/types/test/base.d.ts +43 -0
  32. package/build/types/test/build-check.d.ts +29 -0
  33. package/build/types/test/command-execution.d.ts +31 -0
  34. package/build/types/test/file-content.d.ts +52 -0
  35. package/build/types/test/file-presence.d.ts +24 -0
  36. package/build/types/test/index.d.ts +124 -0
  37. package/build/types/test/llm.d.ts +36 -0
  38. package/build/types/test/playwright-nl.d.ts +28 -0
  39. package/build/types/test/site-config.d.ts +32 -0
  40. package/build/types/test/tool.d.ts +26 -0
  41. package/build/types/test/vitest.d.ts +30 -0
  42. package/package.json +50 -0
@@ -0,0 +1,37 @@
1
+ import { z } from 'zod';
2
+ /**
3
+ * SkillsGroup schema - a collection of skills (replaces TargetGroup for the new model).
4
+ *
5
+ * Eval runs can be scoped to a skills group to run against all skills in the group.
6
+ */
7
+ export declare const SkillsGroupSchema: z.ZodObject<{
8
+ id: z.ZodString;
9
+ name: z.ZodString;
10
+ description: z.ZodString;
11
+ createdAt: z.ZodString;
12
+ updatedAt: z.ZodString;
13
+ deleted: z.ZodOptional<z.ZodBoolean>;
14
+ projectId: z.ZodString;
15
+ skillIds: z.ZodArray<z.ZodString>;
16
+ }, z.core.$strip>;
17
+ export type SkillsGroup = z.infer<typeof SkillsGroupSchema>;
18
+ /**
19
+ * Input schema for creating a new SkillsGroup.
20
+ */
21
+ export declare const CreateSkillsGroupInputSchema: z.ZodObject<{
22
+ name: z.ZodString;
23
+ description: z.ZodString;
24
+ projectId: z.ZodString;
25
+ skillIds: z.ZodArray<z.ZodString>;
26
+ }, z.core.$strip>;
27
+ export type CreateSkillsGroupInput = z.infer<typeof CreateSkillsGroupInputSchema>;
28
+ /**
29
+ * Input schema for updating a SkillsGroup.
30
+ */
31
+ export declare const UpdateSkillsGroupInputSchema: z.ZodObject<{
32
+ name: z.ZodOptional<z.ZodString>;
33
+ description: z.ZodOptional<z.ZodString>;
34
+ projectId: z.ZodOptional<z.ZodString>;
35
+ skillIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
36
+ }, z.core.$strip>;
37
+ export type UpdateSkillsGroupInput = z.infer<typeof UpdateSkillsGroupInputSchema>;
@@ -0,0 +1,17 @@
1
+ import { z } from 'zod';
2
+ /**
3
+ * Target schema - base for all testable entities.
4
+ *
5
+ * All testable entities (Agent, Skill) extend this schema.
6
+ * This creates a unified type hierarchy for what can be evaluated.
7
+ */
8
+ export declare const TargetSchema: z.ZodObject<{
9
+ id: z.ZodString;
10
+ name: z.ZodString;
11
+ description: z.ZodString;
12
+ createdAt: z.ZodString;
13
+ updatedAt: z.ZodString;
14
+ deleted: z.ZodOptional<z.ZodBoolean>;
15
+ projectId: z.ZodString;
16
+ }, z.core.$strip>;
17
+ export type Target = z.infer<typeof TargetSchema>;
@@ -0,0 +1 @@
1
+ export * from './template.js';
@@ -0,0 +1,38 @@
1
+ import { z } from 'zod';
2
+ /**
3
+ * Template schema - a project template that can be used for test environments.
4
+ *
5
+ * Templates are tenant-based entities scoped to a project.
6
+ * They define how to set up a project environment for testing.
7
+ */
8
+ export declare const TemplateSchema: z.ZodObject<{
9
+ id: z.ZodString;
10
+ name: z.ZodString;
11
+ description: z.ZodString;
12
+ createdAt: z.ZodString;
13
+ updatedAt: z.ZodString;
14
+ deleted: z.ZodOptional<z.ZodBoolean>;
15
+ projectId: z.ZodString;
16
+ downloadUrl: z.ZodURL;
17
+ }, z.core.$strip>;
18
+ export type Template = z.infer<typeof TemplateSchema>;
19
+ /**
20
+ * Input schema for creating a new Template.
21
+ */
22
+ export declare const CreateTemplateInputSchema: z.ZodObject<{
23
+ name: z.ZodString;
24
+ description: z.ZodString;
25
+ projectId: z.ZodString;
26
+ downloadUrl: z.ZodURL;
27
+ }, z.core.$strip>;
28
+ export type CreateTemplateInput = z.infer<typeof CreateTemplateInputSchema>;
29
+ /**
30
+ * Input schema for updating a Template.
31
+ */
32
+ export declare const UpdateTemplateInputSchema: z.ZodObject<{
33
+ name: z.ZodOptional<z.ZodString>;
34
+ description: z.ZodOptional<z.ZodString>;
35
+ projectId: z.ZodOptional<z.ZodString>;
36
+ downloadUrl: z.ZodOptional<z.ZodURL>;
37
+ }, z.core.$strip>;
38
+ export type UpdateTemplateInput = z.infer<typeof UpdateTemplateInputSchema>;
@@ -0,0 +1,43 @@
1
+ import { z } from 'zod';
2
+ /**
3
+ * Test types - unified from old and new systems.
4
+ */
5
+ export declare enum TestType {
6
+ LLM = "LLM",
7
+ TOOL = "TOOL",
8
+ SITE_CONFIG = "SITE_CONFIG",
9
+ COMMAND_EXECUTION = "COMMAND_EXECUTION",
10
+ FILE_PRESENCE = "FILE_PRESENCE",
11
+ FILE_CONTENT = "FILE_CONTENT",
12
+ BUILD_CHECK = "BUILD_CHECK",
13
+ VITEST = "VITEST",
14
+ PLAYWRIGHT_NL = "PLAYWRIGHT_NL"
15
+ }
16
+ export declare const TestTypeSchema: z.ZodEnum<typeof TestType>;
17
+ /**
18
+ * Test importance levels.
19
+ */
20
+ export declare enum TestImportance {
21
+ LOW = "low",
22
+ MEDIUM = "medium",
23
+ HIGH = "high",
24
+ CRITICAL = "critical"
25
+ }
26
+ export declare const TestImportanceSchema: z.ZodEnum<typeof TestImportance>;
27
+ /**
28
+ * Base test schema - common fields for all test types.
29
+ */
30
+ export declare const BaseTestSchema: z.ZodObject<{
31
+ id: z.ZodString;
32
+ type: z.ZodEnum<typeof TestType>;
33
+ name: z.ZodString;
34
+ description: z.ZodOptional<z.ZodString>;
35
+ importance: z.ZodOptional<z.ZodEnum<typeof TestImportance>>;
36
+ }, z.core.$strip>;
37
+ export type BaseTest = z.infer<typeof BaseTestSchema>;
38
+ /**
39
+ * Base test result interface.
40
+ */
41
+ export interface BaseTestResult {
42
+ type: TestType;
43
+ }
@@ -0,0 +1,29 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Build Check Test schema - tests that verify build success.
5
+ */
6
+ export declare const BuildCheckTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.BUILD_CHECK>;
12
+ command: z.ZodString;
13
+ expectSuccess: z.ZodBoolean;
14
+ allowedWarnings: z.ZodOptional<z.ZodNumber>;
15
+ timeout: z.ZodOptional<z.ZodNumber>;
16
+ }, z.core.$strip>;
17
+ export type BuildCheckTest = z.infer<typeof BuildCheckTestSchema>;
18
+ /**
19
+ * Build Check Test result.
20
+ */
21
+ export interface BuildCheckTestResult extends BaseTestResult {
22
+ type: TestType.BUILD_CHECK;
23
+ result: boolean;
24
+ exitCode: number;
25
+ stdout: string;
26
+ stderr: string;
27
+ warningCount: number;
28
+ duration: number;
29
+ }
@@ -0,0 +1,31 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Allowed commands for command execution tests.
5
+ */
6
+ export declare const AllowedCommands: readonly ["yarn install --no-immutable && yarn build", "npm run build", "yarn typecheck"];
7
+ /**
8
+ * Command Execution Test schema - tests that verify command execution.
9
+ */
10
+ export declare const CommandExecutionTestSchema: z.ZodObject<{
11
+ id: z.ZodString;
12
+ name: z.ZodString;
13
+ description: z.ZodOptional<z.ZodString>;
14
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
15
+ type: z.ZodLiteral<TestType.COMMAND_EXECUTION>;
16
+ command: z.ZodString;
17
+ expectedExitCode: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
18
+ }, z.core.$strip>;
19
+ export type CommandExecutionTest = z.infer<typeof CommandExecutionTestSchema>;
20
+ /**
21
+ * Command Execution Test result.
22
+ */
23
+ export interface CommandExecutionTestResult extends BaseTestResult {
24
+ type: TestType.COMMAND_EXECUTION;
25
+ stdout: string;
26
+ stderr: string;
27
+ exitCode: number | null;
28
+ expectedExitCode: number;
29
+ result: boolean;
30
+ error?: string;
31
+ }
@@ -0,0 +1,52 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * File content checks schema.
5
+ */
6
+ export declare const FileContentCheckSchema: z.ZodObject<{
7
+ contains: z.ZodOptional<z.ZodArray<z.ZodString>>;
8
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
9
+ matches: z.ZodOptional<z.ZodString>;
10
+ jsonPath: z.ZodOptional<z.ZodArray<z.ZodObject<{
11
+ path: z.ZodString;
12
+ value: z.ZodUnknown;
13
+ }, z.core.$strip>>>;
14
+ added: z.ZodOptional<z.ZodArray<z.ZodString>>;
15
+ removed: z.ZodOptional<z.ZodArray<z.ZodString>>;
16
+ }, z.core.$strip>;
17
+ export type FileContentCheck = z.infer<typeof FileContentCheckSchema>;
18
+ /**
19
+ * File Content Test schema - tests that verify file content.
20
+ *
21
+ * This also covers the FILE_MODIFICATION use case from the old system
22
+ * by supporting added/removed line checks.
23
+ */
24
+ export declare const FileContentTestSchema: z.ZodObject<{
25
+ id: z.ZodString;
26
+ name: z.ZodString;
27
+ description: z.ZodOptional<z.ZodString>;
28
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
29
+ type: z.ZodLiteral<TestType.FILE_CONTENT>;
30
+ path: z.ZodString;
31
+ checks: z.ZodObject<{
32
+ contains: z.ZodOptional<z.ZodArray<z.ZodString>>;
33
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
34
+ matches: z.ZodOptional<z.ZodString>;
35
+ jsonPath: z.ZodOptional<z.ZodArray<z.ZodObject<{
36
+ path: z.ZodString;
37
+ value: z.ZodUnknown;
38
+ }, z.core.$strip>>>;
39
+ added: z.ZodOptional<z.ZodArray<z.ZodString>>;
40
+ removed: z.ZodOptional<z.ZodArray<z.ZodString>>;
41
+ }, z.core.$strip>;
42
+ }, z.core.$strip>;
43
+ export type FileContentTest = z.infer<typeof FileContentTestSchema>;
44
+ /**
45
+ * File Content Test result.
46
+ */
47
+ export interface FileContentTestResult extends BaseTestResult {
48
+ type: TestType.FILE_CONTENT;
49
+ result: boolean;
50
+ diff?: string;
51
+ failedChecks?: string[];
52
+ }
@@ -0,0 +1,24 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * File Presence Test schema - tests that verify file existence.
5
+ */
6
+ export declare const FilePresenceTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.FILE_PRESENCE>;
12
+ paths: z.ZodArray<z.ZodString>;
13
+ shouldExist: z.ZodBoolean;
14
+ }, z.core.$strip>;
15
+ export type FilePresenceTest = z.infer<typeof FilePresenceTestSchema>;
16
+ /**
17
+ * File Presence Test result.
18
+ */
19
+ export interface FilePresenceTestResult extends BaseTestResult {
20
+ type: TestType.FILE_PRESENCE;
21
+ result: boolean;
22
+ existingPaths: string[];
23
+ missingPaths: string[];
24
+ }
@@ -0,0 +1,124 @@
1
+ import { z } from 'zod';
2
+ export * from './base.js';
3
+ export * from './llm.js';
4
+ export * from './tool.js';
5
+ export * from './site-config.js';
6
+ export * from './command-execution.js';
7
+ export * from './file-presence.js';
8
+ export * from './file-content.js';
9
+ export * from './build-check.js';
10
+ export * from './vitest.js';
11
+ export * from './playwright-nl.js';
12
+ import { LLMTestResult } from './llm.js';
13
+ import { ToolTestResult } from './tool.js';
14
+ import { SiteConfigTestResult } from './site-config.js';
15
+ import { CommandExecutionTestResult } from './command-execution.js';
16
+ import { FilePresenceTestResult } from './file-presence.js';
17
+ import { FileContentTestResult } from './file-content.js';
18
+ import { BuildCheckTestResult } from './build-check.js';
19
+ import { VitestTestResult } from './vitest.js';
20
+ import { PlaywrightNLTestResult } from './playwright-nl.js';
21
+ /**
22
+ * Unified Test schema - discriminated union of all 9 test types.
23
+ */
24
+ export declare const TestSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
25
+ id: z.ZodString;
26
+ name: z.ZodString;
27
+ description: z.ZodOptional<z.ZodString>;
28
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
29
+ type: z.ZodLiteral<import("./base.js").TestType.LLM>;
30
+ maxSteps: z.ZodNumber;
31
+ prompt: z.ZodString;
32
+ evaluatorId: z.ZodString;
33
+ }, z.core.$strip>, z.ZodObject<{
34
+ id: z.ZodString;
35
+ name: z.ZodString;
36
+ description: z.ZodOptional<z.ZodString>;
37
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
38
+ type: z.ZodLiteral<import("./base.js").TestType.TOOL>;
39
+ toolName: z.ZodString;
40
+ args: z.ZodRecord<z.ZodString, z.ZodAny>;
41
+ resultsContent: z.ZodString;
42
+ }, z.core.$strip>, z.ZodObject<{
43
+ id: z.ZodString;
44
+ name: z.ZodString;
45
+ description: z.ZodOptional<z.ZodString>;
46
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
47
+ type: z.ZodLiteral<import("./base.js").TestType.SITE_CONFIG>;
48
+ url: z.ZodString;
49
+ method: z.ZodEnum<{
50
+ GET: "GET";
51
+ POST: "POST";
52
+ }>;
53
+ body: z.ZodOptional<z.ZodString>;
54
+ expectedStatusCode: z.ZodNumber;
55
+ expectedResponse: z.ZodOptional<z.ZodString>;
56
+ expectedResponseJMESPath: z.ZodOptional<z.ZodString>;
57
+ }, z.core.$strip>, z.ZodObject<{
58
+ id: z.ZodString;
59
+ name: z.ZodString;
60
+ description: z.ZodOptional<z.ZodString>;
61
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
62
+ type: z.ZodLiteral<import("./base.js").TestType.COMMAND_EXECUTION>;
63
+ command: z.ZodString;
64
+ expectedExitCode: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
65
+ }, z.core.$strip>, z.ZodObject<{
66
+ id: z.ZodString;
67
+ name: z.ZodString;
68
+ description: z.ZodOptional<z.ZodString>;
69
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
70
+ type: z.ZodLiteral<import("./base.js").TestType.FILE_PRESENCE>;
71
+ paths: z.ZodArray<z.ZodString>;
72
+ shouldExist: z.ZodBoolean;
73
+ }, z.core.$strip>, z.ZodObject<{
74
+ id: z.ZodString;
75
+ name: z.ZodString;
76
+ description: z.ZodOptional<z.ZodString>;
77
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
78
+ type: z.ZodLiteral<import("./base.js").TestType.FILE_CONTENT>;
79
+ path: z.ZodString;
80
+ checks: z.ZodObject<{
81
+ contains: z.ZodOptional<z.ZodArray<z.ZodString>>;
82
+ notContains: z.ZodOptional<z.ZodArray<z.ZodString>>;
83
+ matches: z.ZodOptional<z.ZodString>;
84
+ jsonPath: z.ZodOptional<z.ZodArray<z.ZodObject<{
85
+ path: z.ZodString;
86
+ value: z.ZodUnknown;
87
+ }, z.core.$strip>>>;
88
+ added: z.ZodOptional<z.ZodArray<z.ZodString>>;
89
+ removed: z.ZodOptional<z.ZodArray<z.ZodString>>;
90
+ }, z.core.$strip>;
91
+ }, z.core.$strip>, z.ZodObject<{
92
+ id: z.ZodString;
93
+ name: z.ZodString;
94
+ description: z.ZodOptional<z.ZodString>;
95
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
96
+ type: z.ZodLiteral<import("./base.js").TestType.BUILD_CHECK>;
97
+ command: z.ZodString;
98
+ expectSuccess: z.ZodBoolean;
99
+ allowedWarnings: z.ZodOptional<z.ZodNumber>;
100
+ timeout: z.ZodOptional<z.ZodNumber>;
101
+ }, z.core.$strip>, z.ZodObject<{
102
+ id: z.ZodString;
103
+ name: z.ZodString;
104
+ description: z.ZodOptional<z.ZodString>;
105
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
106
+ type: z.ZodLiteral<import("./base.js").TestType.VITEST>;
107
+ testFile: z.ZodString;
108
+ testFileName: z.ZodString;
109
+ minPassRate: z.ZodNumber;
110
+ }, z.core.$strip>, z.ZodObject<{
111
+ id: z.ZodString;
112
+ name: z.ZodString;
113
+ description: z.ZodOptional<z.ZodString>;
114
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
115
+ type: z.ZodLiteral<import("./base.js").TestType.PLAYWRIGHT_NL>;
116
+ steps: z.ZodArray<z.ZodString>;
117
+ expectedOutcome: z.ZodString;
118
+ timeout: z.ZodOptional<z.ZodNumber>;
119
+ }, z.core.$strip>], "type">;
120
+ export type Test = z.infer<typeof TestSchema>;
121
+ /**
122
+ * Union of all test result types.
123
+ */
124
+ export type TestResult = LLMTestResult | ToolTestResult | SiteConfigTestResult | CommandExecutionTestResult | FilePresenceTestResult | FileContentTestResult | BuildCheckTestResult | VitestTestResult | PlaywrightNLTestResult;
@@ -0,0 +1,36 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * LLM Test schema - tests that use an LLM evaluator.
5
+ */
6
+ export declare const LLMTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.LLM>;
12
+ maxSteps: z.ZodNumber;
13
+ prompt: z.ZodString;
14
+ evaluatorId: z.ZodString;
15
+ }, z.core.$strip>;
16
+ export type LLMTest = z.infer<typeof LLMTestSchema>;
17
+ /**
18
+ * LLM Test result.
19
+ */
20
+ export interface LLMTestResult extends BaseTestResult {
21
+ type: TestType.LLM;
22
+ testPrompt: string;
23
+ testSystemPrompt?: string;
24
+ text: string;
25
+ scoreReasoning: string;
26
+ score: number;
27
+ totalMicrocentsSpent?: number;
28
+ usage?: {
29
+ promptTokens?: number;
30
+ completionTokens?: number;
31
+ totalTokens?: number;
32
+ };
33
+ reasoning?: string;
34
+ reasoningDetails?: unknown;
35
+ finishReason?: string;
36
+ }
@@ -0,0 +1,28 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Playwright Natural Language Test schema - tests using natural language descriptions.
5
+ */
6
+ export declare const PlaywrightNLTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.PLAYWRIGHT_NL>;
12
+ steps: z.ZodArray<z.ZodString>;
13
+ expectedOutcome: z.ZodString;
14
+ timeout: z.ZodOptional<z.ZodNumber>;
15
+ }, z.core.$strip>;
16
+ export type PlaywrightNLTest = z.infer<typeof PlaywrightNLTestSchema>;
17
+ /**
18
+ * Playwright NL Test result.
19
+ */
20
+ export interface PlaywrightNLTestResult extends BaseTestResult {
21
+ type: TestType.PLAYWRIGHT_NL;
22
+ result: boolean;
23
+ stepsExecuted: number;
24
+ totalSteps: number;
25
+ failedStep?: string;
26
+ screenshot?: string;
27
+ duration: number;
28
+ }
@@ -0,0 +1,32 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Site Config Test schema - tests that verify site configuration via API.
5
+ */
6
+ export declare const SiteConfigTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.SITE_CONFIG>;
12
+ url: z.ZodString;
13
+ method: z.ZodEnum<{
14
+ GET: "GET";
15
+ POST: "POST";
16
+ }>;
17
+ body: z.ZodOptional<z.ZodString>;
18
+ expectedStatusCode: z.ZodNumber;
19
+ expectedResponse: z.ZodOptional<z.ZodString>;
20
+ expectedResponseJMESPath: z.ZodOptional<z.ZodString>;
21
+ }, z.core.$strip>;
22
+ export type SiteConfigTest = z.infer<typeof SiteConfigTestSchema>;
23
+ /**
24
+ * Site Config Test result.
25
+ */
26
+ export interface SiteConfigTestResult extends BaseTestResult {
27
+ type: TestType.SITE_CONFIG;
28
+ result: boolean;
29
+ actualStatusCode: number;
30
+ actualResponse: unknown;
31
+ allActualResponse: unknown;
32
+ }
@@ -0,0 +1,26 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Tool Test schema - tests that verify tool usage.
5
+ */
6
+ export declare const ToolTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.TOOL>;
12
+ toolName: z.ZodString;
13
+ args: z.ZodRecord<z.ZodString, z.ZodAny>;
14
+ resultsContent: z.ZodString;
15
+ }, z.core.$strip>;
16
+ export type ToolTest = z.infer<typeof ToolTestSchema>;
17
+ /**
18
+ * Tool Test result.
19
+ */
20
+ export interface ToolTestResult extends BaseTestResult {
21
+ type: TestType.TOOL;
22
+ result: boolean;
23
+ toolUsed: boolean;
24
+ actualArgs: Record<string, any>;
25
+ isResultsValid: boolean;
26
+ }
@@ -0,0 +1,30 @@
1
+ import { z } from 'zod';
2
+ import { BaseTestResult, TestType } from './base.js';
3
+ /**
4
+ * Vitest Test schema - tests that run Vitest test files.
5
+ */
6
+ export declare const VitestTestSchema: z.ZodObject<{
7
+ id: z.ZodString;
8
+ name: z.ZodString;
9
+ description: z.ZodOptional<z.ZodString>;
10
+ importance: z.ZodOptional<z.ZodEnum<typeof import("./base.js").TestImportance>>;
11
+ type: z.ZodLiteral<TestType.VITEST>;
12
+ testFile: z.ZodString;
13
+ testFileName: z.ZodString;
14
+ minPassRate: z.ZodNumber;
15
+ }, z.core.$strip>;
16
+ export type VitestTest = z.infer<typeof VitestTestSchema>;
17
+ /**
18
+ * Vitest Test result.
19
+ */
20
+ export interface VitestTestResult extends BaseTestResult {
21
+ type: TestType.VITEST;
22
+ result: boolean;
23
+ passRate: number;
24
+ passed: number;
25
+ failed: number;
26
+ skipped: number;
27
+ total: number;
28
+ duration: number;
29
+ output: string;
30
+ }
package/package.json ADDED
@@ -0,0 +1,50 @@
1
+ {
2
+ "name": "@wix/evalforge-types",
3
+ "version": "0.3.0",
4
+ "description": "Unified types for EvalForge agent evaluation system",
5
+ "files": [
6
+ "build"
7
+ ],
8
+ "scripts": {
9
+ "clean": "rm -rf build",
10
+ "build:cjs": "esbuild src/index.ts --bundle --platform=node --outfile=build/index.js --format=cjs --sourcemap --packages=external",
11
+ "build:esm": "esbuild src/index.ts --bundle --platform=node --outfile=build/index.mjs --format=esm --sourcemap --packages=external",
12
+ "build:types": "tsc --emitDeclarationOnly --outDir ./build/types",
13
+ "build": "yarn run clean && yarn run build:cjs && yarn run build:esm && yarn run build:types",
14
+ "lint": "eslint .",
15
+ "test": "echo 'No tests specified' && exit 0"
16
+ },
17
+ "dependencies": {
18
+ "zod": "^4.3.5"
19
+ },
20
+ "devDependencies": {
21
+ "@eslint/js": "^9.39.2",
22
+ "@types/node": "^22.19.3",
23
+ "esbuild": "^0.27.2",
24
+ "eslint": "^9.39.2",
25
+ "eslint-config-prettier": "^10.1.8",
26
+ "eslint-plugin-prettier": "^5.5.4",
27
+ "prettier": "^3.7.4",
28
+ "typescript": "^5.9.3",
29
+ "typescript-eslint": "^8.51.0"
30
+ },
31
+ "exports": {
32
+ ".": {
33
+ "types": "./build/types/index.d.ts",
34
+ "import": "./build/index.mjs",
35
+ "require": "./build/index.js"
36
+ },
37
+ "./package.json": "./package.json"
38
+ },
39
+ "publishConfig": {
40
+ "registry": "https://registry.npmjs.org/",
41
+ "access": "public"
42
+ },
43
+ "wix": {
44
+ "artifact": {
45
+ "groupId": "com.wixpress",
46
+ "artifactId": "evalforge-types"
47
+ }
48
+ },
49
+ "falconPackageHash": "92edc4b88f336dc91b2021314dd5f083ed1d257b28963bc36995d2b8"
50
+ }