@wix/evalforge-types 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +1047 -0
- package/build/index.js.map +7 -0
- package/build/index.mjs +928 -0
- package/build/index.mjs.map +7 -0
- package/build/types/common/base-entity.d.ts +26 -0
- package/build/types/common/index.d.ts +3 -0
- package/build/types/common/mcp.d.ts +17 -0
- package/build/types/common/models.d.ts +55 -0
- package/build/types/evaluation/eval-result.d.ts +239 -0
- package/build/types/evaluation/eval-run.d.ts +581 -0
- package/build/types/evaluation/index.d.ts +4 -0
- package/build/types/evaluation/live-trace.d.ts +47 -0
- package/build/types/evaluation/metrics.d.ts +157 -0
- package/build/types/improvement/improvement.d.ts +140 -0
- package/build/types/improvement/index.d.ts +1 -0
- package/build/types/index.d.ts +24 -0
- package/build/types/project/index.d.ts +1 -0
- package/build/types/project/project.d.ts +41 -0
- package/build/types/scenario/environment.d.ts +58 -0
- package/build/types/scenario/index.d.ts +2 -0
- package/build/types/scenario/test-scenario.d.ts +50 -0
- package/build/types/suite/index.d.ts +1 -0
- package/build/types/suite/test-suite.d.ts +37 -0
- package/build/types/target/agent.d.ts +53 -0
- package/build/types/target/index.d.ts +4 -0
- package/build/types/target/skill.d.ts +78 -0
- package/build/types/target/skills-group.d.ts +37 -0
- package/build/types/target/target.d.ts +17 -0
- package/build/types/template/index.d.ts +1 -0
- package/build/types/template/template.d.ts +38 -0
- package/build/types/test/base.d.ts +43 -0
- package/build/types/test/build-check.d.ts +29 -0
- package/build/types/test/command-execution.d.ts +31 -0
- package/build/types/test/file-content.d.ts +52 -0
- package/build/types/test/file-presence.d.ts +24 -0
- package/build/types/test/index.d.ts +124 -0
- package/build/types/test/llm.d.ts +36 -0
- package/build/types/test/playwright-nl.d.ts +28 -0
- package/build/types/test/site-config.d.ts +32 -0
- package/build/types/test/tool.d.ts +26 -0
- package/build/types/test/vitest.d.ts +30 -0
- package/package.json +50 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Token usage schema.
|
|
4
|
+
*/
|
|
5
|
+
export declare const TokenUsageSchema: z.ZodObject<{
|
|
6
|
+
prompt: z.ZodNumber;
|
|
7
|
+
completion: z.ZodNumber;
|
|
8
|
+
total: z.ZodNumber;
|
|
9
|
+
}, z.core.$strip>;
|
|
10
|
+
export type TokenUsage = z.infer<typeof TokenUsageSchema>;
|
|
11
|
+
/**
|
|
12
|
+
* Evaluation metrics schema.
|
|
13
|
+
*/
|
|
14
|
+
export declare const EvalMetricsSchema: z.ZodObject<{
|
|
15
|
+
totalAssertions: z.ZodNumber;
|
|
16
|
+
passed: z.ZodNumber;
|
|
17
|
+
failed: z.ZodNumber;
|
|
18
|
+
skipped: z.ZodNumber;
|
|
19
|
+
errors: z.ZodNumber;
|
|
20
|
+
passRate: z.ZodNumber;
|
|
21
|
+
avgDuration: z.ZodNumber;
|
|
22
|
+
totalDuration: z.ZodNumber;
|
|
23
|
+
}, z.core.$strip>;
|
|
24
|
+
export type EvalMetrics = z.infer<typeof EvalMetricsSchema>;
|
|
25
|
+
/**
|
|
26
|
+
* Evaluation status enum.
|
|
27
|
+
*/
|
|
28
|
+
export declare enum EvalStatus {
|
|
29
|
+
PENDING = "pending",
|
|
30
|
+
RUNNING = "running",
|
|
31
|
+
COMPLETED = "completed",
|
|
32
|
+
FAILED = "failed",
|
|
33
|
+
CANCELLED = "cancelled"
|
|
34
|
+
}
|
|
35
|
+
export declare const EvalStatusSchema: z.ZodEnum<typeof EvalStatus>;
|
|
36
|
+
/**
|
|
37
|
+
* LLM step type enum.
|
|
38
|
+
*/
|
|
39
|
+
export declare enum LLMStepType {
|
|
40
|
+
COMPLETION = "completion",
|
|
41
|
+
TOOL_USE = "tool_use",
|
|
42
|
+
TOOL_RESULT = "tool_result",
|
|
43
|
+
THINKING = "thinking"
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* LLM trace step schema.
|
|
47
|
+
*/
|
|
48
|
+
export declare const LLMTraceStepSchema: z.ZodObject<{
|
|
49
|
+
id: z.ZodString;
|
|
50
|
+
stepNumber: z.ZodNumber;
|
|
51
|
+
type: z.ZodEnum<typeof LLMStepType>;
|
|
52
|
+
model: z.ZodString;
|
|
53
|
+
provider: z.ZodString;
|
|
54
|
+
startedAt: z.ZodString;
|
|
55
|
+
durationMs: z.ZodNumber;
|
|
56
|
+
tokenUsage: z.ZodObject<{
|
|
57
|
+
prompt: z.ZodNumber;
|
|
58
|
+
completion: z.ZodNumber;
|
|
59
|
+
total: z.ZodNumber;
|
|
60
|
+
}, z.core.$strip>;
|
|
61
|
+
costUsd: z.ZodNumber;
|
|
62
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
63
|
+
toolArguments: z.ZodOptional<z.ZodString>;
|
|
64
|
+
inputPreview: z.ZodOptional<z.ZodString>;
|
|
65
|
+
outputPreview: z.ZodOptional<z.ZodString>;
|
|
66
|
+
success: z.ZodBoolean;
|
|
67
|
+
error: z.ZodOptional<z.ZodString>;
|
|
68
|
+
}, z.core.$strip>;
|
|
69
|
+
export type LLMTraceStep = z.infer<typeof LLMTraceStepSchema>;
|
|
70
|
+
/**
|
|
71
|
+
* LLM breakdown stats schema.
|
|
72
|
+
*/
|
|
73
|
+
export declare const LLMBreakdownStatsSchema: z.ZodObject<{
|
|
74
|
+
count: z.ZodNumber;
|
|
75
|
+
durationMs: z.ZodNumber;
|
|
76
|
+
tokens: z.ZodNumber;
|
|
77
|
+
costUsd: z.ZodNumber;
|
|
78
|
+
}, z.core.$strip>;
|
|
79
|
+
export type LLMBreakdownStats = z.infer<typeof LLMBreakdownStatsSchema>;
|
|
80
|
+
/**
|
|
81
|
+
* LLM trace summary schema.
|
|
82
|
+
*/
|
|
83
|
+
export declare const LLMTraceSummarySchema: z.ZodObject<{
|
|
84
|
+
totalSteps: z.ZodNumber;
|
|
85
|
+
totalDurationMs: z.ZodNumber;
|
|
86
|
+
totalTokens: z.ZodObject<{
|
|
87
|
+
prompt: z.ZodNumber;
|
|
88
|
+
completion: z.ZodNumber;
|
|
89
|
+
total: z.ZodNumber;
|
|
90
|
+
}, z.core.$strip>;
|
|
91
|
+
totalCostUsd: z.ZodNumber;
|
|
92
|
+
stepTypeBreakdown: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
93
|
+
count: z.ZodNumber;
|
|
94
|
+
durationMs: z.ZodNumber;
|
|
95
|
+
tokens: z.ZodNumber;
|
|
96
|
+
costUsd: z.ZodNumber;
|
|
97
|
+
}, z.core.$strip>>>;
|
|
98
|
+
modelBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
99
|
+
count: z.ZodNumber;
|
|
100
|
+
durationMs: z.ZodNumber;
|
|
101
|
+
tokens: z.ZodNumber;
|
|
102
|
+
costUsd: z.ZodNumber;
|
|
103
|
+
}, z.core.$strip>>;
|
|
104
|
+
modelsUsed: z.ZodArray<z.ZodString>;
|
|
105
|
+
}, z.core.$strip>;
|
|
106
|
+
export type LLMTraceSummary = z.infer<typeof LLMTraceSummarySchema>;
|
|
107
|
+
/**
|
|
108
|
+
* LLM trace schema.
|
|
109
|
+
*/
|
|
110
|
+
export declare const LLMTraceSchema: z.ZodObject<{
|
|
111
|
+
id: z.ZodString;
|
|
112
|
+
steps: z.ZodArray<z.ZodObject<{
|
|
113
|
+
id: z.ZodString;
|
|
114
|
+
stepNumber: z.ZodNumber;
|
|
115
|
+
type: z.ZodEnum<typeof LLMStepType>;
|
|
116
|
+
model: z.ZodString;
|
|
117
|
+
provider: z.ZodString;
|
|
118
|
+
startedAt: z.ZodString;
|
|
119
|
+
durationMs: z.ZodNumber;
|
|
120
|
+
tokenUsage: z.ZodObject<{
|
|
121
|
+
prompt: z.ZodNumber;
|
|
122
|
+
completion: z.ZodNumber;
|
|
123
|
+
total: z.ZodNumber;
|
|
124
|
+
}, z.core.$strip>;
|
|
125
|
+
costUsd: z.ZodNumber;
|
|
126
|
+
toolName: z.ZodOptional<z.ZodString>;
|
|
127
|
+
toolArguments: z.ZodOptional<z.ZodString>;
|
|
128
|
+
inputPreview: z.ZodOptional<z.ZodString>;
|
|
129
|
+
outputPreview: z.ZodOptional<z.ZodString>;
|
|
130
|
+
success: z.ZodBoolean;
|
|
131
|
+
error: z.ZodOptional<z.ZodString>;
|
|
132
|
+
}, z.core.$strip>>;
|
|
133
|
+
summary: z.ZodObject<{
|
|
134
|
+
totalSteps: z.ZodNumber;
|
|
135
|
+
totalDurationMs: z.ZodNumber;
|
|
136
|
+
totalTokens: z.ZodObject<{
|
|
137
|
+
prompt: z.ZodNumber;
|
|
138
|
+
completion: z.ZodNumber;
|
|
139
|
+
total: z.ZodNumber;
|
|
140
|
+
}, z.core.$strip>;
|
|
141
|
+
totalCostUsd: z.ZodNumber;
|
|
142
|
+
stepTypeBreakdown: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
143
|
+
count: z.ZodNumber;
|
|
144
|
+
durationMs: z.ZodNumber;
|
|
145
|
+
tokens: z.ZodNumber;
|
|
146
|
+
costUsd: z.ZodNumber;
|
|
147
|
+
}, z.core.$strip>>>;
|
|
148
|
+
modelBreakdown: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
149
|
+
count: z.ZodNumber;
|
|
150
|
+
durationMs: z.ZodNumber;
|
|
151
|
+
tokens: z.ZodNumber;
|
|
152
|
+
costUsd: z.ZodNumber;
|
|
153
|
+
}, z.core.$strip>>;
|
|
154
|
+
modelsUsed: z.ZodArray<z.ZodString>;
|
|
155
|
+
}, z.core.$strip>;
|
|
156
|
+
}, z.core.$strip>;
|
|
157
|
+
export type LLMTrace = z.infer<typeof LLMTraceSchema>;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Improvement target types.
|
|
4
|
+
*/
|
|
5
|
+
export declare const ImprovementTargetTypeSchema: z.ZodEnum<{
|
|
6
|
+
prompt_agent: "prompt_agent";
|
|
7
|
+
skill: "skill";
|
|
8
|
+
agent: "agent";
|
|
9
|
+
}>;
|
|
10
|
+
export type ImprovementTargetType = z.infer<typeof ImprovementTargetTypeSchema>;
|
|
11
|
+
/**
|
|
12
|
+
* Improvement run status.
|
|
13
|
+
*/
|
|
14
|
+
export declare const ImprovementRunStatusSchema: z.ZodEnum<{
|
|
15
|
+
failed: "failed";
|
|
16
|
+
pending: "pending";
|
|
17
|
+
running: "running";
|
|
18
|
+
completed: "completed";
|
|
19
|
+
}>;
|
|
20
|
+
export type ImprovementRunStatus = z.infer<typeof ImprovementRunStatusSchema>;
|
|
21
|
+
/**
|
|
22
|
+
* Iteration change schema - represents a single change made during an improvement iteration.
|
|
23
|
+
*/
|
|
24
|
+
export declare const IterationChangeSchema: z.ZodObject<{
|
|
25
|
+
field: z.ZodString;
|
|
26
|
+
before: z.ZodString;
|
|
27
|
+
after: z.ZodString;
|
|
28
|
+
reason: z.ZodString;
|
|
29
|
+
}, z.core.$strip>;
|
|
30
|
+
export type IterationChange = z.infer<typeof IterationChangeSchema>;
|
|
31
|
+
/**
|
|
32
|
+
* Improvement iteration schema - represents a single iteration in an improvement run.
|
|
33
|
+
*/
|
|
34
|
+
export declare const ImprovementIterationSchema: z.ZodObject<{
|
|
35
|
+
id: z.ZodString;
|
|
36
|
+
iterationNumber: z.ZodNumber;
|
|
37
|
+
passRate: z.ZodNumber;
|
|
38
|
+
passed: z.ZodNumber;
|
|
39
|
+
failed: z.ZodNumber;
|
|
40
|
+
changes: z.ZodArray<z.ZodObject<{
|
|
41
|
+
field: z.ZodString;
|
|
42
|
+
before: z.ZodString;
|
|
43
|
+
after: z.ZodString;
|
|
44
|
+
reason: z.ZodString;
|
|
45
|
+
}, z.core.$strip>>;
|
|
46
|
+
feedback: z.ZodString;
|
|
47
|
+
targetSnapshot: z.ZodAny;
|
|
48
|
+
evaluatedAt: z.ZodString;
|
|
49
|
+
}, z.core.$strip>;
|
|
50
|
+
export type ImprovementIteration = z.infer<typeof ImprovementIterationSchema>;
|
|
51
|
+
/**
|
|
52
|
+
* Improvement run schema - represents a complete improvement run with iterations.
|
|
53
|
+
*
|
|
54
|
+
* Extends TenantEntity for multi-tenancy support.
|
|
55
|
+
*/
|
|
56
|
+
export declare const ImprovementRunSchema: z.ZodObject<{
|
|
57
|
+
id: z.ZodString;
|
|
58
|
+
name: z.ZodString;
|
|
59
|
+
description: z.ZodString;
|
|
60
|
+
createdAt: z.ZodString;
|
|
61
|
+
updatedAt: z.ZodString;
|
|
62
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
63
|
+
projectId: z.ZodString;
|
|
64
|
+
targetType: z.ZodEnum<{
|
|
65
|
+
prompt_agent: "prompt_agent";
|
|
66
|
+
skill: "skill";
|
|
67
|
+
agent: "agent";
|
|
68
|
+
}>;
|
|
69
|
+
targetId: z.ZodString;
|
|
70
|
+
targetName: z.ZodString;
|
|
71
|
+
testSuiteId: z.ZodString;
|
|
72
|
+
testSuiteName: z.ZodString;
|
|
73
|
+
status: z.ZodEnum<{
|
|
74
|
+
failed: "failed";
|
|
75
|
+
pending: "pending";
|
|
76
|
+
running: "running";
|
|
77
|
+
completed: "completed";
|
|
78
|
+
}>;
|
|
79
|
+
maxIterations: z.ZodNumber;
|
|
80
|
+
iterations: z.ZodArray<z.ZodObject<{
|
|
81
|
+
id: z.ZodString;
|
|
82
|
+
iterationNumber: z.ZodNumber;
|
|
83
|
+
passRate: z.ZodNumber;
|
|
84
|
+
passed: z.ZodNumber;
|
|
85
|
+
failed: z.ZodNumber;
|
|
86
|
+
changes: z.ZodArray<z.ZodObject<{
|
|
87
|
+
field: z.ZodString;
|
|
88
|
+
before: z.ZodString;
|
|
89
|
+
after: z.ZodString;
|
|
90
|
+
reason: z.ZodString;
|
|
91
|
+
}, z.core.$strip>>;
|
|
92
|
+
feedback: z.ZodString;
|
|
93
|
+
targetSnapshot: z.ZodAny;
|
|
94
|
+
evaluatedAt: z.ZodString;
|
|
95
|
+
}, z.core.$strip>>;
|
|
96
|
+
initialPassRate: z.ZodNumber;
|
|
97
|
+
finalPassRate: z.ZodNumber;
|
|
98
|
+
improvement: z.ZodNumber;
|
|
99
|
+
startedAt: z.ZodString;
|
|
100
|
+
completedAt: z.ZodOptional<z.ZodString>;
|
|
101
|
+
}, z.core.$strip>;
|
|
102
|
+
export type ImprovementRun = z.infer<typeof ImprovementRunSchema>;
|
|
103
|
+
/**
|
|
104
|
+
* Input schema for creating a new improvement run.
|
|
105
|
+
*/
|
|
106
|
+
export declare const CreateImprovementRunInputSchema: z.ZodObject<{
|
|
107
|
+
name: z.ZodString;
|
|
108
|
+
description: z.ZodString;
|
|
109
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
110
|
+
targetId: z.ZodString;
|
|
111
|
+
targetName: z.ZodString;
|
|
112
|
+
targetType: z.ZodEnum<{
|
|
113
|
+
prompt_agent: "prompt_agent";
|
|
114
|
+
skill: "skill";
|
|
115
|
+
agent: "agent";
|
|
116
|
+
}>;
|
|
117
|
+
testSuiteId: z.ZodString;
|
|
118
|
+
testSuiteName: z.ZodString;
|
|
119
|
+
maxIterations: z.ZodNumber;
|
|
120
|
+
}, z.core.$strip>;
|
|
121
|
+
export type CreateImprovementRunInput = z.infer<typeof CreateImprovementRunInputSchema>;
|
|
122
|
+
/**
|
|
123
|
+
* Input schema for updating an improvement run.
|
|
124
|
+
*/
|
|
125
|
+
export declare const UpdateImprovementRunInputSchema: z.ZodObject<{
|
|
126
|
+
name: z.ZodOptional<z.ZodString>;
|
|
127
|
+
description: z.ZodOptional<z.ZodString>;
|
|
128
|
+
deleted: z.ZodOptional<z.ZodOptional<z.ZodBoolean>>;
|
|
129
|
+
targetId: z.ZodOptional<z.ZodString>;
|
|
130
|
+
targetName: z.ZodOptional<z.ZodString>;
|
|
131
|
+
targetType: z.ZodOptional<z.ZodEnum<{
|
|
132
|
+
prompt_agent: "prompt_agent";
|
|
133
|
+
skill: "skill";
|
|
134
|
+
agent: "agent";
|
|
135
|
+
}>>;
|
|
136
|
+
testSuiteId: z.ZodOptional<z.ZodString>;
|
|
137
|
+
testSuiteName: z.ZodOptional<z.ZodString>;
|
|
138
|
+
maxIterations: z.ZodOptional<z.ZodNumber>;
|
|
139
|
+
}, z.core.$strip>;
|
|
140
|
+
export type UpdateImprovementRunInput = z.infer<typeof UpdateImprovementRunInputSchema>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './improvement.js';
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @wix/evalforge-types
|
|
3
|
+
*
|
|
4
|
+
* Unified types for the agent evaluation system.
|
|
5
|
+
*
|
|
6
|
+
* Entity Hierarchy:
|
|
7
|
+
* - BaseEntity: id, name, description, dates
|
|
8
|
+
* - TenantEntity: extends BaseEntity with projectId
|
|
9
|
+
* - Target: extends TenantEntity (base for testable entities)
|
|
10
|
+
* - Agent: CLI-based agent (runCommand, modelConfig)
|
|
11
|
+
* - Skill: SKILL.md-based capability
|
|
12
|
+
*
|
|
13
|
+
* Test Types (9 total):
|
|
14
|
+
* - LLM, TOOL, SITE_CONFIG, COMMAND_EXECUTION (from old system)
|
|
15
|
+
* - FILE_PRESENCE, FILE_CONTENT, BUILD_CHECK, VITEST, PLAYWRIGHT_NL (new)
|
|
16
|
+
*/
|
|
17
|
+
export * from './common/index.js';
|
|
18
|
+
export * from './target/index.js';
|
|
19
|
+
export * from './test/index.js';
|
|
20
|
+
export * from './scenario/index.js';
|
|
21
|
+
export * from './suite/index.js';
|
|
22
|
+
export * from './evaluation/index.js';
|
|
23
|
+
export * from './project/index.js';
|
|
24
|
+
export * from './template/index.js';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './project.js';
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Project schema - represents a tenant/workspace for data isolation.
|
|
4
|
+
*
|
|
5
|
+
* All entities belong to a project. Projects are the top-level
|
|
6
|
+
* organizational unit for multi-tenancy.
|
|
7
|
+
*
|
|
8
|
+
* Note: Project extends BaseEntity (not TenantEntity) since
|
|
9
|
+
* Project IS the tenant and doesn't need a projectId.
|
|
10
|
+
*/
|
|
11
|
+
export declare const ProjectSchema: z.ZodObject<{
|
|
12
|
+
id: z.ZodString;
|
|
13
|
+
name: z.ZodString;
|
|
14
|
+
description: z.ZodString;
|
|
15
|
+
createdAt: z.ZodString;
|
|
16
|
+
updatedAt: z.ZodString;
|
|
17
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
18
|
+
appId: z.ZodOptional<z.ZodString>;
|
|
19
|
+
appSecret: z.ZodOptional<z.ZodString>;
|
|
20
|
+
}, z.core.$strip>;
|
|
21
|
+
export type Project = z.infer<typeof ProjectSchema>;
|
|
22
|
+
/**
|
|
23
|
+
* Input schema for creating a new Project.
|
|
24
|
+
*/
|
|
25
|
+
export declare const CreateProjectInputSchema: z.ZodObject<{
|
|
26
|
+
name: z.ZodString;
|
|
27
|
+
description: z.ZodString;
|
|
28
|
+
appId: z.ZodOptional<z.ZodString>;
|
|
29
|
+
appSecret: z.ZodOptional<z.ZodString>;
|
|
30
|
+
}, z.core.$strip>;
|
|
31
|
+
export type CreateProjectInput = z.infer<typeof CreateProjectInputSchema>;
|
|
32
|
+
/**
|
|
33
|
+
* Input schema for updating a Project.
|
|
34
|
+
*/
|
|
35
|
+
export declare const UpdateProjectInputSchema: z.ZodObject<{
|
|
36
|
+
name: z.ZodOptional<z.ZodString>;
|
|
37
|
+
description: z.ZodOptional<z.ZodString>;
|
|
38
|
+
appId: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
39
|
+
appSecret: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
40
|
+
}, z.core.$strip>;
|
|
41
|
+
export type UpdateProjectInput = z.infer<typeof UpdateProjectInputSchema>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Local project configuration schema.
|
|
4
|
+
*/
|
|
5
|
+
export declare const LocalProjectConfigSchema: z.ZodObject<{
|
|
6
|
+
templateId: z.ZodOptional<z.ZodString>;
|
|
7
|
+
files: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
8
|
+
path: z.ZodString;
|
|
9
|
+
content: z.ZodString;
|
|
10
|
+
}, z.core.$strip>>>;
|
|
11
|
+
}, z.core.$strip>;
|
|
12
|
+
export type LocalProjectConfig = z.infer<typeof LocalProjectConfigSchema>;
|
|
13
|
+
/**
|
|
14
|
+
* Meta site configuration schema for API-based setup.
|
|
15
|
+
*/
|
|
16
|
+
export declare const MetaSiteConfigSchema: z.ZodObject<{
|
|
17
|
+
configurations: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
18
|
+
name: z.ZodString;
|
|
19
|
+
apiCalls: z.ZodArray<z.ZodObject<{
|
|
20
|
+
url: z.ZodString;
|
|
21
|
+
method: z.ZodEnum<{
|
|
22
|
+
POST: "POST";
|
|
23
|
+
PUT: "PUT";
|
|
24
|
+
}>;
|
|
25
|
+
body: z.ZodString;
|
|
26
|
+
}, z.core.$strip>>;
|
|
27
|
+
}, z.core.$strip>>>;
|
|
28
|
+
}, z.core.$strip>;
|
|
29
|
+
export type MetaSiteConfig = z.infer<typeof MetaSiteConfigSchema>;
|
|
30
|
+
/**
|
|
31
|
+
* Environment configuration schema.
|
|
32
|
+
*
|
|
33
|
+
* Defines the environment setup required for running a test scenario.
|
|
34
|
+
* Migrated from the old Prompt schema.
|
|
35
|
+
*/
|
|
36
|
+
export declare const EnvironmentSchema: z.ZodObject<{
|
|
37
|
+
localProject: z.ZodOptional<z.ZodObject<{
|
|
38
|
+
templateId: z.ZodOptional<z.ZodString>;
|
|
39
|
+
files: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
40
|
+
path: z.ZodString;
|
|
41
|
+
content: z.ZodString;
|
|
42
|
+
}, z.core.$strip>>>;
|
|
43
|
+
}, z.core.$strip>>;
|
|
44
|
+
metaSite: z.ZodOptional<z.ZodObject<{
|
|
45
|
+
configurations: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
46
|
+
name: z.ZodString;
|
|
47
|
+
apiCalls: z.ZodArray<z.ZodObject<{
|
|
48
|
+
url: z.ZodString;
|
|
49
|
+
method: z.ZodEnum<{
|
|
50
|
+
POST: "POST";
|
|
51
|
+
PUT: "PUT";
|
|
52
|
+
}>;
|
|
53
|
+
body: z.ZodString;
|
|
54
|
+
}, z.core.$strip>>;
|
|
55
|
+
}, z.core.$strip>>>;
|
|
56
|
+
}, z.core.$strip>>;
|
|
57
|
+
}, z.core.$strip>;
|
|
58
|
+
export type Environment = z.infer<typeof EnvironmentSchema>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Expected file schema - a file the agent is expected to create or modify.
|
|
4
|
+
* Used by eval results; not persisted on test_scenarios table.
|
|
5
|
+
*/
|
|
6
|
+
export declare const ExpectedFileSchema: z.ZodObject<{
|
|
7
|
+
path: z.ZodString;
|
|
8
|
+
content: z.ZodOptional<z.ZodString>;
|
|
9
|
+
}, z.core.$strip>;
|
|
10
|
+
export type ExpectedFile = z.infer<typeof ExpectedFileSchema>;
|
|
11
|
+
/**
|
|
12
|
+
* TestScenario schema - defines a single test scenario.
|
|
13
|
+
*
|
|
14
|
+
* Persisted shape matches test_scenarios table:
|
|
15
|
+
* id, project_id, name, description, trigger_prompt, template_id, created_at, updated_at, deleted.
|
|
16
|
+
*/
|
|
17
|
+
export declare const TestScenarioSchema: z.ZodObject<{
|
|
18
|
+
id: z.ZodString;
|
|
19
|
+
name: z.ZodString;
|
|
20
|
+
description: z.ZodString;
|
|
21
|
+
createdAt: z.ZodString;
|
|
22
|
+
updatedAt: z.ZodString;
|
|
23
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
24
|
+
projectId: z.ZodString;
|
|
25
|
+
triggerPrompt: z.ZodString;
|
|
26
|
+
templateId: z.ZodOptional<z.ZodString>;
|
|
27
|
+
}, z.core.$strip>;
|
|
28
|
+
export type TestScenario = z.infer<typeof TestScenarioSchema>;
|
|
29
|
+
/**
|
|
30
|
+
* Input schema for creating a new TestScenario.
|
|
31
|
+
*/
|
|
32
|
+
export declare const CreateTestScenarioInputSchema: z.ZodObject<{
|
|
33
|
+
name: z.ZodString;
|
|
34
|
+
description: z.ZodString;
|
|
35
|
+
projectId: z.ZodString;
|
|
36
|
+
templateId: z.ZodOptional<z.ZodString>;
|
|
37
|
+
triggerPrompt: z.ZodString;
|
|
38
|
+
}, z.core.$strip>;
|
|
39
|
+
export type CreateTestScenarioInput = z.infer<typeof CreateTestScenarioInputSchema>;
|
|
40
|
+
/**
|
|
41
|
+
* Input schema for updating a TestScenario.
|
|
42
|
+
*/
|
|
43
|
+
export declare const UpdateTestScenarioInputSchema: z.ZodObject<{
|
|
44
|
+
name: z.ZodOptional<z.ZodString>;
|
|
45
|
+
description: z.ZodOptional<z.ZodString>;
|
|
46
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
47
|
+
templateId: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
48
|
+
triggerPrompt: z.ZodOptional<z.ZodString>;
|
|
49
|
+
}, z.core.$strip>;
|
|
50
|
+
export type UpdateTestScenarioInput = z.infer<typeof UpdateTestScenarioInputSchema>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './test-suite.js';
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* TestSuite schema - a collection of test scenarios.
|
|
4
|
+
*
|
|
5
|
+
* Suites are used to organize and group related test scenarios.
|
|
6
|
+
*/
|
|
7
|
+
export declare const TestSuiteSchema: z.ZodObject<{
|
|
8
|
+
id: z.ZodString;
|
|
9
|
+
name: z.ZodString;
|
|
10
|
+
description: z.ZodString;
|
|
11
|
+
createdAt: z.ZodString;
|
|
12
|
+
updatedAt: z.ZodString;
|
|
13
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
14
|
+
projectId: z.ZodString;
|
|
15
|
+
scenarioIds: z.ZodArray<z.ZodString>;
|
|
16
|
+
}, z.core.$strip>;
|
|
17
|
+
export type TestSuite = z.infer<typeof TestSuiteSchema>;
|
|
18
|
+
/**
|
|
19
|
+
* Input schema for creating a new TestSuite.
|
|
20
|
+
*/
|
|
21
|
+
export declare const CreateTestSuiteInputSchema: z.ZodObject<{
|
|
22
|
+
name: z.ZodString;
|
|
23
|
+
description: z.ZodString;
|
|
24
|
+
projectId: z.ZodString;
|
|
25
|
+
scenarioIds: z.ZodArray<z.ZodString>;
|
|
26
|
+
}, z.core.$strip>;
|
|
27
|
+
export type CreateTestSuiteInput = z.infer<typeof CreateTestSuiteInputSchema>;
|
|
28
|
+
/**
|
|
29
|
+
* Input schema for updating a TestSuite.
|
|
30
|
+
*/
|
|
31
|
+
export declare const UpdateTestSuiteInputSchema: z.ZodObject<{
|
|
32
|
+
name: z.ZodOptional<z.ZodString>;
|
|
33
|
+
description: z.ZodOptional<z.ZodString>;
|
|
34
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
35
|
+
scenarioIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
36
|
+
}, z.core.$strip>;
|
|
37
|
+
export type UpdateTestSuiteInput = z.infer<typeof UpdateTestSuiteInputSchema>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Agent schema - a CLI-based coding agent.
|
|
4
|
+
*
|
|
5
|
+
* Agents are external CLI tools that can execute coding tasks.
|
|
6
|
+
* Examples: Claude Code CLI, Codex CLI, Cursor CLI.
|
|
7
|
+
*/
|
|
8
|
+
export declare const AgentSchema: z.ZodObject<{
|
|
9
|
+
id: z.ZodString;
|
|
10
|
+
name: z.ZodString;
|
|
11
|
+
description: z.ZodString;
|
|
12
|
+
createdAt: z.ZodString;
|
|
13
|
+
updatedAt: z.ZodString;
|
|
14
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
15
|
+
projectId: z.ZodString;
|
|
16
|
+
runCommand: z.ZodString;
|
|
17
|
+
modelConfig: z.ZodOptional<z.ZodObject<{
|
|
18
|
+
model: z.ZodEnum<typeof import("../common/models.js").ModelIds>;
|
|
19
|
+
temperature: z.ZodOptional<z.ZodNumber>;
|
|
20
|
+
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
21
|
+
}, z.core.$strip>>;
|
|
22
|
+
}, z.core.$strip>;
|
|
23
|
+
export type Agent = z.infer<typeof AgentSchema>;
|
|
24
|
+
/**
|
|
25
|
+
* Input schema for creating a new Agent.
|
|
26
|
+
*/
|
|
27
|
+
export declare const CreateAgentInputSchema: z.ZodObject<{
|
|
28
|
+
name: z.ZodString;
|
|
29
|
+
description: z.ZodString;
|
|
30
|
+
projectId: z.ZodString;
|
|
31
|
+
runCommand: z.ZodString;
|
|
32
|
+
modelConfig: z.ZodOptional<z.ZodObject<{
|
|
33
|
+
model: z.ZodEnum<typeof import("../common/models.js").ModelIds>;
|
|
34
|
+
temperature: z.ZodOptional<z.ZodNumber>;
|
|
35
|
+
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
36
|
+
}, z.core.$strip>>;
|
|
37
|
+
}, z.core.$strip>;
|
|
38
|
+
export type CreateAgentInput = z.infer<typeof CreateAgentInputSchema>;
|
|
39
|
+
/**
|
|
40
|
+
* Input schema for updating an Agent.
|
|
41
|
+
*/
|
|
42
|
+
export declare const UpdateAgentInputSchema: z.ZodObject<{
|
|
43
|
+
name: z.ZodOptional<z.ZodString>;
|
|
44
|
+
description: z.ZodOptional<z.ZodString>;
|
|
45
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
46
|
+
runCommand: z.ZodOptional<z.ZodString>;
|
|
47
|
+
modelConfig: z.ZodOptional<z.ZodOptional<z.ZodObject<{
|
|
48
|
+
model: z.ZodEnum<typeof import("../common/models.js").ModelIds>;
|
|
49
|
+
temperature: z.ZodOptional<z.ZodNumber>;
|
|
50
|
+
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
51
|
+
}, z.core.$strip>>>;
|
|
52
|
+
}, z.core.$strip>;
|
|
53
|
+
export type UpdateAgentInput = z.infer<typeof UpdateAgentInputSchema>;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Skill metadata parsed from SKILL.md frontmatter.
|
|
4
|
+
*
|
|
5
|
+
* Following the agentskills.io specification:
|
|
6
|
+
* - name: Skill identifier (max 64 chars, lowercase + hyphens)
|
|
7
|
+
* - description: What the skill does and when to use it
|
|
8
|
+
* - allowedTools: Pre-approved tools the skill may use
|
|
9
|
+
* - skills: Sub-skills for composite agents
|
|
10
|
+
*/
|
|
11
|
+
export declare const SkillMetadataSchema: z.ZodObject<{
|
|
12
|
+
name: z.ZodString;
|
|
13
|
+
description: z.ZodString;
|
|
14
|
+
allowedTools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
15
|
+
skills: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
16
|
+
}, z.core.$strip>;
|
|
17
|
+
export type SkillMetadata = z.infer<typeof SkillMetadataSchema>;
|
|
18
|
+
/**
|
|
19
|
+
* Skill version - historical snapshot of a Skill's content.
|
|
20
|
+
*/
|
|
21
|
+
export declare const SkillVersionSchema: z.ZodObject<{
|
|
22
|
+
id: z.ZodString;
|
|
23
|
+
skillId: z.ZodString;
|
|
24
|
+
skillMd: z.ZodString;
|
|
25
|
+
metadata: z.ZodObject<{
|
|
26
|
+
name: z.ZodString;
|
|
27
|
+
description: z.ZodString;
|
|
28
|
+
allowedTools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
29
|
+
skills: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
30
|
+
}, z.core.$strip>;
|
|
31
|
+
model: z.ZodOptional<z.ZodObject<{
|
|
32
|
+
model: z.ZodEnum<typeof import("../common/models.js").ModelIds>;
|
|
33
|
+
temperature: z.ZodOptional<z.ZodNumber>;
|
|
34
|
+
maxTokens: z.ZodOptional<z.ZodNumber>;
|
|
35
|
+
}, z.core.$strip>>;
|
|
36
|
+
systemPrompt: z.ZodOptional<z.ZodString>;
|
|
37
|
+
version: z.ZodNumber;
|
|
38
|
+
createdAt: z.ZodString;
|
|
39
|
+
notes: z.ZodOptional<z.ZodString>;
|
|
40
|
+
}, z.core.$strip>;
|
|
41
|
+
export type SkillVersion = z.infer<typeof SkillVersionSchema>;
|
|
42
|
+
/**
|
|
43
|
+
* Skill schema - a coding capability defined by SKILL.md.
|
|
44
|
+
*
|
|
45
|
+
* Skills represent coding capabilities that can be tested.
|
|
46
|
+
* Persisted shape: id, projectId, name, description, createdAt, updatedAt, deleted, skillMd.
|
|
47
|
+
*/
|
|
48
|
+
export declare const SkillSchema: z.ZodObject<{
|
|
49
|
+
id: z.ZodString;
|
|
50
|
+
name: z.ZodString;
|
|
51
|
+
description: z.ZodString;
|
|
52
|
+
createdAt: z.ZodString;
|
|
53
|
+
updatedAt: z.ZodString;
|
|
54
|
+
deleted: z.ZodOptional<z.ZodBoolean>;
|
|
55
|
+
projectId: z.ZodString;
|
|
56
|
+
skillMd: z.ZodString;
|
|
57
|
+
}, z.core.$strip>;
|
|
58
|
+
export type Skill = z.infer<typeof SkillSchema>;
|
|
59
|
+
/**
|
|
60
|
+
* Input schema for creating a new Skill.
|
|
61
|
+
*/
|
|
62
|
+
export declare const CreateSkillInputSchema: z.ZodObject<{
|
|
63
|
+
name: z.ZodString;
|
|
64
|
+
description: z.ZodString;
|
|
65
|
+
projectId: z.ZodString;
|
|
66
|
+
skillMd: z.ZodString;
|
|
67
|
+
}, z.core.$strip>;
|
|
68
|
+
export type CreateSkillInput = z.infer<typeof CreateSkillInputSchema>;
|
|
69
|
+
/**
|
|
70
|
+
* Input schema for updating a Skill.
|
|
71
|
+
*/
|
|
72
|
+
export declare const UpdateSkillInputSchema: z.ZodObject<{
|
|
73
|
+
name: z.ZodOptional<z.ZodString>;
|
|
74
|
+
description: z.ZodOptional<z.ZodString>;
|
|
75
|
+
projectId: z.ZodOptional<z.ZodString>;
|
|
76
|
+
skillMd: z.ZodOptional<z.ZodString>;
|
|
77
|
+
}, z.core.$strip>;
|
|
78
|
+
export type UpdateSkillInput = z.infer<typeof UpdateSkillInputSchema>;
|