@wix/evalforge-types 0.68.0 → 0.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/build/index.js +27 -38
  2. package/build/index.js.map +3 -3
  3. package/build/index.mjs +25 -35
  4. package/build/index.mjs.map +3 -3
  5. package/build/types/evaluation/eval-run.d.ts +87 -169
  6. package/package.json +2 -2
package/build/types/evaluation/eval-run.d.ts CHANGED
@@ -31,26 +31,6 @@ export declare const TriggerSchema: z.ZodObject<{
31
31
  type: z.ZodEnum<typeof TriggerType>;
32
32
  }, z.core.$strip>;
33
33
  export type Trigger = z.infer<typeof TriggerSchema>;
34
- /**
35
- * Failure category enum.
36
- */
37
- export declare enum FailureCategory {
38
- MISSING_FILE = "missing_file",
39
- WRONG_CONTENT = "wrong_content",
40
- BUILD_ERROR = "build_error",
41
- TEST_FAILURE = "test_failure",
42
- RUNTIME_ERROR = "runtime_error",
43
- PERFORMANCE = "performance"
44
- }
45
- /**
46
- * Failure severity enum.
47
- */
48
- export declare enum FailureSeverity {
49
- CRITICAL = "critical",
50
- HIGH = "high",
51
- MEDIUM = "medium",
52
- LOW = "low"
53
- }
54
34
  /**
55
35
  * Diff line type schema.
56
36
  */
@@ -173,59 +153,51 @@ export declare const ExecutionTraceSchema: z.ZodObject<{
173
153
  }, z.core.$strip>;
174
154
  export type ExecutionTrace = z.infer<typeof ExecutionTraceSchema>;
175
155
  /**
176
- * Failure analysis schema.
156
+ * A single finding from the LLM-generated run analysis.
177
157
  */
178
- export declare const FailureAnalysisSchema: z.ZodObject<{
179
- category: z.ZodEnum<typeof FailureCategory>;
180
- severity: z.ZodEnum<typeof FailureSeverity>;
158
+ export declare const RunAnalysisFindingSchema: z.ZodObject<{
159
+ category: z.ZodEnum<{
160
+ failure_pattern: "failure_pattern";
161
+ cost_waste: "cost_waste";
162
+ flakiness: "flakiness";
163
+ inefficiency: "inefficiency";
164
+ positive: "positive";
165
+ }>;
166
+ severity: z.ZodEnum<{
167
+ low: "low";
168
+ medium: "medium";
169
+ high: "high";
170
+ }>;
171
+ description: z.ZodString;
172
+ affectedScenarios: z.ZodArray<z.ZodString>;
173
+ recommendation: z.ZodOptional<z.ZodString>;
174
+ }, z.core.$strip>;
175
+ export type RunAnalysisFinding = z.infer<typeof RunAnalysisFindingSchema>;
176
+ /**
177
+ * LLM-generated analysis of a completed eval run.
178
+ */
179
+ export declare const RunAnalysisSchema: z.ZodObject<{
180
+ generatedAt: z.ZodString;
181
181
  summary: z.ZodString;
182
- details: z.ZodString;
183
- rootCause: z.ZodString;
184
- suggestedFix: z.ZodString;
185
- relatedAssertions: z.ZodArray<z.ZodString>;
186
- codeSnippet: z.ZodOptional<z.ZodString>;
187
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
188
- patternId: z.ZodOptional<z.ZodString>;
189
- diff: z.ZodOptional<z.ZodObject<{
190
- path: z.ZodString;
191
- expected: z.ZodString;
192
- actual: z.ZodString;
193
- diffLines: z.ZodArray<z.ZodObject<{
194
- type: z.ZodEnum<{
195
- added: "added";
196
- removed: "removed";
197
- unchanged: "unchanged";
198
- }>;
199
- content: z.ZodString;
200
- lineNumber: z.ZodNumber;
201
- }, z.core.$strip>>;
202
- renamedFrom: z.ZodOptional<z.ZodString>;
203
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
204
- }, z.core.$strip>>;
205
- executionTrace: z.ZodOptional<z.ZodObject<{
206
- commands: z.ZodArray<z.ZodObject<{
207
- command: z.ZodString;
208
- exitCode: z.ZodNumber;
209
- output: z.ZodOptional<z.ZodString>;
210
- duration: z.ZodNumber;
211
- }, z.core.$strip>>;
212
- filesModified: z.ZodArray<z.ZodObject<{
213
- path: z.ZodString;
214
- action: z.ZodEnum<{
215
- deleted: "deleted";
216
- created: "created";
217
- modified: "modified";
218
- }>;
219
- }, z.core.$strip>>;
220
- apiCalls: z.ZodArray<z.ZodObject<{
221
- endpoint: z.ZodString;
222
- tokensUsed: z.ZodNumber;
223
- duration: z.ZodNumber;
224
- }, z.core.$strip>>;
225
- totalDuration: z.ZodNumber;
182
+ findings: z.ZodArray<z.ZodObject<{
183
+ category: z.ZodEnum<{
184
+ failure_pattern: "failure_pattern";
185
+ cost_waste: "cost_waste";
186
+ flakiness: "flakiness";
187
+ inefficiency: "inefficiency";
188
+ positive: "positive";
189
+ }>;
190
+ severity: z.ZodEnum<{
191
+ low: "low";
192
+ medium: "medium";
193
+ high: "high";
194
+ }>;
195
+ description: z.ZodString;
196
+ affectedScenarios: z.ZodArray<z.ZodString>;
197
+ recommendation: z.ZodOptional<z.ZodString>;
226
198
  }, z.core.$strip>>;
227
199
  }, z.core.$strip>;
228
- export type FailureAnalysis = z.infer<typeof FailureAnalysisSchema>;
200
+ export type RunAnalysis = z.infer<typeof RunAnalysisSchema>;
229
201
  /**
230
202
  * Evaluation run schema.
231
203
  *
@@ -430,56 +402,6 @@ export declare const EvalRunSchema: z.ZodObject<{
430
402
  avgDuration: z.ZodNumber;
431
403
  totalDuration: z.ZodNumber;
432
404
  }, z.core.$strip>;
433
- failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
434
- category: z.ZodEnum<typeof FailureCategory>;
435
- severity: z.ZodEnum<typeof FailureSeverity>;
436
- summary: z.ZodString;
437
- details: z.ZodString;
438
- rootCause: z.ZodString;
439
- suggestedFix: z.ZodString;
440
- relatedAssertions: z.ZodArray<z.ZodString>;
441
- codeSnippet: z.ZodOptional<z.ZodString>;
442
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
443
- patternId: z.ZodOptional<z.ZodString>;
444
- diff: z.ZodOptional<z.ZodObject<{
445
- path: z.ZodString;
446
- expected: z.ZodString;
447
- actual: z.ZodString;
448
- diffLines: z.ZodArray<z.ZodObject<{
449
- type: z.ZodEnum<{
450
- added: "added";
451
- removed: "removed";
452
- unchanged: "unchanged";
453
- }>;
454
- content: z.ZodString;
455
- lineNumber: z.ZodNumber;
456
- }, z.core.$strip>>;
457
- renamedFrom: z.ZodOptional<z.ZodString>;
458
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
459
- }, z.core.$strip>>;
460
- executionTrace: z.ZodOptional<z.ZodObject<{
461
- commands: z.ZodArray<z.ZodObject<{
462
- command: z.ZodString;
463
- exitCode: z.ZodNumber;
464
- output: z.ZodOptional<z.ZodString>;
465
- duration: z.ZodNumber;
466
- }, z.core.$strip>>;
467
- filesModified: z.ZodArray<z.ZodObject<{
468
- path: z.ZodString;
469
- action: z.ZodEnum<{
470
- deleted: "deleted";
471
- created: "created";
472
- modified: "modified";
473
- }>;
474
- }, z.core.$strip>>;
475
- apiCalls: z.ZodArray<z.ZodObject<{
476
- endpoint: z.ZodString;
477
- tokensUsed: z.ZodNumber;
478
- duration: z.ZodNumber;
479
- }, z.core.$strip>>;
480
- totalDuration: z.ZodNumber;
481
- }, z.core.$strip>>;
482
- }, z.core.$strip>>>;
483
405
  llmTraceSummary: z.ZodOptional<z.ZodObject<{
484
406
  totalSteps: z.ZodNumber;
485
407
  totalTurns: z.ZodOptional<z.ZodNumber>;
@@ -558,6 +480,29 @@ export declare const EvalRunSchema: z.ZodObject<{
558
480
  maxTurns: z.ZodPipe<z.ZodTransform<{} | undefined, unknown>, z.ZodOptional<z.ZodNumber>>;
559
481
  }, z.core.$strip>>;
560
482
  }, z.core.$strip>>;
483
+ comparisonGroupId: z.ZodOptional<z.ZodString>;
484
+ comparisonLabel: z.ZodOptional<z.ZodString>;
485
+ runAnalysis: z.ZodOptional<z.ZodObject<{
486
+ generatedAt: z.ZodString;
487
+ summary: z.ZodString;
488
+ findings: z.ZodArray<z.ZodObject<{
489
+ category: z.ZodEnum<{
490
+ failure_pattern: "failure_pattern";
491
+ cost_waste: "cost_waste";
492
+ flakiness: "flakiness";
493
+ inefficiency: "inefficiency";
494
+ positive: "positive";
495
+ }>;
496
+ severity: z.ZodEnum<{
497
+ low: "low";
498
+ medium: "medium";
499
+ high: "high";
500
+ }>;
501
+ description: z.ZodString;
502
+ affectedScenarios: z.ZodArray<z.ZodString>;
503
+ recommendation: z.ZodOptional<z.ZodString>;
504
+ }, z.core.$strip>>;
505
+ }, z.core.$strip>>;
561
506
  }, z.core.$strip>;
562
507
  export type EvalRun = z.infer<typeof EvalRunSchema>;
563
508
  /**
@@ -580,56 +525,6 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
580
525
  ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
581
526
  tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
582
527
  presetId: z.ZodOptional<z.ZodString>;
583
- failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
584
- category: z.ZodEnum<typeof FailureCategory>;
585
- severity: z.ZodEnum<typeof FailureSeverity>;
586
- summary: z.ZodString;
587
- details: z.ZodString;
588
- rootCause: z.ZodString;
589
- suggestedFix: z.ZodString;
590
- relatedAssertions: z.ZodArray<z.ZodString>;
591
- codeSnippet: z.ZodOptional<z.ZodString>;
592
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
593
- patternId: z.ZodOptional<z.ZodString>;
594
- diff: z.ZodOptional<z.ZodObject<{
595
- path: z.ZodString;
596
- expected: z.ZodString;
597
- actual: z.ZodString;
598
- diffLines: z.ZodArray<z.ZodObject<{
599
- type: z.ZodEnum<{
600
- added: "added";
601
- removed: "removed";
602
- unchanged: "unchanged";
603
- }>;
604
- content: z.ZodString;
605
- lineNumber: z.ZodNumber;
606
- }, z.core.$strip>>;
607
- renamedFrom: z.ZodOptional<z.ZodString>;
608
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
609
- }, z.core.$strip>>;
610
- executionTrace: z.ZodOptional<z.ZodObject<{
611
- commands: z.ZodArray<z.ZodObject<{
612
- command: z.ZodString;
613
- exitCode: z.ZodNumber;
614
- output: z.ZodOptional<z.ZodString>;
615
- duration: z.ZodNumber;
616
- }, z.core.$strip>>;
617
- filesModified: z.ZodArray<z.ZodObject<{
618
- path: z.ZodString;
619
- action: z.ZodEnum<{
620
- deleted: "deleted";
621
- created: "created";
622
- modified: "modified";
623
- }>;
624
- }, z.core.$strip>>;
625
- apiCalls: z.ZodArray<z.ZodObject<{
626
- endpoint: z.ZodString;
627
- tokensUsed: z.ZodNumber;
628
- duration: z.ZodNumber;
629
- }, z.core.$strip>>;
630
- totalDuration: z.ZodNumber;
631
- }, z.core.$strip>>;
632
- }, z.core.$strip>>>;
633
528
  llmTraceSummary: z.ZodOptional<z.ZodObject<{
634
529
  totalSteps: z.ZodNumber;
635
530
  totalTurns: z.ZodOptional<z.ZodNumber>;
@@ -685,6 +580,29 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
685
580
  jobError: z.ZodOptional<z.ZodString>;
686
581
  jobStatusCheckedAt: z.ZodOptional<z.ZodString>;
687
582
  runsPerScenario: z.ZodOptional<z.ZodNumber>;
583
+ comparisonGroupId: z.ZodOptional<z.ZodString>;
584
+ comparisonLabel: z.ZodOptional<z.ZodString>;
585
+ runAnalysis: z.ZodOptional<z.ZodObject<{
586
+ generatedAt: z.ZodString;
587
+ summary: z.ZodString;
588
+ findings: z.ZodArray<z.ZodObject<{
589
+ category: z.ZodEnum<{
590
+ failure_pattern: "failure_pattern";
591
+ cost_waste: "cost_waste";
592
+ flakiness: "flakiness";
593
+ inefficiency: "inefficiency";
594
+ positive: "positive";
595
+ }>;
596
+ severity: z.ZodEnum<{
597
+ low: "low";
598
+ medium: "medium";
599
+ high: "high";
600
+ }>;
601
+ description: z.ZodString;
602
+ affectedScenarios: z.ZodArray<z.ZodString>;
603
+ recommendation: z.ZodOptional<z.ZodString>;
604
+ }, z.core.$strip>>;
605
+ }, z.core.$strip>>;
688
606
  scenarioIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
689
607
  }, z.core.$strip>;
690
608
  export type CreateEvalRunInput = z.infer<typeof CreateEvalRunInputSchema>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wix/evalforge-types",
3
- "version": "0.68.0",
3
+ "version": "0.70.0",
4
4
  "description": "Unified types for EvalForge agent evaluation system",
5
5
  "files": [
6
6
  "build"
@@ -46,5 +46,5 @@
46
46
  "artifactId": "evalforge-types"
47
47
  }
48
48
  },
49
- "falconPackageHash": "e4412ad763ca9d048b11048eb13e0ac4f7c76f33cdc5573edf47ff05"
49
+ "falconPackageHash": "63ddf4c9491b8c912b2320516eb657a97b85d1ee43a35d9ea94553f7"
50
50
  }