@wix/evalforge-types 0.69.0 → 0.71.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/build/index.js +23 -38
  2. package/build/index.js.map +3 -3
  3. package/build/index.mjs +21 -35
  4. package/build/index.mjs.map +3 -3
  5. package/build/types/evaluation/eval-run.d.ts +83 -169
  6. package/package.json +2 -2
package/build/types/evaluation/eval-run.d.ts CHANGED
@@ -31,26 +31,6 @@ export declare const TriggerSchema: z.ZodObject<{
31
31
  type: z.ZodEnum<typeof TriggerType>;
32
32
  }, z.core.$strip>;
33
33
  export type Trigger = z.infer<typeof TriggerSchema>;
34
- /**
35
- * Failure category enum.
36
- */
37
- export declare enum FailureCategory {
38
- MISSING_FILE = "missing_file",
39
- WRONG_CONTENT = "wrong_content",
40
- BUILD_ERROR = "build_error",
41
- TEST_FAILURE = "test_failure",
42
- RUNTIME_ERROR = "runtime_error",
43
- PERFORMANCE = "performance"
44
- }
45
- /**
46
- * Failure severity enum.
47
- */
48
- export declare enum FailureSeverity {
49
- CRITICAL = "critical",
50
- HIGH = "high",
51
- MEDIUM = "medium",
52
- LOW = "low"
53
- }
54
34
  /**
55
35
  * Diff line type schema.
56
36
  */
@@ -173,59 +153,51 @@ export declare const ExecutionTraceSchema: z.ZodObject<{
173
153
  }, z.core.$strip>;
174
154
  export type ExecutionTrace = z.infer<typeof ExecutionTraceSchema>;
175
155
  /**
176
- * Failure analysis schema.
156
+ * A single finding from the LLM-generated run analysis.
177
157
  */
178
- export declare const FailureAnalysisSchema: z.ZodObject<{
179
- category: z.ZodEnum<typeof FailureCategory>;
180
- severity: z.ZodEnum<typeof FailureSeverity>;
158
+ export declare const RunAnalysisFindingSchema: z.ZodObject<{
159
+ category: z.ZodEnum<{
160
+ failure_pattern: "failure_pattern";
161
+ cost_waste: "cost_waste";
162
+ flakiness: "flakiness";
163
+ inefficiency: "inefficiency";
164
+ positive: "positive";
165
+ }>;
166
+ severity: z.ZodEnum<{
167
+ low: "low";
168
+ medium: "medium";
169
+ high: "high";
170
+ }>;
171
+ description: z.ZodString;
172
+ affectedScenarios: z.ZodArray<z.ZodString>;
173
+ recommendation: z.ZodOptional<z.ZodString>;
174
+ }, z.core.$strip>;
175
+ export type RunAnalysisFinding = z.infer<typeof RunAnalysisFindingSchema>;
176
+ /**
177
+ * LLM-generated analysis of a completed eval run.
178
+ */
179
+ export declare const RunAnalysisSchema: z.ZodObject<{
180
+ generatedAt: z.ZodString;
181
181
  summary: z.ZodString;
182
- details: z.ZodString;
183
- rootCause: z.ZodString;
184
- suggestedFix: z.ZodString;
185
- relatedAssertions: z.ZodArray<z.ZodString>;
186
- codeSnippet: z.ZodOptional<z.ZodString>;
187
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
188
- patternId: z.ZodOptional<z.ZodString>;
189
- diff: z.ZodOptional<z.ZodObject<{
190
- path: z.ZodString;
191
- expected: z.ZodString;
192
- actual: z.ZodString;
193
- diffLines: z.ZodArray<z.ZodObject<{
194
- type: z.ZodEnum<{
195
- added: "added";
196
- removed: "removed";
197
- unchanged: "unchanged";
198
- }>;
199
- content: z.ZodString;
200
- lineNumber: z.ZodNumber;
201
- }, z.core.$strip>>;
202
- renamedFrom: z.ZodOptional<z.ZodString>;
203
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
204
- }, z.core.$strip>>;
205
- executionTrace: z.ZodOptional<z.ZodObject<{
206
- commands: z.ZodArray<z.ZodObject<{
207
- command: z.ZodString;
208
- exitCode: z.ZodNumber;
209
- output: z.ZodOptional<z.ZodString>;
210
- duration: z.ZodNumber;
211
- }, z.core.$strip>>;
212
- filesModified: z.ZodArray<z.ZodObject<{
213
- path: z.ZodString;
214
- action: z.ZodEnum<{
215
- deleted: "deleted";
216
- created: "created";
217
- modified: "modified";
218
- }>;
219
- }, z.core.$strip>>;
220
- apiCalls: z.ZodArray<z.ZodObject<{
221
- endpoint: z.ZodString;
222
- tokensUsed: z.ZodNumber;
223
- duration: z.ZodNumber;
224
- }, z.core.$strip>>;
225
- totalDuration: z.ZodNumber;
182
+ findings: z.ZodArray<z.ZodObject<{
183
+ category: z.ZodEnum<{
184
+ failure_pattern: "failure_pattern";
185
+ cost_waste: "cost_waste";
186
+ flakiness: "flakiness";
187
+ inefficiency: "inefficiency";
188
+ positive: "positive";
189
+ }>;
190
+ severity: z.ZodEnum<{
191
+ low: "low";
192
+ medium: "medium";
193
+ high: "high";
194
+ }>;
195
+ description: z.ZodString;
196
+ affectedScenarios: z.ZodArray<z.ZodString>;
197
+ recommendation: z.ZodOptional<z.ZodString>;
226
198
  }, z.core.$strip>>;
227
199
  }, z.core.$strip>;
228
- export type FailureAnalysis = z.infer<typeof FailureAnalysisSchema>;
200
+ export type RunAnalysis = z.infer<typeof RunAnalysisSchema>;
229
201
  /**
230
202
  * Evaluation run schema.
231
203
  *
@@ -430,56 +402,6 @@ export declare const EvalRunSchema: z.ZodObject<{
430
402
  avgDuration: z.ZodNumber;
431
403
  totalDuration: z.ZodNumber;
432
404
  }, z.core.$strip>;
433
- failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
434
- category: z.ZodEnum<typeof FailureCategory>;
435
- severity: z.ZodEnum<typeof FailureSeverity>;
436
- summary: z.ZodString;
437
- details: z.ZodString;
438
- rootCause: z.ZodString;
439
- suggestedFix: z.ZodString;
440
- relatedAssertions: z.ZodArray<z.ZodString>;
441
- codeSnippet: z.ZodOptional<z.ZodString>;
442
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
443
- patternId: z.ZodOptional<z.ZodString>;
444
- diff: z.ZodOptional<z.ZodObject<{
445
- path: z.ZodString;
446
- expected: z.ZodString;
447
- actual: z.ZodString;
448
- diffLines: z.ZodArray<z.ZodObject<{
449
- type: z.ZodEnum<{
450
- added: "added";
451
- removed: "removed";
452
- unchanged: "unchanged";
453
- }>;
454
- content: z.ZodString;
455
- lineNumber: z.ZodNumber;
456
- }, z.core.$strip>>;
457
- renamedFrom: z.ZodOptional<z.ZodString>;
458
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
459
- }, z.core.$strip>>;
460
- executionTrace: z.ZodOptional<z.ZodObject<{
461
- commands: z.ZodArray<z.ZodObject<{
462
- command: z.ZodString;
463
- exitCode: z.ZodNumber;
464
- output: z.ZodOptional<z.ZodString>;
465
- duration: z.ZodNumber;
466
- }, z.core.$strip>>;
467
- filesModified: z.ZodArray<z.ZodObject<{
468
- path: z.ZodString;
469
- action: z.ZodEnum<{
470
- deleted: "deleted";
471
- created: "created";
472
- modified: "modified";
473
- }>;
474
- }, z.core.$strip>>;
475
- apiCalls: z.ZodArray<z.ZodObject<{
476
- endpoint: z.ZodString;
477
- tokensUsed: z.ZodNumber;
478
- duration: z.ZodNumber;
479
- }, z.core.$strip>>;
480
- totalDuration: z.ZodNumber;
481
- }, z.core.$strip>>;
482
- }, z.core.$strip>>>;
483
405
  llmTraceSummary: z.ZodOptional<z.ZodObject<{
484
406
  totalSteps: z.ZodNumber;
485
407
  totalTurns: z.ZodOptional<z.ZodNumber>;
@@ -560,6 +482,27 @@ export declare const EvalRunSchema: z.ZodObject<{
560
482
  }, z.core.$strip>>;
561
483
  comparisonGroupId: z.ZodOptional<z.ZodString>;
562
484
  comparisonLabel: z.ZodOptional<z.ZodString>;
485
+ runAnalysis: z.ZodOptional<z.ZodObject<{
486
+ generatedAt: z.ZodString;
487
+ summary: z.ZodString;
488
+ findings: z.ZodArray<z.ZodObject<{
489
+ category: z.ZodEnum<{
490
+ failure_pattern: "failure_pattern";
491
+ cost_waste: "cost_waste";
492
+ flakiness: "flakiness";
493
+ inefficiency: "inefficiency";
494
+ positive: "positive";
495
+ }>;
496
+ severity: z.ZodEnum<{
497
+ low: "low";
498
+ medium: "medium";
499
+ high: "high";
500
+ }>;
501
+ description: z.ZodString;
502
+ affectedScenarios: z.ZodArray<z.ZodString>;
503
+ recommendation: z.ZodOptional<z.ZodString>;
504
+ }, z.core.$strip>>;
505
+ }, z.core.$strip>>;
563
506
  }, z.core.$strip>;
564
507
  export type EvalRun = z.infer<typeof EvalRunSchema>;
565
508
  /**
@@ -582,56 +525,6 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
582
525
  ruleIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
583
526
  tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
584
527
  presetId: z.ZodOptional<z.ZodString>;
585
- failureAnalyses: z.ZodOptional<z.ZodArray<z.ZodObject<{
586
- category: z.ZodEnum<typeof FailureCategory>;
587
- severity: z.ZodEnum<typeof FailureSeverity>;
588
- summary: z.ZodString;
589
- details: z.ZodString;
590
- rootCause: z.ZodString;
591
- suggestedFix: z.ZodString;
592
- relatedAssertions: z.ZodArray<z.ZodString>;
593
- codeSnippet: z.ZodOptional<z.ZodString>;
594
- similarIssues: z.ZodOptional<z.ZodArray<z.ZodString>>;
595
- patternId: z.ZodOptional<z.ZodString>;
596
- diff: z.ZodOptional<z.ZodObject<{
597
- path: z.ZodString;
598
- expected: z.ZodString;
599
- actual: z.ZodString;
600
- diffLines: z.ZodArray<z.ZodObject<{
601
- type: z.ZodEnum<{
602
- added: "added";
603
- removed: "removed";
604
- unchanged: "unchanged";
605
- }>;
606
- content: z.ZodString;
607
- lineNumber: z.ZodNumber;
608
- }, z.core.$strip>>;
609
- renamedFrom: z.ZodOptional<z.ZodString>;
610
- isInfrastructure: z.ZodOptional<z.ZodBoolean>;
611
- }, z.core.$strip>>;
612
- executionTrace: z.ZodOptional<z.ZodObject<{
613
- commands: z.ZodArray<z.ZodObject<{
614
- command: z.ZodString;
615
- exitCode: z.ZodNumber;
616
- output: z.ZodOptional<z.ZodString>;
617
- duration: z.ZodNumber;
618
- }, z.core.$strip>>;
619
- filesModified: z.ZodArray<z.ZodObject<{
620
- path: z.ZodString;
621
- action: z.ZodEnum<{
622
- deleted: "deleted";
623
- created: "created";
624
- modified: "modified";
625
- }>;
626
- }, z.core.$strip>>;
627
- apiCalls: z.ZodArray<z.ZodObject<{
628
- endpoint: z.ZodString;
629
- tokensUsed: z.ZodNumber;
630
- duration: z.ZodNumber;
631
- }, z.core.$strip>>;
632
- totalDuration: z.ZodNumber;
633
- }, z.core.$strip>>;
634
- }, z.core.$strip>>>;
635
528
  llmTraceSummary: z.ZodOptional<z.ZodObject<{
636
529
  totalSteps: z.ZodNumber;
637
530
  totalTurns: z.ZodOptional<z.ZodNumber>;
@@ -689,6 +582,27 @@ export declare const CreateEvalRunInputSchema: z.ZodObject<{
689
582
  runsPerScenario: z.ZodOptional<z.ZodNumber>;
690
583
  comparisonGroupId: z.ZodOptional<z.ZodString>;
691
584
  comparisonLabel: z.ZodOptional<z.ZodString>;
585
+ runAnalysis: z.ZodOptional<z.ZodObject<{
586
+ generatedAt: z.ZodString;
587
+ summary: z.ZodString;
588
+ findings: z.ZodArray<z.ZodObject<{
589
+ category: z.ZodEnum<{
590
+ failure_pattern: "failure_pattern";
591
+ cost_waste: "cost_waste";
592
+ flakiness: "flakiness";
593
+ inefficiency: "inefficiency";
594
+ positive: "positive";
595
+ }>;
596
+ severity: z.ZodEnum<{
597
+ low: "low";
598
+ medium: "medium";
599
+ high: "high";
600
+ }>;
601
+ description: z.ZodString;
602
+ affectedScenarios: z.ZodArray<z.ZodString>;
603
+ recommendation: z.ZodOptional<z.ZodString>;
604
+ }, z.core.$strip>>;
605
+ }, z.core.$strip>>;
692
606
  scenarioIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
693
607
  }, z.core.$strip>;
694
608
  export type CreateEvalRunInput = z.infer<typeof CreateEvalRunInputSchema>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wix/evalforge-types",
3
- "version": "0.69.0",
3
+ "version": "0.71.0",
4
4
  "description": "Unified types for EvalForge agent evaluation system",
5
5
  "files": [
6
6
  "build"
@@ -46,5 +46,5 @@
46
46
  "artifactId": "evalforge-types"
47
47
  }
48
48
  },
49
- "falconPackageHash": "ed54412ea000a3fda6eb44b951270bcf243b0c3dee6bca814014183c"
49
+ "falconPackageHash": "6b1bbe2ce0a08ee2b4b2eba8f75ce3a9a70f4985c78c8990999ec4a2"
50
50
  }