@sentry/warden 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/CHANGELOG.md +246 -0
  2. package/README.md +2 -1
  3. package/coverage/lcov-report/base.css +224 -0
  4. package/coverage/lcov-report/block-navigation.js +87 -0
  5. package/coverage/lcov-report/favicon.png +0 -0
  6. package/coverage/lcov-report/index.html +446 -0
  7. package/coverage/lcov-report/prettify.css +1 -0
  8. package/coverage/lcov-report/prettify.js +2 -0
  9. package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
  10. package/coverage/lcov-report/sorter.js +210 -0
  11. package/coverage/lcov-report/src/action/checks/index.html +116 -0
  12. package/coverage/lcov-report/src/action/checks/manager.ts.html +499 -0
  13. package/coverage/lcov-report/src/action/error-reporting.ts.html +181 -0
  14. package/coverage/lcov-report/src/action/fix-evaluation/github.ts.html +508 -0
  15. package/coverage/lcov-report/src/action/fix-evaluation/index.html +161 -0
  16. package/coverage/lcov-report/src/action/fix-evaluation/index.ts.html +1243 -0
  17. package/coverage/lcov-report/src/action/fix-evaluation/judge.ts.html +817 -0
  18. package/coverage/lcov-report/src/action/fix-evaluation/types.ts.html +274 -0
  19. package/coverage/lcov-report/src/action/index.html +161 -0
  20. package/coverage/lcov-report/src/action/inputs.ts.html +529 -0
  21. package/coverage/lcov-report/src/action/pi-ncc-compat.ts.html +292 -0
  22. package/coverage/lcov-report/src/action/review/coordination.ts.html +208 -0
  23. package/coverage/lcov-report/src/action/review/index.html +131 -0
  24. package/coverage/lcov-report/src/action/review/poster.ts.html +1051 -0
  25. package/coverage/lcov-report/src/action/review-state.ts.html +280 -0
  26. package/coverage/lcov-report/src/action/triggers/executor.ts.html +856 -0
  27. package/coverage/lcov-report/src/action/triggers/index.html +116 -0
  28. package/coverage/lcov-report/src/action/workflow/base.ts.html +1135 -0
  29. package/coverage/lcov-report/src/action/workflow/index.html +146 -0
  30. package/coverage/lcov-report/src/action/workflow/pr-workflow.ts.html +2713 -0
  31. package/coverage/lcov-report/src/action/workflow/schedule.ts.html +1027 -0
  32. package/coverage/lcov-report/src/cli/args.ts.html +1657 -0
  33. package/coverage/lcov-report/src/cli/commands/add.ts.html +1567 -0
  34. package/coverage/lcov-report/src/cli/commands/build.ts.html +1618 -0
  35. package/coverage/lcov-report/src/cli/commands/index.html +191 -0
  36. package/coverage/lcov-report/src/cli/commands/init.ts.html +1357 -0
  37. package/coverage/lcov-report/src/cli/commands/runs.ts.html +2560 -0
  38. package/coverage/lcov-report/src/cli/commands/setup-app/browser.ts.html +214 -0
  39. package/coverage/lcov-report/src/cli/commands/setup-app/credentials.ts.html +220 -0
  40. package/coverage/lcov-report/src/cli/commands/setup-app/index.html +161 -0
  41. package/coverage/lcov-report/src/cli/commands/setup-app/manifest.ts.html +235 -0
  42. package/coverage/lcov-report/src/cli/commands/setup-app/server.ts.html +601 -0
  43. package/coverage/lcov-report/src/cli/commands/setup-app.ts.html +571 -0
  44. package/coverage/lcov-report/src/cli/commands/sync.ts.html +475 -0
  45. package/coverage/lcov-report/src/cli/context.ts.html +532 -0
  46. package/coverage/lcov-report/src/cli/diff-apply.ts.html +334 -0
  47. package/coverage/lcov-report/src/cli/files.ts.html +904 -0
  48. package/coverage/lcov-report/src/cli/fix.ts.html +1060 -0
  49. package/coverage/lcov-report/src/cli/git.ts.html +1201 -0
  50. package/coverage/lcov-report/src/cli/help.ts.html +1696 -0
  51. package/coverage/lcov-report/src/cli/index.html +281 -0
  52. package/coverage/lcov-report/src/cli/input.ts.html +397 -0
  53. package/coverage/lcov-report/src/cli/log-cleanup.ts.html +337 -0
  54. package/coverage/lcov-report/src/cli/main.ts.html +5107 -0
  55. package/coverage/lcov-report/src/cli/output/box.ts.html +790 -0
  56. package/coverage/lcov-report/src/cli/output/formatters.ts.html +1099 -0
  57. package/coverage/lcov-report/src/cli/output/icons.ts.html +142 -0
  58. package/coverage/lcov-report/src/cli/output/index.html +281 -0
  59. package/coverage/lcov-report/src/cli/output/index.ts.html +295 -0
  60. package/coverage/lcov-report/src/cli/output/ink-runner.tsx.html +1597 -0
  61. package/coverage/lcov-report/src/cli/output/jsonl-schema-gen.ts.html +355 -0
  62. package/coverage/lcov-report/src/cli/output/jsonl.ts.html +2137 -0
  63. package/coverage/lcov-report/src/cli/output/live-status.tsx.html +916 -0
  64. package/coverage/lcov-report/src/cli/output/reporter.ts.html +1630 -0
  65. package/coverage/lcov-report/src/cli/output/tasks.ts.html +3193 -0
  66. package/coverage/lcov-report/src/cli/output/tty.ts.html +310 -0
  67. package/coverage/lcov-report/src/cli/output/verbosity.ts.html +181 -0
  68. package/coverage/lcov-report/src/cli/signals.ts.html +199 -0
  69. package/coverage/lcov-report/src/cli/terminal.ts.html +1060 -0
  70. package/coverage/lcov-report/src/config/index.html +161 -0
  71. package/coverage/lcov-report/src/config/index.ts.html +94 -0
  72. package/coverage/lcov-report/src/config/loader.ts.html +1753 -0
  73. package/coverage/lcov-report/src/config/schema.ts.html +910 -0
  74. package/coverage/lcov-report/src/config/writer.ts.html +538 -0
  75. package/coverage/lcov-report/src/diff/apply.ts.html +241 -0
  76. package/coverage/lcov-report/src/diff/classify.ts.html +361 -0
  77. package/coverage/lcov-report/src/diff/coalesce.ts.html +1123 -0
  78. package/coverage/lcov-report/src/diff/context.ts.html +952 -0
  79. package/coverage/lcov-report/src/diff/index.html +191 -0
  80. package/coverage/lcov-report/src/diff/index.ts.html +100 -0
  81. package/coverage/lcov-report/src/diff/parser.ts.html +532 -0
  82. package/coverage/lcov-report/src/evals/harness.ts.html +700 -0
  83. package/coverage/lcov-report/src/evals/index.html +221 -0
  84. package/coverage/lcov-report/src/evals/index.ts.html +955 -0
  85. package/coverage/lcov-report/src/evals/names.ts.html +121 -0
  86. package/coverage/lcov-report/src/evals/runner.ts.html +664 -0
  87. package/coverage/lcov-report/src/evals/scaffold.ts.html +970 -0
  88. package/coverage/lcov-report/src/evals/types.ts.html +661 -0
  89. package/coverage/lcov-report/src/evals/usage.ts.html +178 -0
  90. package/coverage/lcov-report/src/evals/verify.ts.html +1054 -0
  91. package/coverage/lcov-report/src/event/context.ts.html +490 -0
  92. package/coverage/lcov-report/src/event/index.html +116 -0
  93. package/coverage/lcov-report/src/index.html +116 -0
  94. package/coverage/lcov-report/src/output/dedup.ts.html +2950 -0
  95. package/coverage/lcov-report/src/output/github-checks.ts.html +1579 -0
  96. package/coverage/lcov-report/src/output/index.html +161 -0
  97. package/coverage/lcov-report/src/output/renderer.ts.html +988 -0
  98. package/coverage/lcov-report/src/output/stale.ts.html +640 -0
  99. package/coverage/lcov-report/src/sdk/analyze.ts.html +3010 -0
  100. package/coverage/lcov-report/src/sdk/auth.ts.html +217 -0
  101. package/coverage/lcov-report/src/sdk/circuit-breaker.ts.html +304 -0
  102. package/coverage/lcov-report/src/sdk/errors.ts.html +658 -0
  103. package/coverage/lcov-report/src/sdk/extract.ts.html +1810 -0
  104. package/coverage/lcov-report/src/sdk/fix-quality.ts.html +841 -0
  105. package/coverage/lcov-report/src/sdk/haiku.ts.html +1456 -0
  106. package/coverage/lcov-report/src/sdk/index.html +416 -0
  107. package/coverage/lcov-report/src/sdk/json-output.ts.html +508 -0
  108. package/coverage/lcov-report/src/sdk/model-pricing.json.html +499 -0
  109. package/coverage/lcov-report/src/sdk/otel.ts.html +433 -0
  110. package/coverage/lcov-report/src/sdk/post-process.ts.html +412 -0
  111. package/coverage/lcov-report/src/sdk/prepare.ts.html +400 -0
  112. package/coverage/lcov-report/src/sdk/pricing.ts.html +778 -0
  113. package/coverage/lcov-report/src/sdk/prompt-sections.ts.html +595 -0
  114. package/coverage/lcov-report/src/sdk/prompt.ts.html +451 -0
  115. package/coverage/lcov-report/src/sdk/report-files.ts.html +166 -0
  116. package/coverage/lcov-report/src/sdk/retry.ts.html +202 -0
  117. package/coverage/lcov-report/src/sdk/runner.ts.html +400 -0
  118. package/coverage/lcov-report/src/sdk/runtimes/claude.ts.html +1693 -0
  119. package/coverage/lcov-report/src/sdk/runtimes/index.html +176 -0
  120. package/coverage/lcov-report/src/sdk/runtimes/index.ts.html +244 -0
  121. package/coverage/lcov-report/src/sdk/runtimes/model-selectors.ts.html +328 -0
  122. package/coverage/lcov-report/src/sdk/runtimes/pi.ts.html +2104 -0
  123. package/coverage/lcov-report/src/sdk/runtimes/types.ts.html +508 -0
  124. package/coverage/lcov-report/src/sdk/types.ts.html +730 -0
  125. package/coverage/lcov-report/src/sdk/usage.ts.html +502 -0
  126. package/coverage/lcov-report/src/sdk/verify.ts.html +1027 -0
  127. package/coverage/lcov-report/src/sentry.ts.html +994 -0
  128. package/coverage/lcov-report/src/skill-builder/agentic.ts.html +1063 -0
  129. package/coverage/lcov-report/src/skill-builder/authoring-provider.ts.html +334 -0
  130. package/coverage/lcov-report/src/skill-builder/definition.ts.html +748 -0
  131. package/coverage/lcov-report/src/skill-builder/index.html +236 -0
  132. package/coverage/lcov-report/src/skill-builder/outline-contract.ts.html +400 -0
  133. package/coverage/lcov-report/src/skill-builder/outline-state.ts.html +382 -0
  134. package/coverage/lcov-report/src/skill-builder/outline.ts.html +1402 -0
  135. package/coverage/lcov-report/src/skill-builder/skill-contract.ts.html +382 -0
  136. package/coverage/lcov-report/src/skill-builder/skill-prompts.ts.html +1180 -0
  137. package/coverage/lcov-report/src/skill-builder/skill.ts.html +2398 -0
  138. package/coverage/lcov-report/src/skills/index.html +131 -0
  139. package/coverage/lcov-report/src/skills/loader.ts.html +1693 -0
  140. package/coverage/lcov-report/src/skills/remote.ts.html +1909 -0
  141. package/coverage/lcov-report/src/triggers/index.html +116 -0
  142. package/coverage/lcov-report/src/triggers/matcher.ts.html +841 -0
  143. package/coverage/lcov-report/src/types/index.html +116 -0
  144. package/coverage/lcov-report/src/types/index.ts.html +1240 -0
  145. package/coverage/lcov-report/src/utils/async.ts.html +331 -0
  146. package/coverage/lcov-report/src/utils/exec.ts.html +496 -0
  147. package/coverage/lcov-report/src/utils/index.html +176 -0
  148. package/coverage/lcov-report/src/utils/index.ts.html +313 -0
  149. package/coverage/lcov-report/src/utils/path.ts.html +202 -0
  150. package/coverage/lcov-report/src/utils/version.ts.html +136 -0
  151. package/coverage/lcov.info +19134 -0
  152. package/dist/cli/commands/runs.d.ts +1 -1
  153. package/dist/cli/commands/runs.d.ts.map +1 -1
  154. package/dist/cli/commands/runs.js +1 -1
  155. package/dist/cli/commands/runs.js.map +1 -1
  156. package/dist/cli/files.d.ts.map +1 -1
  157. package/dist/cli/files.js +10 -11
  158. package/dist/cli/files.js.map +1 -1
  159. package/dist/sdk/verify.d.ts.map +1 -1
  160. package/dist/sdk/verify.js +3 -2
  161. package/dist/sdk/verify.js.map +1 -1
  162. package/dist/skill-builder/outline.d.ts.map +1 -1
  163. package/dist/skill-builder/outline.js.map +1 -1
  164. package/package.json +8 -3
  165. package/test-results.junit.xml +4329 -0
  166. package/dist/evals/index.d.ts +0 -22
  167. package/dist/evals/index.d.ts.map +0 -1
  168. package/dist/evals/index.js +0 -92
  169. package/dist/evals/index.js.map +0 -1
  170. package/dist/evals/judge.d.ts +0 -12
  171. package/dist/evals/judge.d.ts.map +0 -1
  172. package/dist/evals/judge.js +0 -171
  173. package/dist/evals/judge.js.map +0 -1
  174. package/dist/evals/runner.d.ts +0 -18
  175. package/dist/evals/runner.d.ts.map +0 -1
  176. package/dist/evals/runner.js +0 -149
  177. package/dist/evals/runner.js.map +0 -1
  178. package/dist/evals/setup.d.ts +0 -2
  179. package/dist/evals/setup.d.ts.map +0 -1
  180. package/dist/evals/setup.js +0 -22
  181. package/dist/evals/setup.js.map +0 -1
  182. package/dist/evals/types.d.ts +0 -160
  183. package/dist/evals/types.d.ts.map +0 -1
  184. package/dist/evals/types.js +0 -148
  185. package/dist/evals/types.js.map +0 -1
@@ -1,160 +0,0 @@
1
- import { z } from 'zod';
2
- import type { Finding, SkillReport, UsageStats } from '../types/index.js';
3
- /** Default model for eval skill execution and judging. */
4
- export declare const DEFAULT_EVAL_MODEL = "claude-sonnet-4-6";
5
- /**
6
- * A "should find" assertion in BDD style.
7
- */
8
- export declare const ShouldFindSchema: z.ZodObject<{
9
- finding: z.ZodString;
10
- severity: z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodEnum<{
11
- high: "high";
12
- low: "low";
13
- medium: "medium";
14
- }>>>;
15
- required: z.ZodDefault<z.ZodBoolean>;
16
- }, z.core.$strip>;
17
- export type ShouldFind = z.infer<typeof ShouldFindSchema>;
18
- /**
19
- * A single eval scenario within a YAML eval file.
20
- */
21
- export declare const EvalScenarioSchema: z.ZodObject<{
22
- name: z.ZodString;
23
- given: z.ZodString;
24
- files: z.ZodArray<z.ZodString>;
25
- model: z.ZodOptional<z.ZodString>;
26
- should_find: z.ZodArray<z.ZodObject<{
27
- finding: z.ZodString;
28
- severity: z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodEnum<{
29
- high: "high";
30
- low: "low";
31
- medium: "medium";
32
- }>>>;
33
- required: z.ZodDefault<z.ZodBoolean>;
34
- }, z.core.$strip>>;
35
- should_not_find: z.ZodDefault<z.ZodArray<z.ZodString>>;
36
- }, z.core.$strip>;
37
- export type EvalScenario = z.infer<typeof EvalScenarioSchema>;
38
- /**
39
- * Root schema for a YAML eval file. Each file defines a category of evals
40
- * sharing a common skill.
41
- *
42
- * Example YAML:
43
- * skill: skills/bug-detection.md
44
- * evals:
45
- * - name: null-property-access
46
- * given: code that accesses .find() result without null checking
47
- * files: [fixtures/null-property-access/handler.ts]
48
- * should_find:
49
- * - finding: null access on user.name
50
- * severity: high
51
- */
52
- export declare const EvalFileSchema: z.ZodObject<{
53
- skill: z.ZodString;
54
- model: z.ZodDefault<z.ZodString>;
55
- evals: z.ZodArray<z.ZodObject<{
56
- name: z.ZodString;
57
- given: z.ZodString;
58
- files: z.ZodArray<z.ZodString>;
59
- model: z.ZodOptional<z.ZodString>;
60
- should_find: z.ZodArray<z.ZodObject<{
61
- finding: z.ZodString;
62
- severity: z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodEnum<{
63
- high: "high";
64
- low: "low";
65
- medium: "medium";
66
- }>>>;
67
- required: z.ZodDefault<z.ZodBoolean>;
68
- }, z.core.$strip>>;
69
- should_not_find: z.ZodDefault<z.ZodArray<z.ZodString>>;
70
- }, z.core.$strip>>;
71
- }, z.core.$strip>;
72
- export type EvalFile = z.infer<typeof EvalFileSchema>;
73
- /**
74
- * Resolved eval metadata ready for execution. Combines the file-level
75
- * defaults with scenario-level overrides.
76
- */
77
- export interface EvalMeta {
78
- /** Scenario name (e.g., "null-property-access") */
79
- name: string;
80
- /** Category name from the YAML filename (e.g., "bug-detection") */
81
- category: string;
82
- /** What this eval tests (BDD "given") */
83
- given: string;
84
- /** Resolved absolute path to the skill file */
85
- skillPath: string;
86
- /** Resolved absolute paths to fixture files */
87
- filePaths: string[];
88
- /** Model to use for skill execution */
89
- model: string;
90
- /** What Warden should find */
91
- should_find: ShouldFind[];
92
- /** What Warden should NOT report */
93
- should_not_find: string[];
94
- }
95
- /**
96
- * Judge verdict for a single expectation.
97
- */
98
- export declare const ExpectationVerdictSchema: z.ZodObject<{
99
- met: z.ZodBoolean;
100
- matchedFindingIndex: z.ZodNullable<z.ZodNumber>;
101
- reasoning: z.ZodString;
102
- }, z.core.$strip>;
103
- export type ExpectationVerdict = z.infer<typeof ExpectationVerdictSchema>;
104
- /**
105
- * Judge verdict for a single anti-expectation.
106
- */
107
- export declare const AntiExpectationVerdictSchema: z.ZodObject<{
108
- violated: z.ZodBoolean;
109
- violatingFindingIndex: z.ZodNullable<z.ZodNumber>;
110
- reasoning: z.ZodString;
111
- }, z.core.$strip>;
112
- export type AntiExpectationVerdict = z.infer<typeof AntiExpectationVerdictSchema>;
113
- /**
114
- * Complete judge response for an eval.
115
- */
116
- export declare const JudgeResponseSchema: z.ZodObject<{
117
- expectations: z.ZodArray<z.ZodObject<{
118
- met: z.ZodBoolean;
119
- matchedFindingIndex: z.ZodNullable<z.ZodNumber>;
120
- reasoning: z.ZodString;
121
- }, z.core.$strip>>;
122
- antiExpectations: z.ZodArray<z.ZodObject<{
123
- violated: z.ZodBoolean;
124
- violatingFindingIndex: z.ZodNullable<z.ZodNumber>;
125
- reasoning: z.ZodString;
126
- }, z.core.$strip>>;
127
- }, z.core.$strip>;
128
- export type JudgeResponse = z.infer<typeof JudgeResponseSchema>;
129
- /**
130
- * Result of running a single eval scenario.
131
- */
132
- export interface EvalResult {
133
- /** Display name (e.g., "bug-detection/null-property-access") */
134
- name: string;
135
- /** Eval metadata */
136
- meta: EvalMeta;
137
- /** Whether the eval passed overall */
138
- passed: boolean;
139
- /** Skill report from the agent run */
140
- report: SkillReport;
141
- /** Judge response with per-expectation verdicts */
142
- judgeResponse: JudgeResponse;
143
- /** Verbose logs from the agent run */
144
- logs: string[];
145
- /** Total duration of the eval (agent + judge) in ms */
146
- durationMs: number;
147
- /** Usage from the skill run */
148
- skillUsage?: UsageStats;
149
- /** Usage from the judge call */
150
- judgeUsage?: UsageStats;
151
- }
152
- /**
153
- * Determine if an eval passed based on judge response and eval metadata.
154
- */
155
- export declare function evalPassed(meta: EvalMeta, judgeResponse: JudgeResponse, findings?: Finding[]): boolean;
156
- /**
157
- * Format an eval result for human-readable output.
158
- */
159
- export declare function formatEvalResult(result: EvalResult): string;
160
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE1E,0DAA0D;AAC1D,eAAO,MAAM,kBAAkB,sBAAsB,CAAC;AAEtD;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;iBAO3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;iBAa7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;iBAOzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,mDAAmD;IACnD,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,oCAAoC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB;;;;iBAOnC,CAAC;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;iBAOvC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAElF;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;iBAG9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,QAAQ,CAAC;IACf,sCAAsC;IACtC,MAAM,EAAE,OAAO,CAAC;IAChB,sCAAsC;IACtC,MAAM,EAAE,WAAW,CAAC;IACpB,mDAAmD;IACnD,aAAa,EAAE,aAAa,CAAC;IAC7B,sCAAsC;IACtC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gCAAgC;IAChC,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,QAAQ,EACd,aAAa,EAAE,aAAa,EAC5B,QAAQ,CAAC,EAAE,OAAO,EAAE,GACnB,OAAO,CA2BT;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAoC3D"}
@@ -1,148 +0,0 @@
1
- import { z } from 'zod';
2
- import { SeveritySchema } from '../types/index.js';
3
- /** Default model for eval skill execution and judging. */
4
- export const DEFAULT_EVAL_MODEL = 'claude-sonnet-4-6';
5
- /**
6
- * A "should find" assertion in BDD style.
7
- */
8
- export const ShouldFindSchema = z.object({
9
- /** Natural language description of the expected finding for the LLM judge */
10
- finding: z.string(),
11
- /** Expected severity level. When provided, evals require an exact normalized match. */
12
- severity: SeveritySchema.optional(),
13
- /** If true (default), eval fails when this is not found */
14
- required: z.boolean().default(true),
15
- });
16
- /**
17
- * A single eval scenario within a YAML eval file.
18
- */
19
- export const EvalScenarioSchema = z.object({
20
- /** Scenario name (e.g., "null-property-access") */
21
- name: z.string(),
22
- /** What this eval tests (BDD "given" / description) */
23
- given: z.string(),
24
- /** Fixture files to use, relative to evals/ directory */
25
- files: z.array(z.string()).min(1),
26
- /** Model override for this specific scenario */
27
- model: z.string().optional(),
28
- /** What Warden should find (BDD "then") */
29
- should_find: z.array(ShouldFindSchema).min(1),
30
- /** What Warden should NOT report (precision assertions) */
31
- should_not_find: z.array(z.string()).default([]),
32
- });
33
- /**
34
- * Root schema for a YAML eval file. Each file defines a category of evals
35
- * sharing a common skill.
36
- *
37
- * Example YAML:
38
- * skill: skills/bug-detection.md
39
- * evals:
40
- * - name: null-property-access
41
- * given: code that accesses .find() result without null checking
42
- * files: [fixtures/null-property-access/handler.ts]
43
- * should_find:
44
- * - finding: null access on user.name
45
- * severity: high
46
- */
47
- export const EvalFileSchema = z.object({
48
- /** Skill to run, relative to evals/ directory */
49
- skill: z.string(),
50
- /** Default model for all evals in this file */
51
- model: z.string().default(DEFAULT_EVAL_MODEL),
52
- /** List of eval scenarios */
53
- evals: z.array(EvalScenarioSchema).min(1),
54
- });
55
- /**
56
- * Judge verdict for a single expectation.
57
- */
58
- export const ExpectationVerdictSchema = z.object({
59
- /** Whether this expectation was met */
60
- met: z.boolean(),
61
- /** Which finding matched (by index), or null if none */
62
- matchedFindingIndex: z.number().int().nonnegative().nullable(),
63
- /** Brief reasoning from the judge */
64
- reasoning: z.string(),
65
- });
66
- /**
67
- * Judge verdict for a single anti-expectation.
68
- */
69
- export const AntiExpectationVerdictSchema = z.object({
70
- /** Whether a finding violated this anti-expectation (true = violation found) */
71
- violated: z.boolean(),
72
- /** Which finding violated (by index), or null if none */
73
- violatingFindingIndex: z.number().int().nonnegative().nullable(),
74
- /** Brief reasoning from the judge */
75
- reasoning: z.string(),
76
- });
77
- /**
78
- * Complete judge response for an eval.
79
- */
80
- export const JudgeResponseSchema = z.object({
81
- expectations: z.array(ExpectationVerdictSchema),
82
- antiExpectations: z.array(AntiExpectationVerdictSchema),
83
- });
84
- /**
85
- * Determine if an eval passed based on judge response and eval metadata.
86
- */
87
- export function evalPassed(meta, judgeResponse, findings) {
88
- // Check required should_find assertions are met
89
- for (let i = 0; i < meta.should_find.length; i++) {
90
- const assertion = meta.should_find[i];
91
- const verdict = judgeResponse.expectations[i];
92
- if (assertion?.required && !verdict?.met) {
93
- return false;
94
- }
95
- if (assertion?.required && assertion.severity && verdict?.met && findings) {
96
- const matchedFinding = verdict.matchedFindingIndex === null
97
- ? undefined
98
- : findings[verdict.matchedFindingIndex];
99
- if (matchedFinding?.severity !== assertion.severity) {
100
- return false;
101
- }
102
- }
103
- }
104
- // Check no should_not_find assertions are violated
105
- for (const verdict of judgeResponse.antiExpectations) {
106
- if (verdict.violated) {
107
- return false;
108
- }
109
- }
110
- return true;
111
- }
112
- /**
113
- * Format an eval result for human-readable output.
114
- */
115
- export function formatEvalResult(result) {
116
- const status = result.passed ? 'PASS' : 'FAIL';
117
- const lines = [];
118
- lines.push(`[${status}] ${result.name}`);
119
- lines.push(` Given: ${result.meta.given}`);
120
- lines.push(` Findings: ${result.report.findings.length}`);
121
- for (let i = 0; i < result.meta.should_find.length; i++) {
122
- const assertion = result.meta.should_find[i];
123
- const verdict = result.judgeResponse.expectations[i];
124
- const mark = verdict?.met ? 'PASS' : 'FAIL';
125
- const req = assertion?.required ? '' : ' (optional)';
126
- lines.push(` [${mark}] should find: ${assertion?.finding ?? 'unknown'}${req}`);
127
- if (verdict?.reasoning) {
128
- lines.push(` -> ${verdict.reasoning}`);
129
- }
130
- if (assertion?.severity && verdict?.met && verdict.matchedFindingIndex !== null) {
131
- const matchedSeverity = result.report.findings[verdict.matchedFindingIndex]?.severity ?? 'missing';
132
- if (matchedSeverity !== assertion.severity) {
133
- lines.push(` -> severity mismatch: expected ${assertion.severity}, got ${matchedSeverity}`);
134
- }
135
- }
136
- }
137
- for (let i = 0; i < result.meta.should_not_find.length; i++) {
138
- const assertion = result.meta.should_not_find[i];
139
- const verdict = result.judgeResponse.antiExpectations[i];
140
- const mark = verdict?.violated ? 'FAIL' : 'PASS';
141
- lines.push(` [${mark}] should not find: ${assertion ?? 'unknown'}`);
142
- if (verdict?.reasoning) {
143
- lines.push(` -> ${verdict.reasoning}`);
144
- }
145
- }
146
- return lines.join('\n');
147
- }
148
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,0DAA0D;AAC1D,MAAM,CAAC,MAAM,kBAAkB,GAAG,mBAAmB,CAAC;AAEtD;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,6EAA6E;IAC7E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,uFAAuF;IACvF,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE;IACnC,2DAA2D;IAC3D,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACpC,CAAC,CAAC;AAGH;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,mDAAmD;IACnD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,uDAAuD;IACvD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,yDAAyD;IACzD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,gDAAgD;IAChD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,2CAA2C;IAC3C,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,2DAA2D;IAC3D,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACjD,CAAC,CAAC;AAGH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,iDAAiD;IACjD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,+CAA+C;IAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAC7C,6BAA6B;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC1C,CAAC,CAAC;AA0BH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,uCAAuC;IACvC,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE;IAChB,wDAAwD;IACxD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC9D,qCAAqC;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAGH;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,gFAAgF;IAChF,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE;IACrB,yDAAyD;IACzD,qBAAqB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAChE,qCAAqC;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAGH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC;IAC/C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC;CACxD,CAAC,CAAC;AA2BH;;GAEG;AACH,MAAM,UAAU,UAAU,CACxB,IAAc,EACd,aAA4B,EAC5B,QAAoB;IAEpB,gDAAgD;IAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,SAAS,EAAE,QAAQ,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,SAAS,EAAE,QAAQ,IAAI,SAAS,CAAC,QAAQ,IAAI,OAAO,EAAE,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC1E,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,KAAK,IAAI;gBACzD,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;YAC1C,IAAI,cAAc,EAAE,QAAQ,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACpD,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,KAAK,MAAM,OAAO,IAAI,aAAa,CAAC,gBAAgB,EAAE,CAAC;QACrD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5C,MAAM,GAAG,GAAG,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,kBAAkB,SAAS,EAAE,OAAO,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC,CAAC;QAChF,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,SAAS,EAAE,QAAQ,IAAI,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,mBAAmB,KAAK,IAAI,EAAE,CAAC;YAChF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,QAAQ,IAAI,SAAS,CAAC;YACnG,IAAI,eAAe,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;gBAC3C,KAAK,CAAC,IAAI,CAAC,sCAAsC,SAAS,CAAC,QAAQ,SAAS,eAAe,EAAE,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,sBAAsB,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC;QACrE,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}