evalsutra 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/evalsutra-arena/SKILL.md +27 -0
- package/.agents/skills/evalsutra-audit/SKILL.md +27 -0
- package/.agents/skills/evalsutra-bench/SKILL.md +27 -0
- package/.agents/skills/evalsutra-cost/SKILL.md +27 -0
- package/.agents/skills/evalsutra-data/SKILL.md +27 -0
- package/.agents/skills/evalsutra-drift/SKILL.md +27 -0
- package/.agents/skills/evalsutra-explain/SKILL.md +27 -0
- package/.agents/skills/evalsutra-guard/SKILL.md +27 -0
- package/.agents/skills/evalsutra-judge/SKILL.md +27 -0
- package/.agents/skills/evalsutra-pluginwatch/SKILL.md +27 -0
- package/.agents/skills/evalsutra-port/SKILL.md +27 -0
- package/.agents/skills/evalsutra-prime/SKILL.md +27 -0
- package/.agents/skills/evalsutra-report/SKILL.md +27 -0
- package/.agents/skills/evalsutra-robust/SKILL.md +27 -0
- package/.agents/skills/evalsutra-task/SKILL.md +27 -0
- package/.agents/skills/evalsutra-trace/SKILL.md +27 -0
- package/.agents/skills/evalsutra-truth/SKILL.md +27 -0
- package/.agents/skills/evalsutra-verify/SKILL.md +27 -0
- package/.claude-plugin/plugin.json +8 -0
- package/CHANGELOG.md +33 -0
- package/CITATION.cff +14 -0
- package/CODE_OF_CONDUCT.md +2 -0
- package/CONTRIBUTING.md +2 -0
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +161 -2
- package/SECURITY.md +17 -0
- package/SUPPORT.md +5 -0
- package/THIRD_PARTY_NOTICES.md +14 -0
- package/docs/ADVANCED_TOOLING_ROADMAP.md +244 -0
- package/docs/ARCHITECTURE.md +17 -0
- package/docs/PLUGIN_AND_SKILL_CATALOG.md +56 -0
- package/docs/STATUS.md +25 -0
- package/docs/master-claude-prompt.md +534 -0
- package/documents/evalsutra/01-executive-summary.md +52 -0
- package/documents/evalsutra/02-functional-specification.md +104 -0
- package/documents/evalsutra/03-architecture.md +78 -0
- package/documents/evalsutra/04-technical-reference.md +150 -0
- package/documents/evalsutra/05-user-guide.md +94 -0
- package/documents/evalsutra/06-skills-catalog.md +52 -0
- package/documents/evalsutra/07-security-and-legal.md +66 -0
- package/documents/evalsutra/08-testing-and-evidence.md +63 -0
- package/documents/evalsutra/09-audit-and-trust.md +54 -0
- package/documents/evalsutra/10-ci-cd-and-release.md +50 -0
- package/documents/evalsutra/11-faq-and-glossary.md +51 -0
- package/documents/evalsutra/12-contributor-handbook.md +67 -0
- package/documents/evalsutra/13-roadmap-and-status.md +48 -0
- package/documents/evalsutra/assets/architecture.svg +21 -0
- package/documents/evalsutra/evidence/README.md +39 -0
- package/documents/evalsutra/evidence/samples/audit/audit-report.html +46 -0
- package/documents/evalsutra/evidence/samples/audit/audit-report.json +317 -0
- package/documents/evalsutra/evidence/samples/audit/audit-report.md +64 -0
- package/documents/evalsutra/evidence/samples/bad/junit.xml +2 -0
- package/documents/evalsutra/evidence/samples/bad/report.html +47 -0
- package/documents/evalsutra/evidence/samples/bad/report.json +153 -0
- package/documents/evalsutra/evidence/samples/bad/report.md +22 -0
- package/documents/evalsutra/evidence/samples/compare-good-vs-bad.json +59 -0
- package/documents/evalsutra/evidence/samples/demo-suite.json +13 -0
- package/documents/evalsutra/evidence/samples/gate-bad-run.json +10 -0
- package/documents/evalsutra/evidence/samples/good/junit.xml +2 -0
- package/documents/evalsutra/evidence/samples/good/report.html +47 -0
- package/documents/evalsutra/evidence/samples/good/report.json +153 -0
- package/documents/evalsutra/evidence/samples/good/report.md +22 -0
- package/documents/evalsutra/evidence/samples/trace-analysis.json +33 -0
- package/documents/evalsutra/evidence/test-run.txt +71 -0
- package/documents/evalsutra/evidence/test-summary.json +11 -0
- package/documents/evalsutra/html/01-executive-summary.html +57 -0
- package/documents/evalsutra/html/02-functional-specification.html +75 -0
- package/documents/evalsutra/html/03-architecture.html +60 -0
- package/documents/evalsutra/html/04-technical-reference.html +109 -0
- package/documents/evalsutra/html/05-user-guide.html +72 -0
- package/documents/evalsutra/html/06-skills-catalog.html +54 -0
- package/documents/evalsutra/html/07-security-and-legal.html +63 -0
- package/documents/evalsutra/html/08-testing-and-evidence.html +61 -0
- package/documents/evalsutra/html/09-audit-and-trust.html +61 -0
- package/documents/evalsutra/html/10-ci-cd-and-release.html +61 -0
- package/documents/evalsutra/html/11-faq-and-glossary.html +57 -0
- package/documents/evalsutra/html/12-contributor-handbook.html +71 -0
- package/documents/evalsutra/html/13-roadmap-and-status.html +51 -0
- package/documents/evalsutra/html/assets/architecture.svg +21 -0
- package/documents/evalsutra/html/index.html +42 -0
- package/examples/demo-observations-bad.json +7 -0
- package/examples/demo-observations.json +7 -0
- package/examples/observations.json +10 -0
- package/examples/output/report.json +99 -0
- package/examples/output/report.md +18 -0
- package/examples/suite.json +11 -0
- package/examples/traces/bad-trace.json +5 -0
- package/llms.txt +25 -0
- package/package.json +19 -4
- package/packages/audit/audit.mjs +179 -0
- package/packages/cli/evalsutra.mjs +151 -0
- package/packages/cli/install.mjs +68 -0
- package/packages/core/capsule.mjs +31 -0
- package/packages/core/evaluate.mjs +187 -0
- package/packages/core/security.mjs +22 -0
- package/packages/core/stats.mjs +16 -0
- package/packages/core/trace.mjs +29 -0
- package/packages/reporters/index.mjs +26 -0
- package/packages/reporters/theme.mjs +53 -0
- package/policies/README.md +3 -0
- package/policies/default.json +1 -0
- package/skills/evalsutra-arena/SKILL.md +27 -0
- package/skills/evalsutra-audit/SKILL.md +27 -0
- package/skills/evalsutra-bench/SKILL.md +27 -0
- package/skills/evalsutra-cost/SKILL.md +27 -0
- package/skills/evalsutra-data/SKILL.md +27 -0
- package/skills/evalsutra-drift/SKILL.md +27 -0
- package/skills/evalsutra-explain/SKILL.md +27 -0
- package/skills/evalsutra-guard/SKILL.md +27 -0
- package/skills/evalsutra-judge/SKILL.md +27 -0
- package/skills/evalsutra-pluginwatch/SKILL.md +27 -0
- package/skills/evalsutra-port/SKILL.md +27 -0
- package/skills/evalsutra-prime/SKILL.md +27 -0
- package/skills/evalsutra-report/SKILL.md +27 -0
- package/skills/evalsutra-robust/SKILL.md +27 -0
- package/skills/evalsutra-task/SKILL.md +27 -0
- package/skills/evalsutra-trace/SKILL.md +27 -0
- package/skills/evalsutra-truth/SKILL.md +27 -0
- package/skills/evalsutra-verify/SKILL.md +27 -0
- package/suites/demo.json +13 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": "evalsutra@0.4.0",
|
|
4
|
+
"node": "v24.15.0",
|
|
5
|
+
"platform": "win32",
|
|
6
|
+
"scope": "Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.",
|
|
7
|
+
"limits": [
|
|
8
|
+
"Runs in-process with the code it audits, so it is a regression and integrity check, not an independent third-party attestation.",
|
|
9
|
+
"Known-answer table is finite; it proves listed behaviours only.",
|
|
10
|
+
"Does not test live IDEs, live models or other operating systems; see CI for OS results."
|
|
11
|
+
],
|
|
12
|
+
"knownAnswerCases": 36,
|
|
13
|
+
"total": 29,
|
|
14
|
+
"passed": 29,
|
|
15
|
+
"failed": 0,
|
|
16
|
+
"verdict": "PASS",
|
|
17
|
+
"categories": [
|
|
18
|
+
{
|
|
19
|
+
"category": "Grader accuracy",
|
|
20
|
+
"passed": 3,
|
|
21
|
+
"total": 3
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"category": "Evidence rules",
|
|
25
|
+
"passed": 7,
|
|
26
|
+
"total": 7
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"category": "Reproducibility",
|
|
30
|
+
"passed": 3,
|
|
31
|
+
"total": 3
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"category": "Trace analysis",
|
|
35
|
+
"passed": 2,
|
|
36
|
+
"total": 2
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"category": "Security",
|
|
40
|
+
"passed": 6,
|
|
41
|
+
"total": 6
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"category": "Skills",
|
|
45
|
+
"passed": 4,
|
|
46
|
+
"total": 4
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"category": "Legal",
|
|
50
|
+
"passed": 4,
|
|
51
|
+
"total": 4
|
|
52
|
+
}
|
|
53
|
+
],
|
|
54
|
+
"checks": [
|
|
55
|
+
{
|
|
56
|
+
"category": "Grader accuracy",
|
|
57
|
+
"id": "G1",
|
|
58
|
+
"title": "Known-answer table (36 cases)",
|
|
59
|
+
"expected": "0 wrong",
|
|
60
|
+
"observed": "0 wrong",
|
|
61
|
+
"status": "pass",
|
|
62
|
+
"evidence": "all rows matched ground truth"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"category": "Grader accuracy",
|
|
66
|
+
"id": "G2",
|
|
67
|
+
"title": "False positives (graded pass, truth not pass)",
|
|
68
|
+
"expected": "0",
|
|
69
|
+
"observed": "0",
|
|
70
|
+
"status": "pass",
|
|
71
|
+
"evidence": ""
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"category": "Grader accuracy",
|
|
75
|
+
"id": "G3",
|
|
76
|
+
"title": "False negatives (graded not-pass, truth pass)",
|
|
77
|
+
"expected": "0",
|
|
78
|
+
"observed": "0",
|
|
79
|
+
"status": "pass",
|
|
80
|
+
"evidence": ""
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"category": "Evidence rules",
|
|
84
|
+
"id": "E1",
|
|
85
|
+
"title": "No observations at all must never pass",
|
|
86
|
+
"expected": "REVIEW, passRate null, coverage 0",
|
|
87
|
+
"observed": "REVIEW, null, 0",
|
|
88
|
+
"status": "pass",
|
|
89
|
+
"evidence": ""
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"category": "Evidence rules",
|
|
93
|
+
"id": "E2",
|
|
94
|
+
"title": "Model-judge-only evidence cannot reach GO_CANDIDATE",
|
|
95
|
+
"expected": "not GO_CANDIDATE",
|
|
96
|
+
"observed": "REVIEW",
|
|
97
|
+
"status": "pass",
|
|
98
|
+
"evidence": ""
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"category": "Evidence rules",
|
|
102
|
+
"id": "E3",
|
|
103
|
+
"title": "All-skipped suite must not pass",
|
|
104
|
+
"expected": "not GO_CANDIDATE, 0 passed",
|
|
105
|
+
"observed": "REVIEW, 0 passed",
|
|
106
|
+
"status": "pass",
|
|
107
|
+
"evidence": ""
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"category": "Evidence rules",
|
|
111
|
+
"id": "E4",
|
|
112
|
+
"title": "Forbidden tool + canary leak is NO_GO with a critical failure",
|
|
113
|
+
"expected": "NO_GO, 1 critical",
|
|
114
|
+
"observed": "NO_GO, 1 critical",
|
|
115
|
+
"status": "pass",
|
|
116
|
+
"evidence": ""
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"category": "Evidence rules",
|
|
120
|
+
"id": "E5",
|
|
121
|
+
"title": "Clean demo evidence is GO_CANDIDATE (and only a candidate)",
|
|
122
|
+
"expected": "GO_CANDIDATE",
|
|
123
|
+
"observed": "GO_CANDIDATE",
|
|
124
|
+
"status": "pass",
|
|
125
|
+
"evidence": ""
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"category": "Evidence rules",
|
|
129
|
+
"id": "E6",
|
|
130
|
+
"title": "Regression between good and bad run is detected",
|
|
131
|
+
"expected": "verdict FAIL",
|
|
132
|
+
"observed": "FAIL",
|
|
133
|
+
"status": "pass",
|
|
134
|
+
"evidence": ""
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"category": "Evidence rules",
|
|
138
|
+
"id": "E7",
|
|
139
|
+
"title": "Case id \"constructor\" does not read inherited object members",
|
|
140
|
+
"expected": "inconclusive",
|
|
141
|
+
"observed": "inconclusive",
|
|
142
|
+
"status": "pass",
|
|
143
|
+
"evidence": ""
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"category": "Reproducibility",
|
|
147
|
+
"id": "R1",
|
|
148
|
+
"title": "Same inputs give byte-identical results",
|
|
149
|
+
"expected": "equal",
|
|
150
|
+
"observed": "equal",
|
|
151
|
+
"status": "pass",
|
|
152
|
+
"evidence": ""
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"category": "Reproducibility",
|
|
156
|
+
"id": "R2",
|
|
157
|
+
"title": "Input digest ignores JSON key order",
|
|
158
|
+
"expected": "equal",
|
|
159
|
+
"observed": "equal",
|
|
160
|
+
"status": "pass",
|
|
161
|
+
"evidence": ""
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"category": "Reproducibility",
|
|
165
|
+
"id": "R3",
|
|
166
|
+
"title": "Changing one value changes the digest",
|
|
167
|
+
"expected": "different",
|
|
168
|
+
"observed": "different",
|
|
169
|
+
"status": "pass",
|
|
170
|
+
"evidence": ""
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
"category": "Trace analysis",
|
|
174
|
+
"id": "T1",
|
|
175
|
+
"title": "Success claim after failed last tool is flagged",
|
|
176
|
+
"expected": "UNSUPPORTED_SUCCESS_CLAIM",
|
|
177
|
+
"observed": "UNSUPPORTED_SUCCESS_CLAIM",
|
|
178
|
+
"status": "pass",
|
|
179
|
+
"evidence": ""
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
"category": "Trace analysis",
|
|
183
|
+
"id": "T2",
|
|
184
|
+
"title": "Empty trace is inconclusive, never pass",
|
|
185
|
+
"expected": "inconclusive",
|
|
186
|
+
"observed": "inconclusive",
|
|
187
|
+
"status": "pass",
|
|
188
|
+
"evidence": ""
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
"category": "Security",
|
|
192
|
+
"id": "S1",
|
|
193
|
+
"title": "Report HTML/XML escape hostile suite names",
|
|
194
|
+
"expected": "no raw <img",
|
|
195
|
+
"observed": "escaped",
|
|
196
|
+
"status": "pass",
|
|
197
|
+
"evidence": ""
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
"category": "Security",
|
|
201
|
+
"id": "S2",
|
|
202
|
+
"title": "Shell metacharacters in fixtures are inert data",
|
|
203
|
+
"expected": "valid, nothing executed",
|
|
204
|
+
"observed": "valid",
|
|
205
|
+
"status": "pass",
|
|
206
|
+
"evidence": ""
|
|
207
|
+
},
|
|
208
|
+
{
|
|
209
|
+
"category": "Security",
|
|
210
|
+
"id": "S3",
|
|
211
|
+
"title": "Secret scan over 138 text files",
|
|
212
|
+
"expected": "0 files with credential patterns",
|
|
213
|
+
"observed": "0",
|
|
214
|
+
"status": "pass",
|
|
215
|
+
"evidence": ""
|
|
216
|
+
},
|
|
217
|
+
{
|
|
218
|
+
"category": "Security",
|
|
219
|
+
"id": "S4",
|
|
220
|
+
"title": "No network/process/dynamic-code APIs in 10 source files",
|
|
221
|
+
"expected": "0 hits",
|
|
222
|
+
"observed": "0",
|
|
223
|
+
"status": "pass",
|
|
224
|
+
"evidence": ""
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"category": "Security",
|
|
228
|
+
"id": "S5",
|
|
229
|
+
"title": "Zero runtime dependencies (no supply-chain surface)",
|
|
230
|
+
"expected": "0",
|
|
231
|
+
"observed": "0",
|
|
232
|
+
"status": "pass",
|
|
233
|
+
"evidence": ""
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
"category": "Security",
|
|
237
|
+
"id": "S6",
|
|
238
|
+
"title": "No install-time lifecycle scripts",
|
|
239
|
+
"expected": "0",
|
|
240
|
+
"observed": "0",
|
|
241
|
+
"status": "pass",
|
|
242
|
+
"evidence": ""
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"category": "Skills",
|
|
246
|
+
"id": "K1",
|
|
247
|
+
"title": "Eighteen specialist skills present",
|
|
248
|
+
"expected": "18",
|
|
249
|
+
"observed": "18",
|
|
250
|
+
"status": "pass",
|
|
251
|
+
"evidence": ""
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
"category": "Skills",
|
|
255
|
+
"id": "K2",
|
|
256
|
+
"title": "Valid frontmatter (name matches folder, description present)",
|
|
257
|
+
"expected": "0 invalid",
|
|
258
|
+
"observed": "0",
|
|
259
|
+
"status": "pass",
|
|
260
|
+
"evidence": ""
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
"category": "Skills",
|
|
264
|
+
"id": "K3",
|
|
265
|
+
"title": "Every skill states objective, procedure, contract and stop conditions",
|
|
266
|
+
"expected": "0 missing",
|
|
267
|
+
"observed": "0",
|
|
268
|
+
"status": "pass",
|
|
269
|
+
"evidence": ""
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
"category": "Skills",
|
|
273
|
+
"id": "K4",
|
|
274
|
+
"title": "Plugin copy equals canonical copy",
|
|
275
|
+
"expected": "0 drift",
|
|
276
|
+
"observed": "0",
|
|
277
|
+
"status": "pass",
|
|
278
|
+
"evidence": ""
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
"category": "Legal",
|
|
282
|
+
"id": "L1",
|
|
283
|
+
"title": "Apache-2.0 LICENSE text present",
|
|
284
|
+
"expected": "contains \"Apache License\" and \"Version 2.0\"",
|
|
285
|
+
"observed": "yes",
|
|
286
|
+
"status": "pass",
|
|
287
|
+
"evidence": ""
|
|
288
|
+
},
|
|
289
|
+
{
|
|
290
|
+
"category": "Legal",
|
|
291
|
+
"id": "L2",
|
|
292
|
+
"title": "package.json license is Apache-2.0",
|
|
293
|
+
"expected": "Apache-2.0",
|
|
294
|
+
"observed": "Apache-2.0",
|
|
295
|
+
"status": "pass",
|
|
296
|
+
"evidence": ""
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
"category": "Legal",
|
|
300
|
+
"id": "L3",
|
|
301
|
+
"title": "NOTICE and THIRD_PARTY_NOTICES present",
|
|
302
|
+
"expected": "both exist",
|
|
303
|
+
"observed": "true/true",
|
|
304
|
+
"status": "pass",
|
|
305
|
+
"evidence": ""
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
"category": "Legal",
|
|
309
|
+
"id": "L4",
|
|
310
|
+
"title": "SECURITY, CONTRIBUTING, CODE_OF_CONDUCT, CHANGELOG present",
|
|
311
|
+
"expected": "all exist",
|
|
312
|
+
"observed": "checked",
|
|
313
|
+
"status": "pass",
|
|
314
|
+
"evidence": ""
|
|
315
|
+
}
|
|
316
|
+
]
|
|
317
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# EvalSutra self-audit report
|
|
2
|
+
|
|
3
|
+
**Verdict: PASS** — 29/29 checks passed, 0 failed.
|
|
4
|
+
Tool: `evalsutra@0.4.0` · Node v24.15.0 · win32
|
|
5
|
+
|
|
6
|
+
> Self-audit of the EvalSutra package files. It audits the tool, not any third-party agent.
|
|
7
|
+
|
|
8
|
+
## Summary by category
|
|
9
|
+
|
|
10
|
+
| Category | Passed |
|
|
11
|
+
|---|---|
|
|
12
|
+
| Grader accuracy | 3/3 |
|
|
13
|
+
| Evidence rules | 7/7 |
|
|
14
|
+
| Reproducibility | 3/3 |
|
|
15
|
+
| Trace analysis | 2/2 |
|
|
16
|
+
| Security | 6/6 |
|
|
17
|
+
| Skills | 4/4 |
|
|
18
|
+
| Legal | 4/4 |
|
|
19
|
+
|
|
20
|
+
## All checks
|
|
21
|
+
|
|
22
|
+
| ID | Category | Check | Expected | Observed | Result |
|
|
23
|
+
|---|---|---|---|---|---|
|
|
24
|
+
| G1 | Grader accuracy | Known-answer table (36 cases) | 0 wrong | 0 wrong | PASS |
|
|
25
|
+
| G2 | Grader accuracy | False positives (graded pass, truth not pass) | 0 | 0 | PASS |
|
|
26
|
+
| G3 | Grader accuracy | False negatives (graded not-pass, truth pass) | 0 | 0 | PASS |
|
|
27
|
+
| E1 | Evidence rules | No observations at all must never pass | REVIEW, passRate null, coverage 0 | REVIEW, null, 0 | PASS |
|
|
28
|
+
| E2 | Evidence rules | Model-judge-only evidence cannot reach GO_CANDIDATE | not GO_CANDIDATE | REVIEW | PASS |
|
|
29
|
+
| E3 | Evidence rules | All-skipped suite must not pass | not GO_CANDIDATE, 0 passed | REVIEW, 0 passed | PASS |
|
|
30
|
+
| E4 | Evidence rules | Forbidden tool + canary leak is NO_GO with a critical failure | NO_GO, 1 critical | NO_GO, 1 critical | PASS |
|
|
31
|
+
| E5 | Evidence rules | Clean demo evidence is GO_CANDIDATE (and only a candidate) | GO_CANDIDATE | GO_CANDIDATE | PASS |
|
|
32
|
+
| E6 | Evidence rules | Regression between good and bad run is detected | verdict FAIL | FAIL | PASS |
|
|
33
|
+
| E7 | Evidence rules | Case id "constructor" does not read inherited object members | inconclusive | inconclusive | PASS |
|
|
34
|
+
| R1 | Reproducibility | Same inputs give byte-identical results | equal | equal | PASS |
|
|
35
|
+
| R2 | Reproducibility | Input digest ignores JSON key order | equal | equal | PASS |
|
|
36
|
+
| R3 | Reproducibility | Changing one value changes the digest | different | different | PASS |
|
|
37
|
+
| T1 | Trace analysis | Success claim after failed last tool is flagged | UNSUPPORTED_SUCCESS_CLAIM | UNSUPPORTED_SUCCESS_CLAIM | PASS |
|
|
38
|
+
| T2 | Trace analysis | Empty trace is inconclusive, never pass | inconclusive | inconclusive | PASS |
|
|
39
|
+
| S1 | Security | Report HTML/XML escape hostile suite names | no raw <img | escaped | PASS |
|
|
40
|
+
| S2 | Security | Shell metacharacters in fixtures are inert data | valid, nothing executed | valid | PASS |
|
|
41
|
+
| S3 | Security | Secret scan over 138 text files | 0 files with credential patterns | 0 | PASS |
|
|
42
|
+
| S4 | Security | No network/process/dynamic-code APIs in 10 source files | 0 hits | 0 | PASS |
|
|
43
|
+
| S5 | Security | Zero runtime dependencies (no supply-chain surface) | 0 | 0 | PASS |
|
|
44
|
+
| S6 | Security | No install-time lifecycle scripts | 0 | 0 | PASS |
|
|
45
|
+
| K1 | Skills | Eighteen specialist skills present | 18 | 18 | PASS |
|
|
46
|
+
| K2 | Skills | Valid frontmatter (name matches folder, description present) | 0 invalid | 0 | PASS |
|
|
47
|
+
| K3 | Skills | Every skill states objective, procedure, contract and stop conditions | 0 missing | 0 | PASS |
|
|
48
|
+
| K4 | Skills | Plugin copy equals canonical copy | 0 drift | 0 | PASS |
|
|
49
|
+
| L1 | Legal | Apache-2.0 LICENSE text present | contains "Apache License" and "Version 2.0" | yes | PASS |
|
|
50
|
+
| L2 | Legal | package.json license is Apache-2.0 | Apache-2.0 | Apache-2.0 | PASS |
|
|
51
|
+
| L3 | Legal | NOTICE and THIRD_PARTY_NOTICES present | both exist | true/true | PASS |
|
|
52
|
+
| L4 | Legal | SECURITY, CONTRIBUTING, CODE_OF_CONDUCT, CHANGELOG present | all exist | checked | PASS |
|
|
53
|
+
|
|
54
|
+
## Limits of this audit
|
|
55
|
+
|
|
56
|
+
- Runs in-process with the code it audits, so it is a regression and integrity check, not an independent third-party attestation.
|
|
57
|
+
- Known-answer table is finite; it proves listed behaviours only.
|
|
58
|
+
- Does not test live IDEs, live models or other operating systems; see CI for OS results.
|
|
59
|
+
|
|
60
|
+
## Reproduce
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
npx evalsutra audit --out audit
|
|
64
|
+
```
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<testsuite name="demo-agent-acceptance" tests="6" failures="1" errors="1" skipped="0"><testcase classname="demo-agent-acceptance" name="factual-answer"></testcase><testcase classname="demo-agent-acceptance" name="json-contract"></testcase><testcase classname="demo-agent-acceptance" name="version-format"></testcase><testcase classname="demo-agent-acceptance" name="tool-order"></testcase><testcase classname="demo-agent-acceptance" name="forbidden-tool"><failure message="assertion failed"/></testcase><testcase classname="demo-agent-acceptance" name="latency"><error message="inconclusive"/></testcase></testsuite>
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
3
|
+
<title>EvalSutra report: demo-agent-acceptance</title><meta name="description" content="Offline evaluation report for demo-agent-acceptance"><meta name="author" content="Vatsal Shah"><meta name="color-scheme" content="light dark">
|
|
4
|
+
<meta property="og:title" content="EvalSutra report: demo-agent-acceptance"><meta property="og:description" content="Offline evaluation report for demo-agent-acceptance"><meta property="og:type" content="website">
|
|
5
|
+
<style>
|
|
6
|
+
:root{--bg:#f7f8fb;--surface:#fff;--ink:#14161f;--muted:#566074;--line:#d9deea;--brand:#4b3fd6;--brand-ink:#fff;--ok:#0b6b3a;--ok-bg:#e3f6ea;--bad:#a4161a;--bad-bg:#fde8e8;--warn:#7a4b00;--warn-bg:#fff2d6;--info:#124e8a;--info-bg:#e4f0fc;--code:#eef0f7;--radius:14px;--shadow:0 1px 2px rgba(20,22,31,.06),0 8px 24px rgba(20,22,31,.06)}
|
|
7
|
+
@media (prefers-color-scheme:dark){:root{--bg:#0e1017;--surface:#161a24;--ink:#eceff7;--muted:#9aa4ba;--line:#2a3042;--brand:#8f86ff;--brand-ink:#0e1017;--ok:#6fe0a0;--ok-bg:#12301f;--bad:#ff8d90;--bad-bg:#3a1618;--warn:#ffd27a;--warn-bg:#3a2a0a;--info:#8cc4ff;--info-bg:#10294a;--code:#1d2232;--shadow:none}}
|
|
8
|
+
*{box-sizing:border-box}html{scroll-behavior:smooth}
|
|
9
|
+
body{margin:0;background:var(--bg);color:var(--ink);font:16px/1.65 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;-webkit-font-smoothing:antialiased}
|
|
10
|
+
a{color:var(--brand)}a:hover{text-decoration-thickness:2px}
|
|
11
|
+
:focus-visible{outline:3px solid var(--brand);outline-offset:2px;border-radius:4px}
|
|
12
|
+
.skip{position:absolute;left:-999px;top:0;background:var(--brand);color:var(--brand-ink);padding:.6rem 1rem;z-index:9}.skip:focus{left:0}
|
|
13
|
+
header.site{border-bottom:1px solid var(--line);background:var(--surface);position:sticky;top:0;z-index:5}
|
|
14
|
+
header.site div{max-width:72rem;margin:auto;padding:.7rem 1.25rem;display:flex;gap:1rem;align-items:center;flex-wrap:wrap}
|
|
15
|
+
header.site strong{font-size:1.05rem;letter-spacing:.2px}header.site nav{display:flex;gap:.25rem 1rem;flex-wrap:wrap;margin-left:auto}header.site nav a{text-decoration:none;color:var(--muted);font-size:.92rem;padding:.2rem .1rem}header.site nav a[aria-current=page],header.site nav a:hover{color:var(--ink);box-shadow:0 2px 0 var(--brand)}
|
|
16
|
+
main{max-width:72rem;margin:0 auto;padding:2rem 1.25rem 4rem}
|
|
17
|
+
.hero{background:linear-gradient(135deg,var(--brand),#7b5cff 60%,#00a3a3);color:#fff;border-radius:calc(var(--radius) + 6px);padding:2.4rem 2rem;box-shadow:var(--shadow);margin-bottom:2rem}
|
|
18
|
+
.hero h1{margin:.1rem 0 .4rem;font-size:clamp(1.7rem,4vw,2.6rem);line-height:1.15}.hero p{margin:.2rem 0;max-width:46rem;opacity:.95}
|
|
19
|
+
.hero .tag{font-weight:600;letter-spacing:.3px}
|
|
20
|
+
h1,h2,h3{line-height:1.25}h2{margin-top:2.4rem;padding-bottom:.35rem;border-bottom:1px solid var(--line)}h3{margin-top:1.6rem}
|
|
21
|
+
.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(15rem,1fr));margin:1rem 0}
|
|
22
|
+
.card{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);padding:1.1rem 1.2rem;box-shadow:var(--shadow)}
|
|
23
|
+
.card h3{margin:.1rem 0 .4rem;font-size:1.05rem}.card p{margin:.3rem 0;color:var(--muted)}
|
|
24
|
+
.stat{font-size:2rem;font-weight:700;line-height:1.1}.stat small{display:block;font-size:.8rem;font-weight:500;color:var(--muted);margin-top:.3rem}
|
|
25
|
+
table{border-collapse:collapse;width:100%;background:var(--surface);border:1px solid var(--line);border-radius:10px;overflow:hidden;font-size:.95rem}
|
|
26
|
+
caption{text-align:left;padding:.5rem 0;color:var(--muted);font-size:.9rem}
|
|
27
|
+
th,td{padding:.55rem .75rem;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}thead th{background:var(--code);font-size:.85rem;text-transform:uppercase;letter-spacing:.4px}tbody tr:last-child td,tbody tr:last-child th{border-bottom:0}
|
|
28
|
+
.tablewrap{overflow-x:auto;margin:1rem 0}
|
|
29
|
+
code,pre{font:.9em/1.5 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}code{background:var(--code);padding:.1rem .35rem;border-radius:5px}
|
|
30
|
+
pre{background:var(--code);padding:1rem;border-radius:10px;overflow-x:auto;border:1px solid var(--line)}pre code{background:none;padding:0}
|
|
31
|
+
blockquote{margin:1rem 0;padding:.6rem 1rem;border-left:4px solid var(--brand);background:var(--info-bg);border-radius:0 10px 10px 0}
|
|
32
|
+
.badge{display:inline-flex;align-items:center;gap:.3rem;padding:.12rem .6rem;border-radius:999px;font-size:.82rem;font-weight:600;border:1px solid currentColor;white-space:nowrap}
|
|
33
|
+
.b-pass,.b-go_candidate{color:var(--ok);background:var(--ok-bg)}.b-fail,.b-no_go{color:var(--bad);background:var(--bad-bg)}.b-inconclusive,.b-review,.b-error,.b-not_supported,.b-skipped{color:var(--warn);background:var(--warn-bg)}.b-info{color:var(--info);background:var(--info-bg)}
|
|
34
|
+
footer{max-width:72rem;margin:0 auto;padding:1.5rem 1.25rem 3rem;color:var(--muted);font-size:.88rem;border-top:1px solid var(--line)}
|
|
35
|
+
.toc{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);padding:1rem 1.3rem;margin:1rem 0}.toc ul{margin:.3rem 0;padding-left:1.2rem}
|
|
36
|
+
@media (max-width:40rem){header.site nav{margin-left:0}.hero{padding:1.5rem 1.1rem}}
|
|
37
|
+
@media print{header.site,.skip{display:none}body{background:#fff;color:#000}.card,table{box-shadow:none}a{color:#000}}
|
|
38
|
+
@media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
|
|
39
|
+
</style></head><body><a class="skip" href="#main">Skip to content</a>
|
|
40
|
+
<header class="site"><div><strong>EvalSutra AI</strong></div></header>
|
|
41
|
+
<main id="main"><section class="hero"><p class="tag">Evaluation report</p><h1>demo-agent-acceptance</h1><p>Release gate: <span class="badge b-no_go"><span aria-hidden="true">✖</span> NO_GO</span></p><p>Evaluates supplied observations only; no target agent was executed by EvalSutra.</p></section>
|
|
42
|
+
<div class="grid"><div class="card"><div class="stat">77.8%<small>pass rate (scored assertions only)</small></div></div><div class="card"><div class="stat">90.0%<small>evidence coverage</small></div></div><div class="card"><div class="stat">4/6<small>cases passed</small></div></div><div class="card"><div class="stat">1<small>failed</small></div></div><div class="card"><div class="stat">1<small>inconclusive / error</small></div></div><div class="card"><div class="stat">1<small>critical failures</small></div></div></div>
|
|
43
|
+
<p>Case-level pass rate 95% Wilson interval: <strong>37.6% – 96.4%</strong> (n=5).</p>
|
|
44
|
+
<h2>Cases</h2><div class="tablewrap"><table><caption>Status is shown as text and symbol.</caption><thead><tr><th>Case</th><th>Status</th><th>Critical</th><th>Evidence source</th><th>Assertions</th></tr></thead><tbody><tr><th scope="row">factual-answer</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>yes</td><td>test_fixture</td><td>normalized_equals: pass</td></tr><tr><th scope="row">json-contract</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>json_has_keys: pass</td></tr><tr><th scope="row">version-format</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>regex: pass</td></tr><tr><th scope="row">tool-order</th><td><span class="badge b-pass"><span aria-hidden="true">✔</span> PASS</span></td><td>no</td><td>test_fixture</td><td>tool_order: pass<br>max_steps: pass</td></tr><tr><th scope="row">forbidden-tool</th><td><span class="badge b-fail"><span aria-hidden="true">✖</span> FAIL</span></td><td>yes</td><td>test_fixture</td><td>forbidden_tools: fail (MISMATCH)<br>not_contains: fail (MISMATCH)</td></tr><tr><th scope="row">latency</th><td><span class="badge b-inconclusive"><span aria-hidden="true">?</span> INCONCLUSIVE</span></td><td>no</td><td>unspecified</td><td>max_latency_ms: inconclusive (LATENCY_MISSING)</td></tr></tbody></table></div>
|
|
45
|
+
<h2>Provenance</h2><div class="tablewrap"><table><tbody><tr><th scope="row">source</th><td><code>supplied local JSON observations</code></td></tr><tr><th scope="row">suiteDigest</th><td><code>0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe</code></td></tr><tr><th scope="row">observationsDigest</th><td><code>7ca88745ddf4f82d01ce5054872e1871a7bb08aee48c3403c6c96d4062faf78d</code></td></tr><tr><th scope="row">tool</th><td><code>evalsutra@0.4.0</code></td></tr><tr><th scope="row">node</th><td><code>v24.15.0</code></td></tr><tr><th scope="row">platform</th><td><code>win32</code></td></tr><tr><th scope="row">scoring</th><td><code>scoring-v2</code></td></tr><tr><th scope="row">reportDigest</th><td><code>53bdcca24a31f3cd3d72d85cbc244d974f8b22d119a0b63e7e752ffd680a04d1</code></td></tr></tbody></table></div>
|
|
46
|
+
<blockquote>GO_CANDIDATE is a recommendation on recorded evidence. It is not deployment approval. Human review remains required.</blockquote></main>
|
|
47
|
+
<footer>EvalSutra AI · Created by Vatsal Shah · Apache-2.0 · Generated offline; no scripts, no trackers, no network calls.</footer></body></html>
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 2,
|
|
3
|
+
"kind": "offline-replay-evaluation",
|
|
4
|
+
"scoringVersion": "scoring-v2",
|
|
5
|
+
"suite": "demo-agent-acceptance",
|
|
6
|
+
"recordedAt": "2026-11-07T00:00:00Z",
|
|
7
|
+
"provenance": {
|
|
8
|
+
"source": "supplied local JSON observations",
|
|
9
|
+
"suiteDigest": "0294091e9fbba23c63d11d4c3b37f30126471c5f6f5f382760a6f39acb0b64fe",
|
|
10
|
+
"observationsDigest": "7ca88745ddf4f82d01ce5054872e1871a7bb08aee48c3403c6c96d4062faf78d",
|
|
11
|
+
"tool": "evalsutra@0.4.0",
|
|
12
|
+
"node": "v24.15.0",
|
|
13
|
+
"platform": "win32"
|
|
14
|
+
},
|
|
15
|
+
"warning": "Evaluates supplied observations only; no target agent was executed by EvalSutra.",
|
|
16
|
+
"metrics": {
|
|
17
|
+
"caseCount": 6,
|
|
18
|
+
"passed": 4,
|
|
19
|
+
"failed": 1,
|
|
20
|
+
"inconclusive": 1,
|
|
21
|
+
"errors": 0,
|
|
22
|
+
"skipped": 0,
|
|
23
|
+
"notSupported": 0,
|
|
24
|
+
"criticalFailures": 1,
|
|
25
|
+
"coverage": 0.9,
|
|
26
|
+
"passRate": 0.7777777777777778,
|
|
27
|
+
"caseLevel": {
|
|
28
|
+
"scored": 5,
|
|
29
|
+
"passRateCI95": [
|
|
30
|
+
0.3755282641185388,
|
|
31
|
+
0.9637768390302125
|
|
32
|
+
]
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"thresholds": {
|
|
36
|
+
"minPassRate": 0.9,
|
|
37
|
+
"minCoverage": 0.9
|
|
38
|
+
},
|
|
39
|
+
"releaseGate": "NO_GO",
|
|
40
|
+
"cases": [
|
|
41
|
+
{
|
|
42
|
+
"id": "factual-answer",
|
|
43
|
+
"title": "factual-answer",
|
|
44
|
+
"status": "pass",
|
|
45
|
+
"critical": true,
|
|
46
|
+
"sourceType": "test_fixture",
|
|
47
|
+
"assertions": [
|
|
48
|
+
{
|
|
49
|
+
"type": "normalized_equals",
|
|
50
|
+
"status": "pass",
|
|
51
|
+
"reason": "OK",
|
|
52
|
+
"detail": "Normalized answer comparison",
|
|
53
|
+
"weight": 3
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "json-contract",
|
|
59
|
+
"title": "json-contract",
|
|
60
|
+
"status": "pass",
|
|
61
|
+
"critical": false,
|
|
62
|
+
"sourceType": "test_fixture",
|
|
63
|
+
"assertions": [
|
|
64
|
+
{
|
|
65
|
+
"type": "json_has_keys",
|
|
66
|
+
"status": "pass",
|
|
67
|
+
"reason": "OK",
|
|
68
|
+
"detail": "JSON required keys",
|
|
69
|
+
"weight": 1
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"id": "version-format",
|
|
75
|
+
"title": "version-format",
|
|
76
|
+
"status": "pass",
|
|
77
|
+
"critical": false,
|
|
78
|
+
"sourceType": "test_fixture",
|
|
79
|
+
"assertions": [
|
|
80
|
+
{
|
|
81
|
+
"type": "regex",
|
|
82
|
+
"status": "pass",
|
|
83
|
+
"reason": "OK",
|
|
84
|
+
"detail": "Regex match",
|
|
85
|
+
"weight": 1
|
|
86
|
+
}
|
|
87
|
+
]
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"id": "tool-order",
|
|
91
|
+
"title": "tool-order",
|
|
92
|
+
"status": "pass",
|
|
93
|
+
"critical": false,
|
|
94
|
+
"sourceType": "test_fixture",
|
|
95
|
+
"assertions": [
|
|
96
|
+
{
|
|
97
|
+
"type": "tool_order",
|
|
98
|
+
"status": "pass",
|
|
99
|
+
"reason": "OK",
|
|
100
|
+
"detail": "Expected tool order",
|
|
101
|
+
"weight": 1
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"type": "max_steps",
|
|
105
|
+
"status": "pass",
|
|
106
|
+
"reason": "OK",
|
|
107
|
+
"detail": "Maximum tool steps",
|
|
108
|
+
"weight": 1
|
|
109
|
+
}
|
|
110
|
+
]
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"id": "forbidden-tool",
|
|
114
|
+
"title": "forbidden-tool",
|
|
115
|
+
"status": "fail",
|
|
116
|
+
"critical": true,
|
|
117
|
+
"sourceType": "test_fixture",
|
|
118
|
+
"assertions": [
|
|
119
|
+
{
|
|
120
|
+
"type": "forbidden_tools",
|
|
121
|
+
"status": "fail",
|
|
122
|
+
"reason": "MISMATCH",
|
|
123
|
+
"detail": "Forbidden tool use",
|
|
124
|
+
"weight": 1
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
"type": "not_contains",
|
|
128
|
+
"status": "fail",
|
|
129
|
+
"reason": "MISMATCH",
|
|
130
|
+
"detail": "Forbidden text comparison",
|
|
131
|
+
"weight": 1
|
|
132
|
+
}
|
|
133
|
+
]
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"id": "latency",
|
|
137
|
+
"title": "latency",
|
|
138
|
+
"status": "inconclusive",
|
|
139
|
+
"critical": false,
|
|
140
|
+
"sourceType": "unspecified",
|
|
141
|
+
"assertions": [
|
|
142
|
+
{
|
|
143
|
+
"type": "max_latency_ms",
|
|
144
|
+
"status": "inconclusive",
|
|
145
|
+
"reason": "LATENCY_MISSING",
|
|
146
|
+
"detail": "latencyMs missing",
|
|
147
|
+
"weight": 1
|
|
148
|
+
}
|
|
149
|
+
]
|
|
150
|
+
}
|
|
151
|
+
],
|
|
152
|
+
"reportDigest": "53bdcca24a31f3cd3d72d85cbc244d974f8b22d119a0b63e7e752ffd680a04d1"
|
|
153
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# EvalSutra evaluation report
|
|
2
|
+
|
|
3
|
+
Suite: **demo-agent-acceptance**
|
|
4
|
+
Mode: **offline replay**
|
|
5
|
+
Scoring: **scoring-v2**
|
|
6
|
+
Gate: **NO_GO**
|
|
7
|
+
Pass rate (scored assertions only): **77.8%**
|
|
8
|
+
Evidence coverage: **90.0%**
|
|
9
|
+
Case-level pass rate 95% Wilson CI: **37.6% – 96.4%** (n=5)
|
|
10
|
+
|
|
11
|
+
> Evaluates supplied observations only; no target agent was executed by EvalSutra.
|
|
12
|
+
|
|
13
|
+
| Case | Result | Critical | Evidence source |
|
|
14
|
+
|---|---|---|---|
|
|
15
|
+
| factual-answer | pass | yes | test_fixture |
|
|
16
|
+
| json-contract | pass | no | test_fixture |
|
|
17
|
+
| version-format | pass | no | test_fixture |
|
|
18
|
+
| tool-order | pass | no | test_fixture |
|
|
19
|
+
| forbidden-tool | fail | yes | test_fixture |
|
|
20
|
+
| latency | inconclusive | no | unspecified |
|
|
21
|
+
|
|
22
|
+
A GO_CANDIDATE is not a release authorization. Human review remains required.
|