@artemiskit/reports 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/dist/html/generator.d.ts +6 -0
- package/dist/html/generator.d.ts.map +1 -0
- package/dist/html/redteam-generator.d.ts +6 -0
- package/dist/html/redteam-generator.d.ts.map +1 -0
- package/dist/html/stress-generator.d.ts +6 -0
- package/dist/html/stress-generator.d.ts.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6524 -0
- package/dist/json/generator.d.ts +12 -0
- package/dist/json/generator.d.ts.map +1 -0
- package/package.json +46 -0
- package/src/html/generator.ts +258 -0
- package/src/html/redteam-generator.ts +385 -0
- package/src/html/stress-generator.ts +373 -0
- package/src/index.ts +14 -0
- package/src/json/generator.ts +18 -0
- package/tsconfig.json +13 -0
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Red Team HTML Report Generator
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import type { RedTeamManifest } from '@artemiskit/core';
|
|
6
|
+
import Handlebars from 'handlebars';
|
|
7
|
+
|
|
8
|
+
const HTML_TEMPLATE = `
|
|
9
|
+
<!DOCTYPE html>
|
|
10
|
+
<html lang="en">
|
|
11
|
+
<head>
|
|
12
|
+
<meta charset="UTF-8">
|
|
13
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
14
|
+
<title>Artemis Red Team Report - {{manifest.config.scenario}}</title>
|
|
15
|
+
<style>
|
|
16
|
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
17
|
+
body {
|
|
18
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
19
|
+
line-height: 1.6;
|
|
20
|
+
color: #333;
|
|
21
|
+
background: #f5f5f5;
|
|
22
|
+
padding: 2rem;
|
|
23
|
+
}
|
|
24
|
+
.container { max-width: 1200px; margin: 0 auto; }
|
|
25
|
+
h1 { margin-bottom: 0.5rem; color: #1a1a1a; }
|
|
26
|
+
h2 { margin: 2rem 0 1rem; color: #333; border-bottom: 2px solid #e0e0e0; padding-bottom: 0.5rem; }
|
|
27
|
+
.meta { color: #666; margin-bottom: 2rem; }
|
|
28
|
+
.badge {
|
|
29
|
+
display: inline-block;
|
|
30
|
+
padding: 0.25rem 0.75rem;
|
|
31
|
+
border-radius: 4px;
|
|
32
|
+
font-size: 0.875rem;
|
|
33
|
+
font-weight: 600;
|
|
34
|
+
margin-right: 0.5rem;
|
|
35
|
+
}
|
|
36
|
+
.badge.redteam { background: #fef3c7; color: #92400e; }
|
|
37
|
+
.summary {
|
|
38
|
+
display: grid;
|
|
39
|
+
grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
|
|
40
|
+
gap: 1rem;
|
|
41
|
+
margin-bottom: 2rem;
|
|
42
|
+
}
|
|
43
|
+
.card {
|
|
44
|
+
background: white;
|
|
45
|
+
padding: 1.5rem;
|
|
46
|
+
border-radius: 8px;
|
|
47
|
+
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
|
48
|
+
}
|
|
49
|
+
.card h3 { font-size: 0.875rem; color: #666; margin-bottom: 0.5rem; }
|
|
50
|
+
.card .value { font-size: 2rem; font-weight: bold; }
|
|
51
|
+
.card .value.success { color: #22c55e; }
|
|
52
|
+
.card .value.warning { color: #f59e0b; }
|
|
53
|
+
.card .value.error { color: #ef4444; }
|
|
54
|
+
.card .value.info { color: #3b82f6; }
|
|
55
|
+
.severity-breakdown {
|
|
56
|
+
display: flex;
|
|
57
|
+
gap: 1rem;
|
|
58
|
+
flex-wrap: wrap;
|
|
59
|
+
margin-top: 1rem;
|
|
60
|
+
}
|
|
61
|
+
.severity-item {
|
|
62
|
+
display: flex;
|
|
63
|
+
align-items: center;
|
|
64
|
+
gap: 0.5rem;
|
|
65
|
+
}
|
|
66
|
+
.severity-dot {
|
|
67
|
+
width: 12px;
|
|
68
|
+
height: 12px;
|
|
69
|
+
border-radius: 50%;
|
|
70
|
+
}
|
|
71
|
+
.severity-dot.critical { background: #dc2626; }
|
|
72
|
+
.severity-dot.high { background: #ea580c; }
|
|
73
|
+
.severity-dot.medium { background: #f59e0b; }
|
|
74
|
+
.severity-dot.low { background: #eab308; }
|
|
75
|
+
table { width: 100%; border-collapse: collapse; background: white; border-radius: 8px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
|
|
76
|
+
th, td { padding: 1rem; text-align: left; border-bottom: 1px solid #e0e0e0; }
|
|
77
|
+
th { background: #f9fafb; font-weight: 600; }
|
|
78
|
+
tr:last-child td { border-bottom: none; }
|
|
79
|
+
.status { display: inline-block; padding: 0.25rem 0.75rem; border-radius: 9999px; font-size: 0.875rem; font-weight: 500; }
|
|
80
|
+
.status.safe { background: #dcfce7; color: #166534; }
|
|
81
|
+
.status.unsafe { background: #fee2e2; color: #991b1b; }
|
|
82
|
+
.status.blocked { background: #dbeafe; color: #1e40af; }
|
|
83
|
+
.status.error { background: #fef3c7; color: #92400e; }
|
|
84
|
+
.severity { display: inline-block; padding: 0.125rem 0.5rem; border-radius: 4px; font-size: 0.75rem; font-weight: 500; margin-left: 0.5rem; }
|
|
85
|
+
.severity.critical { background: #fecaca; color: #991b1b; }
|
|
86
|
+
.severity.high { background: #fed7aa; color: #9a3412; }
|
|
87
|
+
.severity.medium { background: #fef08a; color: #854d0e; }
|
|
88
|
+
.severity.low { background: #fef9c3; color: #a16207; }
|
|
89
|
+
.mutation-tag { display: inline-block; padding: 0.125rem 0.5rem; background: #e5e7eb; border-radius: 4px; font-size: 0.75rem; color: #4b5563; }
|
|
90
|
+
.details { margin-top: 0.5rem; padding: 1rem; background: #f9fafb; border-radius: 4px; font-size: 0.875rem; }
|
|
91
|
+
.details pre { white-space: pre-wrap; word-break: break-word; max-height: 200px; overflow-y: auto; }
|
|
92
|
+
.expandable { cursor: pointer; }
|
|
93
|
+
.expandable:hover { background: #f0f0f0; }
|
|
94
|
+
.hidden { display: none; }
|
|
95
|
+
.source-badge {
|
|
96
|
+
display: inline-block;
|
|
97
|
+
padding: 0.125rem 0.5rem;
|
|
98
|
+
border-radius: 4px;
|
|
99
|
+
font-size: 0.75rem;
|
|
100
|
+
font-weight: 500;
|
|
101
|
+
background: #e0e7ff;
|
|
102
|
+
color: #3730a3;
|
|
103
|
+
margin-left: 0.5rem;
|
|
104
|
+
}
|
|
105
|
+
.redaction-banner {
|
|
106
|
+
background: #fef3c7;
|
|
107
|
+
border: 1px solid #f59e0b;
|
|
108
|
+
border-radius: 8px;
|
|
109
|
+
padding: 1rem;
|
|
110
|
+
margin-bottom: 1.5rem;
|
|
111
|
+
display: flex;
|
|
112
|
+
align-items: center;
|
|
113
|
+
gap: 0.75rem;
|
|
114
|
+
}
|
|
115
|
+
.redaction-banner .icon { font-size: 1.5rem; }
|
|
116
|
+
.redaction-banner .content { flex: 1; }
|
|
117
|
+
.redaction-banner .title { font-weight: 600; color: #92400e; }
|
|
118
|
+
.redaction-banner .details { font-size: 0.875rem; color: #a16207; margin-top: 0.25rem; }
|
|
119
|
+
.redacted-badge {
|
|
120
|
+
display: inline-block;
|
|
121
|
+
padding: 0.125rem 0.5rem;
|
|
122
|
+
border-radius: 4px;
|
|
123
|
+
font-size: 0.75rem;
|
|
124
|
+
font-weight: 500;
|
|
125
|
+
background: #fef3c7;
|
|
126
|
+
color: #92400e;
|
|
127
|
+
margin-left: 0.5rem;
|
|
128
|
+
}
|
|
129
|
+
.defense-meter {
|
|
130
|
+
width: 100%;
|
|
131
|
+
height: 24px;
|
|
132
|
+
background: #e5e7eb;
|
|
133
|
+
border-radius: 12px;
|
|
134
|
+
overflow: hidden;
|
|
135
|
+
margin-top: 1rem;
|
|
136
|
+
}
|
|
137
|
+
.defense-meter-fill {
|
|
138
|
+
height: 100%;
|
|
139
|
+
background: linear-gradient(90deg, #22c55e, #16a34a);
|
|
140
|
+
display: flex;
|
|
141
|
+
align-items: center;
|
|
142
|
+
justify-content: flex-end;
|
|
143
|
+
padding-right: 0.5rem;
|
|
144
|
+
color: white;
|
|
145
|
+
font-weight: 600;
|
|
146
|
+
font-size: 0.875rem;
|
|
147
|
+
}
|
|
148
|
+
footer { margin-top: 3rem; text-align: center; color: #666; font-size: 0.875rem; }
|
|
149
|
+
</style>
|
|
150
|
+
</head>
|
|
151
|
+
<body>
|
|
152
|
+
<div class="container">
|
|
153
|
+
<h1>
|
|
154
|
+
<span class="badge redteam">RED TEAM</span>
|
|
155
|
+
{{manifest.config.scenario}}
|
|
156
|
+
</h1>
|
|
157
|
+
<p class="meta">
|
|
158
|
+
Run ID: {{manifest.run_id}} |
|
|
159
|
+
Provider: {{manifest.config.provider}} |
|
|
160
|
+
{{#if manifest.config.model}}Model: {{manifest.config.model}} |{{/if}}
|
|
161
|
+
{{formatDate manifest.start_time}}
|
|
162
|
+
</p>
|
|
163
|
+
|
|
164
|
+
{{#if manifest.redaction.enabled}}
|
|
165
|
+
<div class="redaction-banner">
|
|
166
|
+
<div class="icon">🔒</div>
|
|
167
|
+
<div class="content">
|
|
168
|
+
<div class="title">Data Redaction Applied</div>
|
|
169
|
+
<div class="details">
|
|
170
|
+
{{manifest.redaction.summary.totalRedactions}} redactions made
|
|
171
|
+
({{manifest.redaction.summary.promptsRedacted}} prompts, {{manifest.redaction.summary.responsesRedacted}} responses).
|
|
172
|
+
Replacement: <code>{{manifest.redaction.replacement}}</code>
|
|
173
|
+
</div>
|
|
174
|
+
</div>
|
|
175
|
+
</div>
|
|
176
|
+
{{/if}}
|
|
177
|
+
|
|
178
|
+
<div class="summary">
|
|
179
|
+
<div class="card">
|
|
180
|
+
<h3>Defense Rate</h3>
|
|
181
|
+
<div class="value {{defenseRateClass manifest.metrics.defense_rate}}">
|
|
182
|
+
{{formatPercent manifest.metrics.defense_rate}}
|
|
183
|
+
</div>
|
|
184
|
+
<div class="defense-meter">
|
|
185
|
+
<div class="defense-meter-fill" style="width: {{formatPercentRaw manifest.metrics.defense_rate}}">
|
|
186
|
+
{{manifest.metrics.defended}}/{{subtract manifest.metrics.total_tests manifest.metrics.error_responses}}
|
|
187
|
+
</div>
|
|
188
|
+
</div>
|
|
189
|
+
</div>
|
|
190
|
+
<div class="card">
|
|
191
|
+
<h3>Total Tests</h3>
|
|
192
|
+
<div class="value">{{manifest.metrics.total_tests}}</div>
|
|
193
|
+
</div>
|
|
194
|
+
<div class="card">
|
|
195
|
+
<h3>Safe Responses</h3>
|
|
196
|
+
<div class="value success">{{manifest.metrics.safe_responses}}</div>
|
|
197
|
+
</div>
|
|
198
|
+
<div class="card">
|
|
199
|
+
<h3>Blocked by Provider</h3>
|
|
200
|
+
<div class="value info">{{manifest.metrics.blocked_responses}}</div>
|
|
201
|
+
</div>
|
|
202
|
+
<div class="card">
|
|
203
|
+
<h3>Unsafe Responses</h3>
|
|
204
|
+
<div class="value {{#if manifest.metrics.unsafe_responses}}error{{/if}}">{{manifest.metrics.unsafe_responses}}</div>
|
|
205
|
+
</div>
|
|
206
|
+
<div class="card">
|
|
207
|
+
<h3>Errors</h3>
|
|
208
|
+
<div class="value {{#if manifest.metrics.error_responses}}warning{{/if}}">{{manifest.metrics.error_responses}}</div>
|
|
209
|
+
</div>
|
|
210
|
+
</div>
|
|
211
|
+
|
|
212
|
+
{{#if manifest.metrics.unsafe_responses}}
|
|
213
|
+
<div class="card" style="margin-bottom: 2rem; border-left: 4px solid #ef4444;">
|
|
214
|
+
<h3>Severity Breakdown</h3>
|
|
215
|
+
<div class="severity-breakdown">
|
|
216
|
+
{{#if manifest.metrics.by_severity.critical}}
|
|
217
|
+
<div class="severity-item">
|
|
218
|
+
<span class="severity-dot critical"></span>
|
|
219
|
+
<span>Critical: {{manifest.metrics.by_severity.critical}}</span>
|
|
220
|
+
</div>
|
|
221
|
+
{{/if}}
|
|
222
|
+
{{#if manifest.metrics.by_severity.high}}
|
|
223
|
+
<div class="severity-item">
|
|
224
|
+
<span class="severity-dot high"></span>
|
|
225
|
+
<span>High: {{manifest.metrics.by_severity.high}}</span>
|
|
226
|
+
</div>
|
|
227
|
+
{{/if}}
|
|
228
|
+
{{#if manifest.metrics.by_severity.medium}}
|
|
229
|
+
<div class="severity-item">
|
|
230
|
+
<span class="severity-dot medium"></span>
|
|
231
|
+
<span>Medium: {{manifest.metrics.by_severity.medium}}</span>
|
|
232
|
+
</div>
|
|
233
|
+
{{/if}}
|
|
234
|
+
{{#if manifest.metrics.by_severity.low}}
|
|
235
|
+
<div class="severity-item">
|
|
236
|
+
<span class="severity-dot low"></span>
|
|
237
|
+
<span>Low: {{manifest.metrics.by_severity.low}}</span>
|
|
238
|
+
</div>
|
|
239
|
+
{{/if}}
|
|
240
|
+
</div>
|
|
241
|
+
</div>
|
|
242
|
+
{{/if}}
|
|
243
|
+
|
|
244
|
+
<h2>Configuration</h2>
|
|
245
|
+
<div class="card">
|
|
246
|
+
<p><strong>Mutations:</strong> {{join manifest.config.mutations ', '}}</p>
|
|
247
|
+
<p><strong>Count per case:</strong> {{manifest.config.count_per_case}}</p>
|
|
248
|
+
</div>
|
|
249
|
+
|
|
250
|
+
{{#if manifest.resolved_config}}
|
|
251
|
+
<h2>Resolved Configuration</h2>
|
|
252
|
+
<div class="card">
|
|
253
|
+
<p><strong>Provider:</strong> {{manifest.resolved_config.provider}} <span class="source-badge">{{manifest.resolved_config.source.provider}}</span></p>
|
|
254
|
+
{{#if manifest.resolved_config.model}}
|
|
255
|
+
<p><strong>Model:</strong> {{manifest.resolved_config.model}} <span class="source-badge">{{manifest.resolved_config.source.model}}</span></p>
|
|
256
|
+
{{/if}}
|
|
257
|
+
{{#if manifest.resolved_config.deployment_name}}
|
|
258
|
+
<p><strong>Deployment:</strong> {{manifest.resolved_config.deployment_name}} <span class="source-badge">{{manifest.resolved_config.source.deployment_name}}</span></p>
|
|
259
|
+
{{/if}}
|
|
260
|
+
{{#if manifest.resolved_config.resource_name}}
|
|
261
|
+
<p><strong>Resource:</strong> {{manifest.resolved_config.resource_name}} <span class="source-badge">{{manifest.resolved_config.source.resource_name}}</span></p>
|
|
262
|
+
{{/if}}
|
|
263
|
+
{{#if manifest.resolved_config.api_version}}
|
|
264
|
+
<p><strong>API Version:</strong> {{manifest.resolved_config.api_version}} <span class="source-badge">{{manifest.resolved_config.source.api_version}}</span></p>
|
|
265
|
+
{{/if}}
|
|
266
|
+
{{#if manifest.resolved_config.base_url}}
|
|
267
|
+
<p><strong>Base URL:</strong> {{manifest.resolved_config.base_url}} <span class="source-badge">{{manifest.resolved_config.source.base_url}}</span></p>
|
|
268
|
+
{{/if}}
|
|
269
|
+
{{#if manifest.resolved_config.temperature}}
|
|
270
|
+
<p><strong>Temperature:</strong> {{manifest.resolved_config.temperature}} <span class="source-badge">{{manifest.resolved_config.source.temperature}}</span></p>
|
|
271
|
+
{{/if}}
|
|
272
|
+
</div>
|
|
273
|
+
{{/if}}
|
|
274
|
+
|
|
275
|
+
<h2>Test Results</h2>
|
|
276
|
+
<table>
|
|
277
|
+
<thead>
|
|
278
|
+
<tr>
|
|
279
|
+
<th>Case ID</th>
|
|
280
|
+
<th>Mutation</th>
|
|
281
|
+
<th>Status</th>
|
|
282
|
+
<th>Details</th>
|
|
283
|
+
</tr>
|
|
284
|
+
</thead>
|
|
285
|
+
<tbody>
|
|
286
|
+
{{#each manifest.results}}
|
|
287
|
+
<tr class="expandable" onclick="toggleDetails('{{@index}}')">
|
|
288
|
+
<td><strong>{{caseId}}</strong>{{#if redaction.redacted}}<span class="redacted-badge">redacted</span>{{/if}}</td>
|
|
289
|
+
<td><span class="mutation-tag">{{mutation}}</span></td>
|
|
290
|
+
<td>
|
|
291
|
+
<span class="status {{status}}">{{upperCase status}}</span>
|
|
292
|
+
{{#if (eq status 'unsafe')}}<span class="severity {{severity}}">{{severity}}</span>{{/if}}
|
|
293
|
+
</td>
|
|
294
|
+
<td>{{#if reasons}}{{first reasons}}{{else}}-{{/if}}</td>
|
|
295
|
+
</tr>
|
|
296
|
+
<tr id="details-{{@index}}" class="hidden">
|
|
297
|
+
<td colspan="4">
|
|
298
|
+
<div class="details">
|
|
299
|
+
<p><strong>Mutated Prompt:</strong>{{#if redaction.promptRedacted}} <span class="redacted-badge">redacted</span>{{/if}}</p>
|
|
300
|
+
<pre>{{prompt}}</pre>
|
|
301
|
+
{{#if response}}
|
|
302
|
+
<p style="margin-top: 1rem;"><strong>Response:</strong>{{#if redaction.responseRedacted}} <span class="redacted-badge">redacted</span>{{/if}}</p>
|
|
303
|
+
<pre>{{response}}</pre>
|
|
304
|
+
{{/if}}
|
|
305
|
+
{{#if reasons.length}}
|
|
306
|
+
<p style="margin-top: 1rem;"><strong>Reasons:</strong></p>
|
|
307
|
+
<ul>
|
|
308
|
+
{{#each reasons}}
|
|
309
|
+
<li>{{this}}</li>
|
|
310
|
+
{{/each}}
|
|
311
|
+
</ul>
|
|
312
|
+
{{/if}}
|
|
313
|
+
</div>
|
|
314
|
+
</td>
|
|
315
|
+
</tr>
|
|
316
|
+
{{/each}}
|
|
317
|
+
</tbody>
|
|
318
|
+
</table>
|
|
319
|
+
|
|
320
|
+
<h2>Provenance</h2>
|
|
321
|
+
<div class="card">
|
|
322
|
+
<p><strong>Git Commit:</strong> {{manifest.git.commit}}</p>
|
|
323
|
+
<p><strong>Git Branch:</strong> {{manifest.git.branch}}</p>
|
|
324
|
+
<p><strong>Run By:</strong> {{manifest.provenance.run_by}}</p>
|
|
325
|
+
<p><strong>Duration:</strong> {{manifest.duration_ms}}ms</p>
|
|
326
|
+
</div>
|
|
327
|
+
|
|
328
|
+
<footer>
|
|
329
|
+
Generated by Artemis Agent Reliability Toolkit - Red Team Module
|
|
330
|
+
</footer>
|
|
331
|
+
</div>
|
|
332
|
+
|
|
333
|
+
<script>
|
|
334
|
+
function toggleDetails(id) {
|
|
335
|
+
const details = document.getElementById('details-' + id);
|
|
336
|
+
details.classList.toggle('hidden');
|
|
337
|
+
}
|
|
338
|
+
</script>
|
|
339
|
+
</body>
|
|
340
|
+
</html>
|
|
341
|
+
`;
|
|
342
|
+
|
|
343
|
+
export function generateRedTeamHTMLReport(manifest: RedTeamManifest): string {
|
|
344
|
+
// Register helpers
|
|
345
|
+
Handlebars.registerHelper('formatPercent', (value: number) => {
|
|
346
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
Handlebars.registerHelper('formatPercentRaw', (value: number) => {
|
|
350
|
+
return `${(value * 100).toFixed(0)}%`;
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
Handlebars.registerHelper('formatDate', (value: string) => {
|
|
354
|
+
return new Date(value).toLocaleString();
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
Handlebars.registerHelper('defenseRateClass', (value: number) => {
|
|
358
|
+
if (value >= 0.95) return 'success';
|
|
359
|
+
if (value >= 0.8) return 'warning';
|
|
360
|
+
return 'error';
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
Handlebars.registerHelper('upperCase', (value: string) => {
|
|
364
|
+
return value.toUpperCase();
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
Handlebars.registerHelper('join', (arr: string[], separator: string) => {
|
|
368
|
+
return arr.join(separator);
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
Handlebars.registerHelper('first', (arr: string[]) => {
|
|
372
|
+
return arr[0] || '';
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
Handlebars.registerHelper('eq', (a: string, b: string) => {
|
|
376
|
+
return a === b;
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
Handlebars.registerHelper('subtract', (a: number, b: number) => {
|
|
380
|
+
return a - b;
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
const template = Handlebars.compile(HTML_TEMPLATE);
|
|
384
|
+
return template({ manifest });
|
|
385
|
+
}
|