clawarmor 3.5.1 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +29 -0
- package/cli.js +24 -1
- package/lib/report-compare.js +350 -0
- package/package.json +1 -1
- package/test-report-compare.js +158 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [3.6.0] — 2026-03-08
|
|
4
|
+
|
|
5
|
+
### New Features
|
|
6
|
+
|
|
7
|
+
#### `clawarmor report --compare` — Security Drift Detection
|
|
8
|
+
Diff two ClawArmor report JSON files (scan or harden) to track security posture changes over time.
|
|
9
|
+
|
|
10
|
+
**Usage:**
|
|
11
|
+
```bash
|
|
12
|
+
clawarmor report --compare baseline.json current.json
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
**Output sections:**
|
|
16
|
+
- **Regressions** (red) — was PASS, now FAIL/WARN — the important ones
|
|
17
|
+
- **Improvements** (green) — was FAIL/WARN, now PASS
|
|
18
|
+
- **New Issues** — check ID in current but not in baseline
|
|
19
|
+
- **Resolved** — check ID in baseline but no longer present
|
|
20
|
+
- **Unchanged** — count only
|
|
21
|
+
|
|
22
|
+
Score delta shown when available: `Score: 72 → 85 (+13)`
|
|
23
|
+
|
|
24
|
+
**CI-safe exit codes:** `0` = no regressions, `1` = regressions found.
|
|
25
|
+
|
|
26
|
+
Handles mismatched report types (harden vs scan) with a warning but continues.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
3
30
|
## [3.4.0] — 2026-03-08
|
|
4
31
|
|
|
5
32
|
### New Features
|
package/README.md
CHANGED
|
@@ -101,6 +101,7 @@ clawarmor invariant status # check what's deployed
|
|
|
101
101
|
|---|---|
|
|
102
102
|
| `trend` | ASCII chart of your security score over time |
|
|
103
103
|
| `compare` | Compare coverage vs openclaw security audit |
|
|
104
|
+
| `report --compare` | Diff two report JSON files — show security drift over time (v3.6.0) |
|
|
104
105
|
| `log` | View the audit event log |
|
|
105
106
|
| `digest` | Show weekly security digest |
|
|
106
107
|
| `watch` | Monitor config and skill changes in real time |
|
|
@@ -193,6 +194,34 @@ Example JSON structure:
|
|
|
193
194
|
|
|
194
195
|
Terminal output is still shown when `--report` is used — the flag only adds file output on top.
|
|
195
196
|
|
|
197
|
+
**Report comparison / security drift** (v3.6.0) — Diff two ClawArmor report files to see what changed:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
# Compare two scan reports and show what got worse (regressions), what improved, and new/resolved issues
|
|
201
|
+
clawarmor report --compare ~/.openclaw/clawarmor-scan-report-2026-03-01.json \
|
|
202
|
+
~/.openclaw/clawarmor-scan-report-2026-03-08.json
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Output sections:
|
|
206
|
+
- **Regressions** (red) — checks that were PASS and are now FAIL or WARN
|
|
207
|
+
- **Improvements** (green) — checks that were FAIL/WARN and are now PASS
|
|
208
|
+
- **New Issues** — check IDs in the current report but not in the baseline
|
|
209
|
+
- **Resolved** — check IDs in baseline but no longer present (and they were failing)
|
|
210
|
+
- **Unchanged** — count only, not listed
|
|
211
|
+
|
|
212
|
+
Score delta is shown when available: `Score: 72 → 85 (+13)`
|
|
213
|
+
|
|
214
|
+
Works with both scan reports and harden reports. Shows a warning when comparing different report types.
|
|
215
|
+
|
|
216
|
+
**CI-safe exit codes:**
|
|
217
|
+
- Exit `0` — no regressions (safe to merge/deploy)
|
|
218
|
+
- Exit `1` — one or more regressions detected
|
|
219
|
+
|
|
220
|
+
Example CI usage:
|
|
221
|
+
```bash
|
|
222
|
+
clawarmor report --compare baseline.json current.json || exit 1
|
|
223
|
+
```
|
|
224
|
+
|
|
196
225
|
## Philosophy
|
|
197
226
|
|
|
198
227
|
ClawArmor runs entirely on your machine — no telemetry, no cloud, no accounts.
|
package/cli.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import { paint } from './lib/output/colors.js';
|
|
5
5
|
|
|
6
|
-
const VERSION = '3.
|
|
6
|
+
const VERSION = '3.6.0';
|
|
7
7
|
const GATEWAY_PORT_DEFAULT = 18789;
|
|
8
8
|
|
|
9
9
|
function isLocalhost(host) {
|
|
@@ -170,6 +170,29 @@ if (cmd === 'compare') {
|
|
|
170
170
|
process.exit(await runCompare());
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
if (cmd === 'report') {
|
|
174
|
+
const compareIdx = args.indexOf('--compare');
|
|
175
|
+
if (compareIdx !== -1) {
|
|
176
|
+
const file1 = args[compareIdx + 1];
|
|
177
|
+
const file2 = args[compareIdx + 2];
|
|
178
|
+
const { runReportCompare } = await import('./lib/report-compare.js');
|
|
179
|
+
process.exit(await runReportCompare(file1, file2));
|
|
180
|
+
}
|
|
181
|
+
// Default: show usage for report subcommand
|
|
182
|
+
console.log('');
|
|
183
|
+
console.log(` ${paint.bold('report')} — report management`);
|
|
184
|
+
console.log('');
|
|
185
|
+
console.log(` ${paint.cyan('Usage:')}`);
|
|
186
|
+
console.log(` clawarmor report --compare <baseline.json> <current.json>`);
|
|
187
|
+
console.log('');
|
|
188
|
+
console.log(` ${paint.dim('Flags:')}`);
|
|
189
|
+
console.log(` ${paint.dim('--compare <file1> <file2>')} Diff two report JSON files, show security drift`);
|
|
190
|
+
console.log('');
|
|
191
|
+
console.log(` ${paint.dim('Exit codes:')} 0 = no regressions, 1 = regressions found (CI-safe)`);
|
|
192
|
+
console.log('');
|
|
193
|
+
process.exit(0);
|
|
194
|
+
}
|
|
195
|
+
|
|
173
196
|
|
|
174
197
|
if (cmd === 'fix') {
|
|
175
198
|
const { runFix } = await import('./lib/fix.js');
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
// ClawArmor v3.6.0 — report --compare command
|
|
2
|
+
// Diffs two ClawArmor report JSON files to show security drift over time.
|
|
3
|
+
|
|
4
|
+
import { readFileSync, existsSync } from 'fs';
|
|
5
|
+
import { paint, severityColor } from './output/colors.js';
|
|
6
|
+
|
|
7
|
+
const SEP = paint.dim('─'.repeat(60));
|
|
8
|
+
const SEP_SHORT = paint.dim('─'.repeat(40));
|
|
9
|
+
|
|
10
|
+
function box(title) {
|
|
11
|
+
const W = 60, pad = W - 2 - title.length;
|
|
12
|
+
const l = Math.floor(pad / 2), r = pad - l;
|
|
13
|
+
return [
|
|
14
|
+
paint.dim('╔' + '═'.repeat(W - 2) + '╗'),
|
|
15
|
+
paint.dim('║') + ' '.repeat(l) + paint.bold(title) + ' '.repeat(r) + paint.dim('║'),
|
|
16
|
+
paint.dim('╚' + '═'.repeat(W - 2) + '╝'),
|
|
17
|
+
].join('\n');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function formatTimestamp(ts) {
|
|
21
|
+
if (!ts) return 'unknown';
|
|
22
|
+
try {
|
|
23
|
+
return new Date(ts).toLocaleString('en-US', {
|
|
24
|
+
year: 'numeric', month: 'short', day: 'numeric',
|
|
25
|
+
hour: '2-digit', minute: '2-digit', hour12: false,
|
|
26
|
+
});
|
|
27
|
+
} catch {
|
|
28
|
+
return ts;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function getScore(report) {
|
|
33
|
+
// scan reports have top-level score, harden reports don't
|
|
34
|
+
if (typeof report.score === 'number') return report.score;
|
|
35
|
+
const summary = report.summary || {};
|
|
36
|
+
if (typeof summary.score === 'number') return summary.score;
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function getReportType(report) {
|
|
41
|
+
// Detect report type from command field or structure
|
|
42
|
+
if (report.command) return report.command;
|
|
43
|
+
if (Array.isArray(report.checks)) return 'scan';
|
|
44
|
+
if (Array.isArray(report.items)) return 'harden';
|
|
45
|
+
return 'unknown';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Normalize checks from any report format into a common shape:
|
|
50
|
+
* { id, name, status, severity, detail }
|
|
51
|
+
*
|
|
52
|
+
* Scan reports: checks[] with { name, status, severity, detail }
|
|
53
|
+
* Harden reports: items[] with { check, status, action }
|
|
54
|
+
*/
|
|
55
|
+
function normalizeChecks(report) {
|
|
56
|
+
const checks = [];
|
|
57
|
+
|
|
58
|
+
// Scan report format
|
|
59
|
+
if (Array.isArray(report.checks)) {
|
|
60
|
+
for (const c of report.checks) {
|
|
61
|
+
const id = c.id || c.name;
|
|
62
|
+
checks.push({
|
|
63
|
+
id,
|
|
64
|
+
name: c.name || id,
|
|
65
|
+
status: c.status, // 'pass' | 'warn' | 'block' | 'info'
|
|
66
|
+
severity: c.severity || 'NONE',
|
|
67
|
+
detail: c.detail || '',
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Harden report format
|
|
73
|
+
if (Array.isArray(report.items)) {
|
|
74
|
+
for (const item of report.items) {
|
|
75
|
+
const id = item.check || item.id;
|
|
76
|
+
checks.push({
|
|
77
|
+
id,
|
|
78
|
+
name: item.check || id,
|
|
79
|
+
status: item.status, // 'hardened' | 'already_good' | 'skipped' | 'failed'
|
|
80
|
+
severity: item.severity || 'NONE',
|
|
81
|
+
detail: item.action || item.detail || '',
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return checks;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function isPass(status) {
|
|
90
|
+
return status === 'pass' || status === 'already_good';
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function isFail(status) {
|
|
94
|
+
return status === 'block' || status === 'fail' || status === 'failed';
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function isWarn(status) {
|
|
98
|
+
return status === 'warn' || status === 'skipped';
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function statusLabel(status) {
|
|
102
|
+
if (isPass(status)) return paint.green(status.toUpperCase());
|
|
103
|
+
if (isFail(status)) return paint.red(status.toUpperCase());
|
|
104
|
+
if (isWarn(status)) return paint.yellow(status.toUpperCase());
|
|
105
|
+
return paint.dim(status.toUpperCase());
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Core diff logic — exported for unit testing.
|
|
110
|
+
*
|
|
111
|
+
* Returns:
|
|
112
|
+
* {
|
|
113
|
+
* regressions: [{ id, name, oldStatus, newStatus, severity, detail }]
|
|
114
|
+
* improvements: [{ id, name, oldStatus, newStatus, severity, detail }]
|
|
115
|
+
* newIssues: [{ id, name, status, severity, detail }]
|
|
116
|
+
* resolved: [{ id, name, status, severity, detail }]
|
|
117
|
+
* unchanged: number
|
|
118
|
+
* scoreOld: number|null
|
|
119
|
+
* scoreNew: number|null
|
|
120
|
+
* }
|
|
121
|
+
*/
|
|
122
|
+
export function diffReports(report1, report2) {
|
|
123
|
+
const checks1 = normalizeChecks(report1);
|
|
124
|
+
const checks2 = normalizeChecks(report2);
|
|
125
|
+
|
|
126
|
+
const map1 = new Map(checks1.map(c => [c.id, c]));
|
|
127
|
+
const map2 = new Map(checks2.map(c => [c.id, c]));
|
|
128
|
+
|
|
129
|
+
const regressions = [];
|
|
130
|
+
const improvements = [];
|
|
131
|
+
const newIssues = [];
|
|
132
|
+
const resolved = [];
|
|
133
|
+
let unchanged = 0;
|
|
134
|
+
|
|
135
|
+
// Check all IDs from file2
|
|
136
|
+
for (const [id, c2] of map2) {
|
|
137
|
+
if (!map1.has(id)) {
|
|
138
|
+
// New check in file2
|
|
139
|
+
if (!isPass(c2.status)) {
|
|
140
|
+
newIssues.push(c2);
|
|
141
|
+
}
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const c1 = map1.get(id);
|
|
146
|
+
if (isPass(c1.status) && !isPass(c2.status)) {
|
|
147
|
+
// Was passing, now failing/warning → regression
|
|
148
|
+
regressions.push({
|
|
149
|
+
id,
|
|
150
|
+
name: c2.name,
|
|
151
|
+
oldStatus: c1.status,
|
|
152
|
+
newStatus: c2.status,
|
|
153
|
+
severity: c2.severity,
|
|
154
|
+
detail: c2.detail,
|
|
155
|
+
});
|
|
156
|
+
} else if (!isPass(c1.status) && isPass(c2.status)) {
|
|
157
|
+
// Was failing/warning, now passing → improvement
|
|
158
|
+
improvements.push({
|
|
159
|
+
id,
|
|
160
|
+
name: c2.name,
|
|
161
|
+
oldStatus: c1.status,
|
|
162
|
+
newStatus: c2.status,
|
|
163
|
+
severity: c1.severity,
|
|
164
|
+
detail: c2.detail,
|
|
165
|
+
});
|
|
166
|
+
} else {
|
|
167
|
+
unchanged++;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Checks only in file1 (resolved / no longer present)
|
|
172
|
+
for (const [id, c1] of map1) {
|
|
173
|
+
if (!map2.has(id) && !isPass(c1.status)) {
|
|
174
|
+
resolved.push(c1);
|
|
175
|
+
} else if (!map2.has(id) && isPass(c1.status)) {
|
|
176
|
+
// Was passing, no longer checked — just ignore (counts as unchanged context)
|
|
177
|
+
unchanged++;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
regressions,
|
|
183
|
+
improvements,
|
|
184
|
+
newIssues,
|
|
185
|
+
resolved,
|
|
186
|
+
unchanged,
|
|
187
|
+
scoreOld: getScore(report1),
|
|
188
|
+
scoreNew: getScore(report2),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export async function runReportCompare(file1, file2) {
|
|
193
|
+
// ── Validate inputs ──────────────────────────────────────────────────────────
|
|
194
|
+
if (!file1 || !file2) {
|
|
195
|
+
console.error('');
|
|
196
|
+
console.error(` ${paint.red('✗')} Usage: clawarmor report --compare <file1> <file2>`);
|
|
197
|
+
console.error(` ${paint.dim('Example: clawarmor report --compare old-report.json new-report.json')}`);
|
|
198
|
+
console.error('');
|
|
199
|
+
return 1;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (!existsSync(file1)) {
|
|
203
|
+
console.error('');
|
|
204
|
+
console.error(` ${paint.red('✗')} File not found: ${file1}`);
|
|
205
|
+
console.error('');
|
|
206
|
+
return 1;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (!existsSync(file2)) {
|
|
210
|
+
console.error('');
|
|
211
|
+
console.error(` ${paint.red('✗')} File not found: ${file2}`);
|
|
212
|
+
console.error('');
|
|
213
|
+
return 1;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
let report1, report2;
|
|
217
|
+
try {
|
|
218
|
+
report1 = JSON.parse(readFileSync(file1, 'utf8'));
|
|
219
|
+
} catch (e) {
|
|
220
|
+
console.error('');
|
|
221
|
+
console.error(` ${paint.red('✗')} Could not parse ${file1}: ${e.message}`);
|
|
222
|
+
console.error('');
|
|
223
|
+
return 1;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
try {
|
|
227
|
+
report2 = JSON.parse(readFileSync(file2, 'utf8'));
|
|
228
|
+
} catch (e) {
|
|
229
|
+
console.error('');
|
|
230
|
+
console.error(` ${paint.red('✗')} Could not parse ${file2}: ${e.message}`);
|
|
231
|
+
console.error('');
|
|
232
|
+
return 1;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// ── Header ───────────────────────────────────────────────────────────────────
|
|
236
|
+
console.log('');
|
|
237
|
+
console.log(box('ClawArmor — Security Drift Report'));
|
|
238
|
+
console.log('');
|
|
239
|
+
|
|
240
|
+
const type1 = getReportType(report1);
|
|
241
|
+
const type2 = getReportType(report2);
|
|
242
|
+
|
|
243
|
+
if (type1 !== type2) {
|
|
244
|
+
console.log(` ${paint.yellow('⚠')} Report types differ: ${paint.bold(type1)} vs ${paint.bold(type2)}`);
|
|
245
|
+
console.log(` ${paint.dim('Comparison may be incomplete — check IDs may not align.')}`);
|
|
246
|
+
console.log('');
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
console.log(` ${paint.dim('Baseline:')} ${formatTimestamp(report1.timestamp)} ${paint.dim('(' + file1 + ')')}`);
|
|
250
|
+
console.log(` ${paint.dim('Current:')} ${formatTimestamp(report2.timestamp)} ${paint.dim('(' + file2 + ')')}`);
|
|
251
|
+
console.log('');
|
|
252
|
+
|
|
253
|
+
// ── Score delta ───────────────────────────────────────────────────────────────
|
|
254
|
+
const diff = diffReports(report1, report2);
|
|
255
|
+
|
|
256
|
+
if (diff.scoreOld !== null && diff.scoreNew !== null) {
|
|
257
|
+
const delta = diff.scoreNew - diff.scoreOld;
|
|
258
|
+
const deltaStr = delta > 0 ? paint.green(`+${delta}`) : delta < 0 ? paint.red(`${delta}`) : paint.dim('±0');
|
|
259
|
+
const oldColor = diff.scoreOld >= 80 ? paint.green : diff.scoreOld >= 60 ? paint.yellow : paint.red;
|
|
260
|
+
const newColor = diff.scoreNew >= 80 ? paint.green : diff.scoreNew >= 60 ? paint.yellow : paint.red;
|
|
261
|
+
console.log(` ${paint.bold('Score:')} ${oldColor(String(diff.scoreOld))} → ${newColor(String(diff.scoreNew))} (${deltaStr})`);
|
|
262
|
+
console.log('');
|
|
263
|
+
} else if (diff.scoreOld !== null || diff.scoreNew !== null) {
|
|
264
|
+
const s = diff.scoreOld ?? diff.scoreNew;
|
|
265
|
+
console.log(` ${paint.bold('Score:')} ${paint.dim('N/A')} → ${s} ${paint.dim('(only one report has a score)')}`);
|
|
266
|
+
console.log('');
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// ── Regressions ──────────────────────────────────────────────────────────────
|
|
270
|
+
console.log(SEP);
|
|
271
|
+
if (diff.regressions.length === 0) {
|
|
272
|
+
console.log(` ${paint.green('✓')} ${paint.bold('No regressions')} — nothing got worse`);
|
|
273
|
+
} else {
|
|
274
|
+
console.log(` ${paint.red('✗')} ${paint.bold(`Regressions (${diff.regressions.length})`)} — ${paint.red('things that got worse')}`);
|
|
275
|
+
console.log(SEP);
|
|
276
|
+
for (const r of diff.regressions) {
|
|
277
|
+
const sevCol = severityColor[r.severity] || paint.dim;
|
|
278
|
+
console.log(` ${paint.red('↓')} ${paint.bold(r.name)}`);
|
|
279
|
+
console.log(` ${paint.dim('Was:')} ${statusLabel(r.oldStatus)} ${paint.dim('→')} ${paint.dim('Now:')} ${statusLabel(r.newStatus)} ${sevCol('[' + r.severity + ']')}`);
|
|
280
|
+
if (r.detail) console.log(` ${paint.dim(r.detail)}`);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
console.log('');
|
|
284
|
+
|
|
285
|
+
// ── Improvements ─────────────────────────────────────────────────────────────
|
|
286
|
+
console.log(SEP);
|
|
287
|
+
if (diff.improvements.length === 0) {
|
|
288
|
+
console.log(` ${paint.dim('─')} No improvements`);
|
|
289
|
+
} else {
|
|
290
|
+
console.log(` ${paint.green('✓')} ${paint.bold(`Improvements (${diff.improvements.length})`)} — ${paint.green('things that got better')}`);
|
|
291
|
+
console.log(SEP);
|
|
292
|
+
for (const r of diff.improvements) {
|
|
293
|
+
const sevCol = severityColor[r.severity] || paint.dim;
|
|
294
|
+
console.log(` ${paint.green('↑')} ${paint.bold(r.name)}`);
|
|
295
|
+
console.log(` ${paint.dim('Was:')} ${statusLabel(r.oldStatus)} ${paint.dim('→')} ${paint.dim('Now:')} ${statusLabel(r.newStatus)} ${sevCol('[' + r.severity + ']')}`);
|
|
296
|
+
if (r.detail) console.log(` ${paint.dim(r.detail)}`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
console.log('');
|
|
300
|
+
|
|
301
|
+
// ── New Issues ────────────────────────────────────────────────────────────────
|
|
302
|
+
console.log(SEP);
|
|
303
|
+
if (diff.newIssues.length === 0) {
|
|
304
|
+
console.log(` ${paint.dim('─')} No new issues`);
|
|
305
|
+
} else {
|
|
306
|
+
console.log(` ${paint.yellow('!')} ${paint.bold(`New Issues (${diff.newIssues.length})`)} — ${paint.yellow('checks not in baseline report')}`);
|
|
307
|
+
console.log(SEP);
|
|
308
|
+
for (const r of diff.newIssues) {
|
|
309
|
+
const sevCol = severityColor[r.severity] || paint.dim;
|
|
310
|
+
console.log(` ${paint.yellow('+')} ${paint.bold(r.name)} ${sevCol('[' + r.severity + ']')}`);
|
|
311
|
+
if (r.detail) console.log(` ${paint.dim(r.detail)}`);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
console.log('');
|
|
315
|
+
|
|
316
|
+
// ── Resolved ─────────────────────────────────────────────────────────────────
|
|
317
|
+
console.log(SEP);
|
|
318
|
+
if (diff.resolved.length === 0) {
|
|
319
|
+
console.log(` ${paint.dim('─')} Nothing resolved`);
|
|
320
|
+
} else {
|
|
321
|
+
console.log(` ${paint.green('✓')} ${paint.bold(`Resolved (${diff.resolved.length})`)} — ${paint.green('issues no longer present')}`);
|
|
322
|
+
console.log(SEP);
|
|
323
|
+
for (const r of diff.resolved) {
|
|
324
|
+
console.log(` ${paint.green('✓')} ${paint.bold(r.name)} ${paint.dim('[' + r.severity + ']')}`);
|
|
325
|
+
if (r.detail) console.log(` ${paint.dim(r.detail)}`);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
console.log('');
|
|
329
|
+
|
|
330
|
+
// ── Summary ───────────────────────────────────────────────────────────────────
|
|
331
|
+
console.log(SEP);
|
|
332
|
+
console.log(` ${paint.bold('Summary')}`);
|
|
333
|
+
console.log(SEP);
|
|
334
|
+
console.log(` ${paint.red('Regressions:')} ${diff.regressions.length}`);
|
|
335
|
+
console.log(` ${paint.green('Improvements:')} ${diff.improvements.length}`);
|
|
336
|
+
console.log(` ${paint.yellow('New Issues:')} ${diff.newIssues.length}`);
|
|
337
|
+
console.log(` ${paint.green('Resolved:')} ${diff.resolved.length}`);
|
|
338
|
+
console.log(` ${paint.dim('Unchanged:')} ${diff.unchanged}`);
|
|
339
|
+
console.log('');
|
|
340
|
+
|
|
341
|
+
if (diff.regressions.length > 0) {
|
|
342
|
+
console.log(` ${paint.red('⚠')} ${paint.bold(diff.regressions.length + ' regression(s) detected.')} Exit code: 1`);
|
|
343
|
+
console.log('');
|
|
344
|
+
return 1;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
console.log(` ${paint.green('✓')} ${paint.bold('No regressions.')} Exit code: 0`);
|
|
348
|
+
console.log('');
|
|
349
|
+
return 0;
|
|
350
|
+
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Unit tests for report-compare diffReports logic
|
|
3
|
+
// Run: node test-report-compare.js
|
|
4
|
+
|
|
5
|
+
import assert from 'assert';
|
|
6
|
+
import { diffReports } from './lib/report-compare.js';
|
|
7
|
+
|
|
8
|
+
let passed = 0;
|
|
9
|
+
let failed = 0;
|
|
10
|
+
|
|
11
|
+
function test(name, fn) {
|
|
12
|
+
try {
|
|
13
|
+
fn();
|
|
14
|
+
console.log(` ✓ ${name}`);
|
|
15
|
+
passed++;
|
|
16
|
+
} catch (e) {
|
|
17
|
+
console.log(` ✗ ${name}`);
|
|
18
|
+
console.log(` ${e.message}`);
|
|
19
|
+
failed++;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const baseReport = (checks, score = 80) => ({
|
|
24
|
+
version: '3.5.1',
|
|
25
|
+
timestamp: '2026-03-01T10:00:00Z',
|
|
26
|
+
score,
|
|
27
|
+
checks,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
console.log('\n ClawArmor — diffReports unit tests\n');
|
|
31
|
+
|
|
32
|
+
// ── Regressions ──────────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
test('detects regression: PASS → WARN', () => {
|
|
35
|
+
const r1 = baseReport([{ name: 'gateway', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
36
|
+
const r2 = baseReport([{ name: 'gateway', status: 'warn', severity: 'HIGH', detail: 'exposed' }]);
|
|
37
|
+
const diff = diffReports(r1, r2);
|
|
38
|
+
assert.strictEqual(diff.regressions.length, 1);
|
|
39
|
+
assert.strictEqual(diff.regressions[0].id, 'gateway');
|
|
40
|
+
assert.strictEqual(diff.regressions[0].oldStatus, 'pass');
|
|
41
|
+
assert.strictEqual(diff.regressions[0].newStatus, 'warn');
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test('detects regression: PASS → block (CRITICAL)', () => {
|
|
45
|
+
const r1 = baseReport([{ name: 'auth', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
46
|
+
const r2 = baseReport([{ name: 'auth', status: 'block', severity: 'CRITICAL', detail: 'exposed token' }]);
|
|
47
|
+
const diff = diffReports(r1, r2);
|
|
48
|
+
assert.strictEqual(diff.regressions.length, 1);
|
|
49
|
+
assert.strictEqual(diff.regressions[0].severity, 'CRITICAL');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// ── Improvements ────────────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
test('detects improvement: WARN → PASS', () => {
|
|
55
|
+
const r1 = baseReport([{ name: 'gateway', status: 'warn', severity: 'HIGH', detail: 'exposed' }]);
|
|
56
|
+
const r2 = baseReport([{ name: 'gateway', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
57
|
+
const diff = diffReports(r1, r2);
|
|
58
|
+
assert.strictEqual(diff.improvements.length, 1);
|
|
59
|
+
assert.strictEqual(diff.improvements[0].id, 'gateway');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('detects improvement: block → PASS', () => {
|
|
63
|
+
const r1 = baseReport([{ name: 'auth', status: 'block', severity: 'CRITICAL', detail: '' }]);
|
|
64
|
+
const r2 = baseReport([{ name: 'auth', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
65
|
+
const diff = diffReports(r1, r2);
|
|
66
|
+
assert.strictEqual(diff.improvements.length, 1);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
// ── New Issues ───────────────────────────────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
test('detects new issues (check in file2 but not file1)', () => {
|
|
72
|
+
const r1 = baseReport([{ name: 'old-check', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
73
|
+
const r2 = baseReport([
|
|
74
|
+
{ name: 'old-check', status: 'pass', severity: 'NONE', detail: '' },
|
|
75
|
+
{ name: 'new-check', status: 'warn', severity: 'HIGH', detail: 'new problem' },
|
|
76
|
+
]);
|
|
77
|
+
const diff = diffReports(r1, r2);
|
|
78
|
+
assert.strictEqual(diff.newIssues.length, 1);
|
|
79
|
+
assert.strictEqual(diff.newIssues[0].id, 'new-check');
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test('new passing checks are NOT in newIssues', () => {
|
|
83
|
+
const r1 = baseReport([]);
|
|
84
|
+
const r2 = baseReport([{ name: 'new-pass', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
85
|
+
const diff = diffReports(r1, r2);
|
|
86
|
+
assert.strictEqual(diff.newIssues.length, 0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
// ── Resolved ─────────────────────────────────────────────────────────────────
|
|
90
|
+
|
|
91
|
+
test('detects resolved: failing check no longer in file2', () => {
|
|
92
|
+
const r1 = baseReport([
|
|
93
|
+
{ name: 'old-issue', status: 'block', severity: 'CRITICAL', detail: 'bad' },
|
|
94
|
+
{ name: 'keep', status: 'pass', severity: 'NONE', detail: '' },
|
|
95
|
+
]);
|
|
96
|
+
const r2 = baseReport([{ name: 'keep', status: 'pass', severity: 'NONE', detail: '' }]);
|
|
97
|
+
const diff = diffReports(r1, r2);
|
|
98
|
+
assert.strictEqual(diff.resolved.length, 1);
|
|
99
|
+
assert.strictEqual(diff.resolved[0].id, 'old-issue');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// ── Unchanged ────────────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
test('unchanged count is correct', () => {
|
|
105
|
+
const r1 = baseReport([
|
|
106
|
+
{ name: 'a', status: 'pass', severity: 'NONE', detail: '' },
|
|
107
|
+
{ name: 'b', status: 'warn', severity: 'HIGH', detail: '' },
|
|
108
|
+
]);
|
|
109
|
+
const r2 = baseReport([
|
|
110
|
+
{ name: 'a', status: 'pass', severity: 'NONE', detail: '' },
|
|
111
|
+
{ name: 'b', status: 'warn', severity: 'HIGH', detail: 'same' },
|
|
112
|
+
]);
|
|
113
|
+
const diff = diffReports(r1, r2);
|
|
114
|
+
assert.strictEqual(diff.unchanged, 2);
|
|
115
|
+
assert.strictEqual(diff.regressions.length, 0);
|
|
116
|
+
assert.strictEqual(diff.improvements.length, 0);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// ── Score delta ──────────────────────────────────────────────────────────────
|
|
120
|
+
|
|
121
|
+
test('score delta is extracted', () => {
|
|
122
|
+
const r1 = baseReport([], 72);
|
|
123
|
+
const r2 = baseReport([], 85);
|
|
124
|
+
const diff = diffReports(r1, r2);
|
|
125
|
+
assert.strictEqual(diff.scoreOld, 72);
|
|
126
|
+
assert.strictEqual(diff.scoreNew, 85);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test('score null when not in report', () => {
|
|
130
|
+
const r1 = { version: '1.0', timestamp: '2026-01-01T00:00:00Z', items: [] };
|
|
131
|
+
const r2 = { version: '1.0', timestamp: '2026-01-01T00:00:00Z', items: [] };
|
|
132
|
+
const diff = diffReports(r1, r2);
|
|
133
|
+
assert.strictEqual(diff.scoreOld, null);
|
|
134
|
+
assert.strictEqual(diff.scoreNew, null);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
// ── Harden report format ─────────────────────────────────────────────────────
|
|
138
|
+
|
|
139
|
+
test('handles harden report format (items[] instead of checks[])', () => {
|
|
140
|
+
const r1 = {
|
|
141
|
+
version: '3.5.0', timestamp: '2026-03-01T00:00:00Z',
|
|
142
|
+
items: [{ check: 'cred-perms', status: 'already_good', action: 'Files are 600' }],
|
|
143
|
+
};
|
|
144
|
+
const r2 = {
|
|
145
|
+
version: '3.5.1', timestamp: '2026-03-08T00:00:00Z',
|
|
146
|
+
items: [{ check: 'cred-perms', status: 'failed', action: 'Could not fix' }],
|
|
147
|
+
};
|
|
148
|
+
const diff = diffReports(r1, r2);
|
|
149
|
+
assert.strictEqual(diff.regressions.length, 1);
|
|
150
|
+
assert.strictEqual(diff.regressions[0].id, 'cred-perms');
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
// ── Results ──────────────────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
console.log('');
|
|
156
|
+
console.log(` Results: ${passed} passed, ${failed} failed`);
|
|
157
|
+
console.log('');
|
|
158
|
+
if (failed > 0) process.exit(1);
|