llm-scanner 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/reporter.js +29 -10
- package/package.json +1 -1
package/dist/reporter.js
CHANGED
|
@@ -95,22 +95,37 @@ function printFinalReport(results, verbose, debug = false) {
|
|
|
95
95
|
console.log(chalk_1.default.bold(BAR));
|
|
96
96
|
console.log();
|
|
97
97
|
if (!debug) {
|
|
98
|
+
const grouped = new Map();
|
|
98
99
|
for (const r of results) {
|
|
99
100
|
if (r.verdict !== "FAIL")
|
|
100
101
|
continue;
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
const key = r.reason || "Model behavior indicates a potential policy bypass.";
|
|
103
|
+
if (!grouped.has(key))
|
|
104
|
+
grouped.set(key, []);
|
|
105
|
+
grouped.get(key).push(r);
|
|
106
|
+
}
|
|
107
|
+
for (const [reason, group] of grouped.entries()) {
|
|
108
|
+
const sample = group[0];
|
|
109
|
+
const confidence = confidenceForFail(reason, sample.rawResponse);
|
|
110
|
+
const reproBody = JSON.stringify({ message: sample.attack.prompt });
|
|
111
|
+
const categories = Array.from(new Set(group.map((g) => g.attack.category)));
|
|
112
|
+
const head = `${severityIcon(sample.attack.severity)} ${sample.attack.severity} — ROOT ISSUE`;
|
|
104
113
|
console.log(` ${head}`);
|
|
105
114
|
console.log();
|
|
106
|
-
console.log(" ---
|
|
107
|
-
console.log(` ${
|
|
115
|
+
console.log(" --- ISSUE ---");
|
|
116
|
+
console.log(` ${reason}`);
|
|
108
117
|
console.log();
|
|
109
|
-
console.log(" ---
|
|
110
|
-
|
|
118
|
+
console.log(" --- TRIGGERED BY ---");
|
|
119
|
+
for (const category of categories) {
|
|
120
|
+
console.log(` * ${category}`);
|
|
121
|
+
}
|
|
122
|
+
console.log();
|
|
123
|
+
console.log(" --- EXAMPLE ---");
|
|
124
|
+
console.log(" ATTACK:");
|
|
125
|
+
console.log(` ${sample.attack.prompt}`);
|
|
111
126
|
console.log();
|
|
112
|
-
console.log("
|
|
113
|
-
console.log(` ${
|
|
127
|
+
console.log(" FULL RESPONSE:");
|
|
128
|
+
console.log(` ${sample.rawResponse || "(empty)"}`);
|
|
114
129
|
console.log();
|
|
115
130
|
console.log(" --- REPRODUCE ---");
|
|
116
131
|
console.log(" curl -X POST <endpoint> \\");
|
|
@@ -170,7 +185,11 @@ function printFinalReport(results, verbose, debug = false) {
|
|
|
170
185
|
: chalk_1.default.yellow(` Score: ${score}/100 · ${label}`);
|
|
171
186
|
console.log(vulnLine);
|
|
172
187
|
console.log(fails.length > 0
|
|
173
|
-
?
|
|
188
|
+
? (() => {
|
|
189
|
+
const uniqueIssues = new Set(fails.map((r) => r.reason || "Model behavior indicates a potential policy bypass.")).size;
|
|
190
|
+
const severityLabel = uniqueIssues === 1 ? "critical vulnerability" : "critical vulnerabilities";
|
|
191
|
+
return chalk_1.default.red(` ${uniqueIssues} ${severityLabel} found (triggered by ${fails.length} tests)`);
|
|
192
|
+
})()
|
|
174
193
|
: judged === 0
|
|
175
194
|
? chalk_1.default.yellow(` All ${results.length} tests were skipped`)
|
|
176
195
|
: chalk_1.default.green(" No vulnerabilities found"));
|