llm-scanner 0.1.14 โ 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/reporter.js +49 -10
- package/package.json +1 -1
package/dist/reporter.js
CHANGED
|
@@ -95,22 +95,57 @@ function printFinalReport(results, verbose, debug = false) {
|
|
|
95
95
|
console.log(chalk_1.default.bold(BAR));
|
|
96
96
|
console.log();
|
|
97
97
|
if (!debug) {
|
|
98
|
+
const grouped = new Map();
|
|
98
99
|
for (const r of results) {
|
|
99
100
|
if (r.verdict !== "FAIL")
|
|
100
101
|
continue;
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
102
|
+
const key = r.reason || "Model behavior indicates a potential policy bypass.";
|
|
103
|
+
if (!grouped.has(key))
|
|
104
|
+
grouped.set(key, []);
|
|
105
|
+
grouped.get(key).push(r);
|
|
106
|
+
}
|
|
107
|
+
for (const [reason, group] of grouped.entries()) {
|
|
108
|
+
const sample = group[0];
|
|
109
|
+
const confidence = confidenceForFail(reason, sample.rawResponse);
|
|
110
|
+
const reproBody = JSON.stringify({ message: sample.attack.prompt });
|
|
111
|
+
const categories = Array.from(new Set(group.map((g) => g.attack.category)));
|
|
112
|
+
const isSystemPromptLeak = reason.toLowerCase().includes("system_prompt");
|
|
113
|
+
const title = isSystemPromptLeak
|
|
114
|
+
? "Internal system prompt exposed in API response"
|
|
115
|
+
: "Critical security issue detected in API response";
|
|
116
|
+
console.log(` ${chalk_1.default.red("๐ด CRITICAL ISSUE DETECTED")}`);
|
|
105
117
|
console.log();
|
|
106
|
-
console.log(
|
|
107
|
-
console.log(` ${
|
|
118
|
+
console.log(` ${chalk_1.default.bold("Title:")}`);
|
|
119
|
+
console.log(` ${title}`);
|
|
108
120
|
console.log();
|
|
109
|
-
console.log(
|
|
110
|
-
|
|
121
|
+
console.log(` ${chalk_1.default.bold("Explanation:")}`);
|
|
122
|
+
if (isSystemPromptLeak) {
|
|
123
|
+
console.log(' Your API is returning internal system instructions ("system_prompt") to the user.');
|
|
124
|
+
console.log(" This is sensitive data and should never be exposed.");
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
console.log(` ${reason}`);
|
|
128
|
+
}
|
|
129
|
+
console.log();
|
|
130
|
+
console.log(` ${chalk_1.default.bold("Impact:")}`);
|
|
131
|
+
console.log(" * Users can see hidden instructions");
|
|
132
|
+
console.log(" * Attackers can reverse engineer behavior");
|
|
133
|
+
console.log(" * May weaken safety protections");
|
|
134
|
+
console.log();
|
|
135
|
+
console.log(" --- TRIGGERED BY ---");
|
|
136
|
+
for (const category of categories) {
|
|
137
|
+
console.log(` * ${category}`);
|
|
138
|
+
}
|
|
139
|
+
console.log();
|
|
140
|
+
console.log(" --- EXAMPLE ---");
|
|
141
|
+
console.log(" ATTACK:");
|
|
142
|
+
console.log(` ${sample.attack.prompt}`);
|
|
143
|
+
console.log();
|
|
144
|
+
console.log(" FULL RESPONSE:");
|
|
145
|
+
console.log(` ${sample.rawResponse || "(empty)"}`);
|
|
111
146
|
console.log();
|
|
112
147
|
console.log(" --- EVIDENCE ---");
|
|
113
|
-
console.log(` ${
|
|
148
|
+
console.log(` ${reason}`);
|
|
114
149
|
console.log();
|
|
115
150
|
console.log(" --- REPRODUCE ---");
|
|
116
151
|
console.log(" curl -X POST <endpoint> \\");
|
|
@@ -170,7 +205,11 @@ function printFinalReport(results, verbose, debug = false) {
|
|
|
170
205
|
: chalk_1.default.yellow(` Score: ${score}/100 ยท ${label}`);
|
|
171
206
|
console.log(vulnLine);
|
|
172
207
|
console.log(fails.length > 0
|
|
173
|
-
?
|
|
208
|
+
? (() => {
|
|
209
|
+
const uniqueIssues = new Set(fails.map((r) => r.reason || "Model behavior indicates a potential policy bypass.")).size;
|
|
210
|
+
const severityLabel = uniqueIssues === 1 ? "critical vulnerability" : "critical vulnerabilities";
|
|
211
|
+
return chalk_1.default.red(` ${uniqueIssues} ${severityLabel} found (triggered by ${fails.length} tests)`);
|
|
212
|
+
})()
|
|
174
213
|
: judged === 0
|
|
175
214
|
? chalk_1.default.yellow(` All ${results.length} tests were skipped`)
|
|
176
215
|
: chalk_1.default.green(" No vulnerabilities found"));
|