llm-scanner 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +11 -5
- package/dist/reporter.js +27 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -79,8 +79,11 @@ program
|
|
|
79
79
|
.option("--fast", "Run 5 most critical attacks only")
|
|
80
80
|
.option("--dry-run", "Print first 3 attacks and bodies; do not send requests")
|
|
81
81
|
.option("--verbose", "Include raw responses in the report")
|
|
82
|
+
.option("--debug", "Show attack/response details for all tests")
|
|
82
83
|
.option("--header <header>", 'HTTP header to include, format: "Key: Value"')
|
|
83
84
|
.action(async (opts) => {
|
|
85
|
+
const args = process.argv;
|
|
86
|
+
const debug = args.includes("--debug");
|
|
84
87
|
process.env.AISEC_VERBOSE = opts.verbose ? "true" : "false";
|
|
85
88
|
if (!opts.dryRun && !opts.endpoint) {
|
|
86
89
|
console.error("error: --endpoint is required unless using --dry-run");
|
|
@@ -95,14 +98,16 @@ program
|
|
|
95
98
|
return;
|
|
96
99
|
}
|
|
97
100
|
const results = [];
|
|
98
|
-
const progress = (0, reporter_1.createProgress)(attacks.length);
|
|
101
|
+
const progress = debug ? null : (0, reporter_1.createProgress)(attacks.length);
|
|
99
102
|
let i = 0;
|
|
100
103
|
for (const attack of attacks) {
|
|
101
104
|
i += 1;
|
|
102
|
-
progress
|
|
105
|
+
if (progress)
|
|
106
|
+
progress.start(i, attack.category);
|
|
103
107
|
const call = await (0, caller_1.callEndpoint)(opts.endpoint, opts.bodyTemplate, attack.prompt, opts.responsePath, 12000, requestHeaders);
|
|
104
108
|
if (call.status === "skip") {
|
|
105
|
-
progress
|
|
109
|
+
if (progress)
|
|
110
|
+
progress.finish(i, attack.category, "SKIP");
|
|
106
111
|
results.push({
|
|
107
112
|
attack,
|
|
108
113
|
verdict: "SKIP",
|
|
@@ -113,7 +118,8 @@ program
|
|
|
113
118
|
continue;
|
|
114
119
|
}
|
|
115
120
|
const j = await (0, judge_1.judge)(attack, call.text);
|
|
116
|
-
progress
|
|
121
|
+
if (progress)
|
|
122
|
+
progress.finish(i, attack.category, j.verdict);
|
|
117
123
|
results.push({
|
|
118
124
|
attack,
|
|
119
125
|
verdict: j.verdict,
|
|
@@ -122,7 +128,7 @@ program
|
|
|
122
128
|
skipReason: j.verdict === "SKIP" ? j.reason : undefined,
|
|
123
129
|
});
|
|
124
130
|
}
|
|
125
|
-
(0, reporter_1.printFinalReport)(results, Boolean(opts.verbose));
|
|
131
|
+
(0, reporter_1.printFinalReport)(results, Boolean(opts.verbose), debug);
|
|
126
132
|
const failures = results.filter((r) => r.verdict === "FAIL").length;
|
|
127
133
|
process.exit(failures > 0 ? 1 : 0);
|
|
128
134
|
});
|
package/dist/reporter.js
CHANGED
|
@@ -81,7 +81,7 @@ function confidenceForFail(reason, rawResponse) {
|
|
|
81
81
|
];
|
|
82
82
|
return highSignals.some((s) => signal.includes(s)) ? "HIGH" : "MEDIUM";
|
|
83
83
|
}
|
|
84
|
-
function printFinalReport(results, verbose) {
|
|
84
|
+
function printFinalReport(results, verbose, debug = false) {
|
|
85
85
|
const fails = results.filter((r) => r.verdict === "FAIL");
|
|
86
86
|
const passes = results.filter((r) => r.verdict === "PASS").length;
|
|
87
87
|
const skipped = results.filter((r) => r.verdict === "SKIP").length;
|
|
@@ -118,6 +118,32 @@ function printFinalReport(results, verbose) {
|
|
|
118
118
|
console.log(` ${confidence}`);
|
|
119
119
|
console.log();
|
|
120
120
|
}
|
|
121
|
+
if (debug) {
|
|
122
|
+
for (const r of results) {
|
|
123
|
+
const verdictTag = `[${r.verdict}] ${r.attack.category}`;
|
|
124
|
+
console.log(` ${verdictTag}`);
|
|
125
|
+
console.log();
|
|
126
|
+
console.log(" --- ATTACK ---");
|
|
127
|
+
console.log(` ${r.attack.prompt}`);
|
|
128
|
+
console.log();
|
|
129
|
+
console.log(" --- RESPONSE ---");
|
|
130
|
+
console.log(` ${r.rawResponse || "(empty)"}`);
|
|
131
|
+
console.log();
|
|
132
|
+
console.log(" --- NOTE ---");
|
|
133
|
+
if (r.verdict === "PASS") {
|
|
134
|
+
console.log(" Model correctly handled the attack (refused or ignored malicious input)");
|
|
135
|
+
}
|
|
136
|
+
else if (r.verdict === "FAIL") {
|
|
137
|
+
console.log(` ${r.reason || "Model behavior indicates a potential policy bypass."}`);
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
console.log(` ${r.reason || "Test skipped due to request/judge error."}`);
|
|
141
|
+
}
|
|
142
|
+
console.log();
|
|
143
|
+
console.log(" ---");
|
|
144
|
+
console.log();
|
|
145
|
+
}
|
|
146
|
+
}
|
|
121
147
|
if (verbose) {
|
|
122
148
|
console.log(chalk_1.default.bold(" Raw responses"));
|
|
123
149
|
for (const r of results) {
|