@kevinrabun/judges 3.126.0 → 3.126.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/llm-benchmark.js +19 -13
- package/package.json +1 -1
- package/server.json +2 -2
|
@@ -148,13 +148,15 @@ export function parseLlmRuleIds(response) {
|
|
|
148
148
|
const validPrefixes = getValidRulePrefixes();
|
|
149
149
|
const pattern = /\b([A-Z][A-Z0-9]+)-(\d{1,3})\b/g;
|
|
150
150
|
const found = new Set();
|
|
151
|
-
// Split response into
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
const
|
|
151
|
+
// Split response into judge sections (separated by --- dividers or ## headers)
|
|
152
|
+
// and skip entire sections that declare zero/no findings. This prevents rule
|
|
153
|
+
// IDs mentioned in rationale text or findings tables of "clean" judge sections
|
|
154
|
+
// from being counted as detections.
|
|
155
|
+
const sections = response.split(/(?:^|\n)---\s*\n|(?=^## )/m);
|
|
156
|
+
const zeroFindingsPattern = /\*?\*?(?:ZERO|zero|0|no)\s+findings?\*?\*?|(?:findings?|issues?)[\s:]*\*?\*?(?:none|0|zero)\*?\*?|no\s+(?:issues?|findings?|problems?|concerns?)\s+(?:found|detected|identified|reported)|report(?:ing)?\s+zero|Score\s*[|:]\s*\*?\*?100\s*\/?\s*100\*?\*?/i;
|
|
156
157
|
for (const section of sections) {
|
|
157
|
-
// If this section explicitly declares zero/no findings
|
|
158
|
+
// If this section explicitly declares zero/no findings or a perfect score,
|
|
159
|
+
// skip rule ID extraction — any rule IDs are explanatory references
|
|
158
160
|
if (zeroFindingsPattern.test(section))
|
|
159
161
|
continue;
|
|
160
162
|
let match;
|
|
@@ -165,13 +167,17 @@ export function parseLlmRuleIds(response) {
|
|
|
165
167
|
}
|
|
166
168
|
}
|
|
167
169
|
}
|
|
168
|
-
// Secondary pass
|
|
169
|
-
//
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
170
|
+
// Secondary pass: extract compound IDs like DEPS-TYPO-001 from sections
|
|
171
|
+
// that did NOT declare zero findings (reuse the filtered sections)
|
|
172
|
+
for (const section of sections) {
|
|
173
|
+
if (zeroFindingsPattern.test(section))
|
|
174
|
+
continue;
|
|
175
|
+
const compoundPattern = /\b([A-Z][A-Z0-9]+)-[A-Z][A-Z0-9]+-(\d{1,3})\b/g;
|
|
176
|
+
let match;
|
|
177
|
+
while ((match = compoundPattern.exec(section)) !== null) {
|
|
178
|
+
if (validPrefixes.has(match[1])) {
|
|
179
|
+
found.add(`${match[1]}-${match[2]}`);
|
|
180
|
+
}
|
|
175
181
|
}
|
|
176
182
|
}
|
|
177
183
|
return [...found];
|
package/package.json
CHANGED
package/server.json
CHANGED
|
@@ -16,12 +16,12 @@
|
|
|
16
16
|
"mimeType": "image/png"
|
|
17
17
|
}
|
|
18
18
|
],
|
|
19
|
-
"version": "3.126.
|
|
19
|
+
"version": "3.126.2",
|
|
20
20
|
"packages": [
|
|
21
21
|
{
|
|
22
22
|
"registryType": "npm",
|
|
23
23
|
"identifier": "@kevinrabun/judges",
|
|
24
|
-
"version": "3.126.
|
|
24
|
+
"version": "3.126.2",
|
|
25
25
|
"transport": {
|
|
26
26
|
"type": "stdio"
|
|
27
27
|
}
|