@infinitedusky/indusk-mcp 1.24.3 → 1.24.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
8
8
|
import { dirname, join } from "node:path";
|
|
9
|
+
import { getScorecardQuestions } from "./scorecard-extractor.js";
|
|
9
10
|
function getFindingsPath(projectRoot) {
|
|
10
11
|
return join(projectRoot, ".indusk", "eval", "findings.json");
|
|
11
12
|
}
|
|
@@ -46,7 +47,11 @@ export function markFinding(projectRoot, key, state) {
|
|
|
46
47
|
export function ingestScorecard(projectRoot, scorecard) {
|
|
47
48
|
const findings = readFindings(projectRoot);
|
|
48
49
|
let added = 0;
|
|
49
|
-
|
|
50
|
+
// Defensive: the model occasionally returns a scorecard with a missing,
|
|
51
|
+
// null, or non-array `questions` field (it invents its own schema). See
|
|
52
|
+
// `scorecard-extractor.ts` getScorecardQuestions for the central guard.
|
|
53
|
+
const questions = getScorecardQuestions(scorecard);
|
|
54
|
+
for (const q of questions) {
|
|
50
55
|
if (q.answer === "yes")
|
|
51
56
|
continue; // no finding for passing questions
|
|
52
57
|
const key = `${scorecard.changeId}:${q.id}`;
|
|
@@ -16,7 +16,7 @@ import { EvalLogWriter } from "./log-writer.js";
|
|
|
16
16
|
import { initEvalOtel, initEvalOtelLogs, logEvalContent, shutdownEvalOtel, withSpan, } from "./otel.js";
|
|
17
17
|
import { buildEvaluatorPrompt } from "./prompt-builder.js";
|
|
18
18
|
import { V1_RUBRIC } from "./rubric.js";
|
|
19
|
-
import { extractScorecardJson, formatParseError } from "./scorecard-extractor.js";
|
|
19
|
+
import { extractScorecardJson, formatParseError, getScorecardQuestions, } from "./scorecard-extractor.js";
|
|
20
20
|
function getSessionPath(projectRoot) {
|
|
21
21
|
return join(projectRoot, ".indusk", "eval", "evaluator-session.json");
|
|
22
22
|
}
|
|
@@ -263,7 +263,11 @@ Output ONLY the JSON scorecard as before — no commentary.`;
|
|
|
263
263
|
rootSpan.setAttribute("scorecard.output_tokens", scorecard.usage.outputTokens);
|
|
264
264
|
}
|
|
265
265
|
const answerCounts = { yes: 0, no: 0, partial: 0 };
|
|
266
|
-
|
|
266
|
+
// Use the central guard from scorecard-extractor — `?? []` here was
|
|
267
|
+
// the bug: it only catches null/undefined, not non-array shapes like
|
|
268
|
+
// `{}` (which the model has been observed to return — e.g. on Numero
|
|
269
|
+
// 2026-04-19 19:54 with `questions: { conventions: {...} }` keyed by id).
|
|
270
|
+
for (const q of getScorecardQuestions(scorecard)) {
|
|
267
271
|
if (q.answer in answerCounts)
|
|
268
272
|
answerCounts[q.answer]++;
|
|
269
273
|
}
|
|
@@ -26,6 +26,26 @@
|
|
|
26
26
|
* This function only locates the JSON; it doesn't validate it.
|
|
27
27
|
*/
|
|
28
28
|
export declare function extractScorecardJson(text: string): string | null;
|
|
29
|
+
/**
|
|
30
|
+
* Defensive accessor for `scorecard.questions`. Returns the array if the
|
|
31
|
+
* field is array-shaped; returns `[]` for any other shape (missing, null,
|
|
32
|
+
* boolean, number, object-keyed-by-id, etc.). The model occasionally invents
|
|
33
|
+
* its own scorecard schema and puts non-arrays here — the wrapper must not
|
|
34
|
+
* crash when that happens.
|
|
35
|
+
*
|
|
36
|
+
* Use this everywhere the wrapper iterates `scorecard.questions`. Never
|
|
37
|
+
* iterate the field directly (with `?? []` or otherwise) — `?? []` only
|
|
38
|
+
* catches null/undefined, not falsy-but-not-nullish values like `false`,
|
|
39
|
+
* `0`, `""`, or non-array objects.
|
|
40
|
+
*
|
|
41
|
+
* Surfaced bugs this prevents:
|
|
42
|
+
* - `for (const q of scorecard.questions)` when `questions` is missing
|
|
43
|
+
* - `for (const q of scorecard.questions ?? [])` when `questions` is `{}`
|
|
44
|
+
* (e.g., model returned `questions: { conventions: {...} }` keyed by id)
|
|
45
|
+
*/
|
|
46
|
+
export declare function getScorecardQuestions<T>(scorecard: {
|
|
47
|
+
questions?: unknown;
|
|
48
|
+
}): T[];
|
|
29
49
|
/**
|
|
30
50
|
* Build an error message for the case where scorecard parsing failed.
|
|
31
51
|
* Includes the underlying error and a snippet of the raw stdout so post-
|
|
@@ -118,6 +118,26 @@ function findFirstBalancedJsonObject(text) {
|
|
|
118
118
|
// Walked to end of string without closing the outermost brace.
|
|
119
119
|
return null;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Defensive accessor for `scorecard.questions`. Returns the array if the
|
|
123
|
+
* field is array-shaped; returns `[]` for any other shape (missing, null,
|
|
124
|
+
* boolean, number, object-keyed-by-id, etc.). The model occasionally invents
|
|
125
|
+
* its own scorecard schema and puts non-arrays here — the wrapper must not
|
|
126
|
+
* crash when that happens.
|
|
127
|
+
*
|
|
128
|
+
* Use this everywhere the wrapper iterates `scorecard.questions`. Never
|
|
129
|
+
* iterate the field directly (with `?? []` or otherwise) — `?? []` only
|
|
130
|
+
* catches null/undefined, not falsy-but-not-nullish values like `false`,
|
|
131
|
+
* `0`, `""`, or non-array objects.
|
|
132
|
+
*
|
|
133
|
+
* Surfaced bugs this prevents:
|
|
134
|
+
* - `for (const q of scorecard.questions)` when `questions` is missing
|
|
135
|
+
* - `for (const q of scorecard.questions ?? [])` when `questions` is `{}`
|
|
136
|
+
* (e.g., model returned `questions: { conventions: {...} }` keyed by id)
|
|
137
|
+
*/
|
|
138
|
+
export function getScorecardQuestions(scorecard) {
|
|
139
|
+
return Array.isArray(scorecard.questions) ? scorecard.questions : [];
|
|
140
|
+
}
|
|
121
141
|
/**
|
|
122
142
|
* Build an error message for the case where scorecard parsing failed.
|
|
123
143
|
* Includes the underlying error and a snippet of the raw stdout so post-
|