@smithers-orchestrator/scorers 0.20.4 → 0.22.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smithers-orchestrator/scorers",
-  "version": "0.20.4",
+  "version": "0.22.0",
   "description": "Smithers scorer definitions, execution, aggregation, and persistence helpers",
   "type": "module",
   "sideEffects": false,
@@ -33,12 +33,12 @@
     "drizzle-orm": "^0.45.2",
     "effect": "^3.21.1",
     "zod": "^4.3.6",
-    "@smithers-orchestrator/db": "0.20.4",
-    "@smithers-orchestrator/agents": "0.20.4",
-    "@smithers-orchestrator/errors": "0.20.4",
-    "@smithers-orchestrator/observability": "0.20.4",
-    "@smithers-orchestrator/scheduler": "0.20.4",
-    "@smithers-orchestrator/graph": "0.20.4"
+    "@smithers-orchestrator/agents": "0.22.0",
+    "@smithers-orchestrator/graph": "0.22.0",
+    "@smithers-orchestrator/db": "0.22.0",
+    "@smithers-orchestrator/errors": "0.22.0",
+    "@smithers-orchestrator/observability": "0.22.0",
+    "@smithers-orchestrator/scheduler": "0.22.0"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/llmJudge.js

@@ -3,6 +3,69 @@
 /** @typedef {import("./types.js").ScorerInput} ScorerInput */
 /** @typedef {import("./types.js").ScoreResult} ScoreResult */
 
+/**
+ * Extracts and parses a JSON object from a judge's free-form text response.
+ *
+ * First tries `JSON.parse` on the trimmed text. If that fails, scans for the
+ * first `{` and walks forward tracking brace depth while respecting string
+ * literals and escape sequences, so braces inside string values (e.g. a brace
+ * in the `reason`) do not prematurely close the object.
+ *
+ * @param {string} text
+ * @returns {Record<string, unknown> | undefined}
+ */
+function parseJudgeJson(text) {
+    const trimmed = text.trim();
+    try {
+        return JSON.parse(trimmed);
+    }
+    catch {
+        // fall through to balanced-brace extraction
+    }
+    const start = trimmed.indexOf("{");
+    if (start === -1) {
+        return undefined;
+    }
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < trimmed.length; i++) {
+        const char = trimmed[i];
+        if (escaped) {
+            escaped = false;
+            continue;
+        }
+        if (char === "\\") {
+            if (inString) {
+                escaped = true;
+            }
+            continue;
+        }
+        if (char === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString) {
+            continue;
+        }
+        if (char === "{") {
+            depth++;
+        }
+        else if (char === "}") {
+            depth--;
+            if (depth === 0) {
+                try {
+                    return JSON.parse(trimmed.slice(start, i + 1));
+                }
+                catch {
+                    return undefined;
+                }
+            }
+        }
+    }
+    return undefined;
+}
+
 /**
  * Creates an LLM-as-judge scorer that delegates evaluation to an AI agent.
  *
@@ -41,23 +104,19 @@ export function llmJudge(config) {
             : typeof response?.text === "string"
                 ? response.text
                 : JSON.stringify(response);
-        // Try to parse JSON from the response
-        const jsonMatch = text.match(/\{[\s\S]*?"score"\s*:\s*[\d.]+[\s\S]*?\}/);
-        if (jsonMatch) {
-            try {
-                const parsed = JSON.parse(jsonMatch[0]);
-                const rawScore = Number(parsed.score);
-                return {
-                    score: Number.isFinite(rawScore)
-                        ? Math.max(0, Math.min(1, rawScore))
-                        : 0,
-                    reason: typeof parsed.reason === "string" ? parsed.reason : undefined,
-                    meta: { raw: text },
-                };
-            }
-            catch {
-                // fall through to default
-            }
+        // Try to parse JSON from the response. First attempt the whole trimmed
+        // text, then fall back to the outermost balanced-brace object so that a
+        // brace inside the judge's `reason` string does not truncate the match.
+        const parsed = parseJudgeJson(text);
+        if (parsed && typeof parsed === "object") {
+            const rawScore = Number(parsed.score);
+            return {
+                score: Number.isFinite(rawScore)
+                    ? Math.max(0, Math.min(1, rawScore))
+                    : 0,
+                reason: typeof parsed.reason === "string" ? parsed.reason : undefined,
+                meta: { raw: text },
+            };
         }
         // If we can't parse JSON, return a low-confidence score
         return {