@agentdeck/bridge 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apme/classifier.d.ts.map +1 -1
- package/dist/apme/classifier.js +18 -10
- package/dist/apme/classifier.js.map +1 -1
- package/dist/apme/claude-transcript-reader.d.ts +36 -0
- package/dist/apme/claude-transcript-reader.d.ts.map +1 -1
- package/dist/apme/claude-transcript-reader.js +78 -2
- package/dist/apme/claude-transcript-reader.js.map +1 -1
- package/dist/apme/collector.d.ts +109 -3
- package/dist/apme/collector.d.ts.map +1 -1
- package/dist/apme/collector.js +408 -44
- package/dist/apme/collector.js.map +1 -1
- package/dist/apme/dashboard-html.d.ts.map +1 -1
- package/dist/apme/dashboard-html.js +4 -2
- package/dist/apme/dashboard-html.js.map +1 -1
- package/dist/apme/graph.d.ts +1 -1
- package/dist/apme/graph.d.ts.map +1 -1
- package/dist/apme/graph.js +51 -1
- package/dist/apme/graph.js.map +1 -1
- package/dist/apme/http.d.ts.map +1 -1
- package/dist/apme/http.js +4 -0
- package/dist/apme/http.js.map +1 -1
- package/dist/apme/index.d.ts +4 -0
- package/dist/apme/index.d.ts.map +1 -1
- package/dist/apme/index.js +13 -1
- package/dist/apme/index.js.map +1 -1
- package/dist/apme/runner.d.ts +90 -0
- package/dist/apme/runner.d.ts.map +1 -1
- package/dist/apme/runner.js +524 -48
- package/dist/apme/runner.js.map +1 -1
- package/dist/apme/sample-to-timeline.d.ts.map +1 -1
- package/dist/apme/sample-to-timeline.js +5 -0
- package/dist/apme/sample-to-timeline.js.map +1 -1
- package/dist/apme/settings.d.ts +8 -0
- package/dist/apme/settings.d.ts.map +1 -1
- package/dist/apme/settings.js +4 -0
- package/dist/apme/settings.js.map +1 -1
- package/dist/apme/store.d.ts +77 -8
- package/dist/apme/store.d.ts.map +1 -1
- package/dist/apme/store.js +189 -17
- package/dist/apme/store.js.map +1 -1
- package/dist/apme/task-gradeability.d.ts +82 -0
- package/dist/apme/task-gradeability.d.ts.map +1 -0
- package/dist/apme/task-gradeability.js +126 -0
- package/dist/apme/task-gradeability.js.map +1 -0
- package/dist/awaiting-overlay.d.ts +13 -26
- package/dist/awaiting-overlay.d.ts.map +1 -1
- package/dist/awaiting-overlay.js +7 -40
- package/dist/awaiting-overlay.js.map +1 -1
- package/dist/claude-permission-rules.d.ts +20 -32
- package/dist/claude-permission-rules.d.ts.map +1 -1
- package/dist/claude-permission-rules.js +19 -102
- package/dist/claude-permission-rules.js.map +1 -1
- package/dist/claude-usage-recovery.d.ts +54 -0
- package/dist/claude-usage-recovery.d.ts.map +1 -0
- package/dist/claude-usage-recovery.js +200 -0
- package/dist/claude-usage-recovery.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +28 -8
- package/dist/cli.js.map +1 -1
- package/dist/codex-rollout-response.d.ts +43 -0
- package/dist/codex-rollout-response.d.ts.map +1 -1
- package/dist/codex-rollout-response.js +67 -0
- package/dist/codex-rollout-response.js.map +1 -1
- package/dist/coordination-evidence.d.ts +150 -0
- package/dist/coordination-evidence.d.ts.map +1 -0
- package/dist/coordination-evidence.js +455 -0
- package/dist/coordination-evidence.js.map +1 -0
- package/dist/daemon-server.d.ts.map +1 -1
- package/dist/daemon-server.js +329 -34
- package/dist/daemon-server.js.map +1 -1
- package/dist/daemon.js +1 -1
- package/dist/deferred-reply-read.d.ts +24 -0
- package/dist/deferred-reply-read.d.ts.map +1 -0
- package/dist/deferred-reply-read.js +42 -0
- package/dist/deferred-reply-read.js.map +1 -0
- package/dist/esp32-serial.d.ts +11 -0
- package/dist/esp32-serial.d.ts.map +1 -1
- package/dist/esp32-serial.js +57 -2
- package/dist/esp32-serial.js.map +1 -1
- package/dist/hook-codex-sessions.d.ts +9 -0
- package/dist/hook-codex-sessions.d.ts.map +1 -1
- package/dist/hook-codex-sessions.js +35 -2
- package/dist/hook-codex-sessions.js.map +1 -1
- package/dist/hook-opencode-sessions.d.ts +62 -0
- package/dist/hook-opencode-sessions.d.ts.map +1 -0
- package/dist/hook-opencode-sessions.js +180 -0
- package/dist/hook-opencode-sessions.js.map +1 -0
- package/dist/hook-response-source.d.ts +40 -0
- package/dist/hook-response-source.d.ts.map +1 -0
- package/dist/hook-response-source.js +31 -0
- package/dist/hook-response-source.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/native-dependency-diagnostics.d.ts +29 -0
- package/dist/native-dependency-diagnostics.d.ts.map +1 -0
- package/dist/native-dependency-diagnostics.js +95 -0
- package/dist/native-dependency-diagnostics.js.map +1 -0
- package/dist/observed-steering.d.ts +4 -4
- package/dist/observed-steering.d.ts.map +1 -1
- package/dist/observed-steering.js +28 -32
- package/dist/observed-steering.js.map +1 -1
- package/dist/openclaw-gateway-stability.d.ts +94 -0
- package/dist/openclaw-gateway-stability.d.ts.map +1 -0
- package/dist/openclaw-gateway-stability.js +176 -0
- package/dist/openclaw-gateway-stability.js.map +1 -0
- package/dist/passive-observer.d.ts +5 -0
- package/dist/passive-observer.d.ts.map +1 -1
- package/dist/passive-observer.js +6 -0
- package/dist/passive-observer.js.map +1 -1
- package/dist/review-runner.js +1 -1
- package/dist/subagent-timeline.d.ts +27 -0
- package/dist/subagent-timeline.d.ts.map +1 -1
- package/dist/subagent-timeline.js +52 -5
- package/dist/subagent-timeline.js.map +1 -1
- package/dist/usage-api.d.ts +3 -0
- package/dist/usage-api.d.ts.map +1 -1
- package/dist/usage-api.js +101 -36
- package/dist/usage-api.js.map +1 -1
- package/dist/usage-event.d.ts.map +1 -1
- package/dist/usage-event.js +2 -1
- package/dist/usage-event.js.map +1 -1
- package/package.json +5 -5
package/dist/apme/runner.js
CHANGED
|
@@ -11,13 +11,23 @@
|
|
|
11
11
|
* by `shouldJudge()` so the common "clear pass" case skips layer 2 entirely.
|
|
12
12
|
*/
|
|
13
13
|
import { spawn } from 'child_process';
|
|
14
|
+
import { taskGradeability, notGradeableNotes } from './task-gradeability.js';
|
|
14
15
|
import { existsSync, readdirSync } from 'fs';
|
|
15
|
-
import { debug } from '../logger.js';
|
|
16
|
+
import { debug, log } from '../logger.js';
|
|
16
17
|
import { loadApmeConfig, shouldJudge, judgeBackendSupported, DEFAULT_APME_CONFIG } from './settings.js';
|
|
17
18
|
import { loadMlxSettings, mlxChatUrl } from '@agentdeck/shared';
|
|
18
19
|
import { callFoundationModelsHelper, probeFoundationModelsHelper } from '../foundation-models-helper.js';
|
|
19
20
|
import { runSampleScorers } from './scorers/index.js';
|
|
20
21
|
import { execSync } from 'child_process';
|
|
22
|
+
/** Judge attempts per task before the backlog drain stops offering it. */
|
|
23
|
+
const TASK_EVAL_MAX_ATTEMPTS = 2;
|
|
24
|
+
/** How long a task stays parked after that. A park that lasted the whole
|
|
25
|
+
* process meant one busy hour at the judge parked the entire backlog: the
|
|
26
|
+
* drain fed three tasks per 30 s tick to a local model that answers one at a
|
|
27
|
+
* time, requests queued past the 60 s call timeout, every task burned its
|
|
28
|
+
* two attempts, and the drain went from 140 tasks/hour to zero with 289
|
|
29
|
+
* still pending and not one line in the log (2026-09-03). */
|
|
30
|
+
export const TASK_EVAL_PARK_MS = 30 * 60_000;
|
|
21
31
|
function deriveTaskOutcome(score) {
|
|
22
32
|
if (score == null)
|
|
23
33
|
return 'pending';
|
|
@@ -165,11 +175,99 @@ export class ApmeRunner {
|
|
|
165
175
|
debug('APME', `skip duplicate task eval taskId=${job.taskId}`);
|
|
166
176
|
return;
|
|
167
177
|
}
|
|
178
|
+
// A task whose judge call failed (unparseable verdict, backend error)
|
|
179
|
+
// leaves no row behind, so the backlog drain would offer it again every
|
|
180
|
+
// sweep. Two attempts per process; the next daemon start gets two more.
|
|
181
|
+
const failed = this.taskEvalFailures.get(job.taskId);
|
|
182
|
+
if (failed && failed.attempts >= TASK_EVAL_MAX_ATTEMPTS) {
|
|
183
|
+
if (Date.now() - failed.lastAt < TASK_EVAL_PARK_MS) {
|
|
184
|
+
debug('APME', `skip task eval taskId=${job.taskId.slice(0, 8)} — parked after ${failed.attempts} failed attempts`);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
// The park expired: the judge may be back. Two fresh attempts.
|
|
188
|
+
this.taskEvalFailures.delete(job.taskId);
|
|
189
|
+
}
|
|
168
190
|
this.runningTaskIds.add(job.taskId);
|
|
169
|
-
void this.runTaskEval(job).
|
|
191
|
+
void this.runTaskEval(job).then((ok) => {
|
|
192
|
+
if (ok !== false)
|
|
193
|
+
return;
|
|
194
|
+
const attempts = (this.taskEvalFailures.get(job.taskId)?.attempts ?? 0) + 1;
|
|
195
|
+
this.taskEvalFailures.set(job.taskId, { attempts, lastAt: Date.now() });
|
|
196
|
+
if (attempts >= TASK_EVAL_MAX_ATTEMPTS)
|
|
197
|
+
this.noteTaskParked(job.taskId);
|
|
198
|
+
}).finally(() => {
|
|
170
199
|
this.runningTaskIds.delete(job.taskId);
|
|
171
200
|
});
|
|
172
201
|
}
|
|
202
|
+
/** Failed judge attempts per task and when the last one was; in-memory on
|
|
203
|
+
* purpose, like `judgeFailures`. See `enqueueTask`. */
|
|
204
|
+
taskEvalFailures = new Map();
|
|
205
|
+
/** Why the most recent task judge call failed — the one fact the log needs
|
|
206
|
+
* when the backlog stops moving. */
|
|
207
|
+
lastTaskEvalFailure = '';
|
|
208
|
+
parkedTaskCount = 0;
|
|
209
|
+
/** Task judge calls in flight right now, whatever enqueued them. The
|
|
210
|
+
* backlog drain reads this so it never piles onto a judge that is still
|
|
211
|
+
* answering — the local backends answer one prompt at a time. */
|
|
212
|
+
get inFlightTaskEvals() {
|
|
213
|
+
return this.runningTaskIds.size;
|
|
214
|
+
}
|
|
215
|
+
/** The backlog candidates this tick may actually feed, newest first and at
|
|
216
|
+
* most `limit` of them.
|
|
217
|
+
*
|
|
218
|
+
* The drain feeds exactly one task per tick, taken from the head of a query
|
|
219
|
+
* ordered by `ended_at DESC`, and `enqueueTask` drops a parked task
|
|
220
|
+
* silently — so a task that fails every attempt owns that head and every
|
|
221
|
+
* tick spends its one slot on it while everything behind it starves.
|
|
222
|
+
* Measured 2026-09-06: 156 closed tasks from 2026-08-07..23 unjudged for
|
|
223
|
+
* two weeks, while the day's own tasks (83 of 95) were judged normally by
|
|
224
|
+
* the live close path. The count is the tell — 217 -> 156 while the head
|
|
225
|
+
* was still judgeable, then flat.
|
|
226
|
+
*
|
|
227
|
+
* A truth table over `isTaskParked` rather than a loop at the call site,
|
|
228
|
+
* because a call site that forgets the check leaves every test green while
|
|
229
|
+
* the backlog stops moving. */
|
|
230
|
+
pickBacklogTasks(candidates, limit) {
|
|
231
|
+
const picked = [];
|
|
232
|
+
for (const candidate of candidates) {
|
|
233
|
+
if (picked.length >= limit)
|
|
234
|
+
break;
|
|
235
|
+
if (this.isTaskParked(candidate.id))
|
|
236
|
+
continue;
|
|
237
|
+
picked.push(candidate);
|
|
238
|
+
}
|
|
239
|
+
// A window in which EVERY candidate is parked is the stall this function
|
|
240
|
+
// exists to prevent, one level up: the drain silently does nothing again,
|
|
241
|
+
// and silence is what let the original bug run for two weeks. Say it, at
|
|
242
|
+
// most once per park period so a wide outage is one line, not one a tick.
|
|
243
|
+
if (picked.length === 0 && candidates.length > 0) {
|
|
244
|
+
const now = Date.now();
|
|
245
|
+
if (now - this.lastAllParkedLogAt >= TASK_EVAL_PARK_MS) {
|
|
246
|
+
this.lastAllParkedLogAt = now;
|
|
247
|
+
log(`APME task judge: all ${candidates.length} backlog candidate(s) in the window are parked — the drain is idle until a park expires (latest failure: ${this.lastTaskEvalFailure || 'unknown'})`);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return picked;
|
|
251
|
+
}
|
|
252
|
+
lastAllParkedLogAt = 0;
|
|
253
|
+
/** Whether this task is parked right now, i.e. `enqueueTask` would drop it. */
|
|
254
|
+
isTaskParked(taskId) {
|
|
255
|
+
const failed = this.taskEvalFailures.get(taskId);
|
|
256
|
+
if (!failed || failed.attempts < TASK_EVAL_MAX_ATTEMPTS)
|
|
257
|
+
return false;
|
|
258
|
+
return Date.now() - failed.lastAt < TASK_EVAL_PARK_MS;
|
|
259
|
+
}
|
|
260
|
+
/** One visible line per decade of parks (the 1st, 10th, 100th…): enough to
|
|
261
|
+
* see a stalled judge in the daemon log, never a line per task. */
|
|
262
|
+
noteTaskParked(taskId) {
|
|
263
|
+
this.parkedTaskCount++;
|
|
264
|
+
const n = this.parkedTaskCount;
|
|
265
|
+
if (n === 1 || Number.isInteger(Math.log10(n))) {
|
|
266
|
+
log(`APME task judge: ${n} task(s) parked for ${Math.round(TASK_EVAL_PARK_MS / 60_000)} min after ${TASK_EVAL_MAX_ATTEMPTS} failed attempts (latest ${taskId.slice(0, 8)}: ${this.lastTaskEvalFailure || 'unknown'})`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** Resolves `false` when the judge was called and produced nothing usable
|
|
270
|
+
* (a failed attempt worth counting); `true` or `undefined` otherwise. */
|
|
173
271
|
async runTaskEval({ runId, taskId, category, boundarySignal }) {
|
|
174
272
|
const cfg = this.configOverride ?? loadApmeConfig();
|
|
175
273
|
if (!cfg.enabled)
|
|
@@ -184,18 +282,26 @@ export class ApmeRunner {
|
|
|
184
282
|
const turns = this.store.listTurnsForTask(taskId);
|
|
185
283
|
if (turns.length === 0)
|
|
186
284
|
return;
|
|
187
|
-
//
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
285
|
+
// A verdict about the agent's work needs the agent's work — see
|
|
286
|
+
// task-gradeability.ts for what that means and what it measured. A
|
|
287
|
+
// declined task is stamped with its reason so the row can say so.
|
|
288
|
+
const gradeability = taskGradeability(turns);
|
|
289
|
+
if (!gradeability.gradeable) {
|
|
290
|
+
debug('APME', `runTaskEval declined task=${taskId.slice(0, 8)} — ${gradeability.reason}`);
|
|
291
|
+
try {
|
|
292
|
+
this.store.updateTask(taskId, { notesJson: notGradeableNotes(gradeability.reason) });
|
|
293
|
+
}
|
|
294
|
+
catch { /* ignore */ }
|
|
197
295
|
return;
|
|
198
296
|
}
|
|
297
|
+
// The category the rubric keys on: the caller's, else the row's, else
|
|
298
|
+
// the run's. A reaped task used to reach here with none and fall to the
|
|
299
|
+
// generic rubric while its run had long been classified.
|
|
300
|
+
if (!category || category === 'unknown') {
|
|
301
|
+
const fromTask = task.taskCategory && task.taskCategory !== 'unknown' ? task.taskCategory : undefined;
|
|
302
|
+
const fromRun = this.store.getRun(runId)?.taskCategory;
|
|
303
|
+
category = fromTask ?? (fromRun && fromRun !== 'unknown' ? fromRun : undefined);
|
|
304
|
+
}
|
|
199
305
|
// Select rubric: task_rollup preferred, fall back to category, then general.
|
|
200
306
|
const rubric = this.store.getCurrentRubric('task_rollup')
|
|
201
307
|
?? (category ? this.store.getCurrentRubric(category) : null)
|
|
@@ -234,7 +340,7 @@ export class ApmeRunner {
|
|
|
234
340
|
'',
|
|
235
341
|
'--- TURNS ---',
|
|
236
342
|
...lines,
|
|
237
|
-
...(trajectoryLines.length ? ['', '---
|
|
343
|
+
...(trajectoryLines.length ? ['', '--- TASK TRAJECTORY ---', ...trajectoryLines] : []),
|
|
238
344
|
'',
|
|
239
345
|
'Respond with strict JSON only.',
|
|
240
346
|
].join('\n');
|
|
@@ -247,8 +353,9 @@ export class ApmeRunner {
|
|
|
247
353
|
: await callJudgeWithMeta(judgePrompt, cfg.judge);
|
|
248
354
|
const parsed = parseJudgeJson(judgeResult.text);
|
|
249
355
|
if (!parsed) {
|
|
356
|
+
this.lastTaskEvalFailure = `unparseable verdict from ${judgeResult.effectiveLabel}`;
|
|
250
357
|
debug('APME', `runTaskEval parse failed task=${taskId.slice(0, 8)}`);
|
|
251
|
-
return;
|
|
358
|
+
return false;
|
|
252
359
|
}
|
|
253
360
|
const now = Date.now();
|
|
254
361
|
const judgeModel = judgeResult.effectiveLabel;
|
|
@@ -353,8 +460,11 @@ export class ApmeRunner {
|
|
|
353
460
|
}
|
|
354
461
|
}
|
|
355
462
|
catch (err) {
|
|
463
|
+
this.lastTaskEvalFailure = String(err).slice(0, 160);
|
|
356
464
|
debug('APME', `task eval error taskId=${taskId.slice(0, 8)}: ${String(err)}`);
|
|
465
|
+
return false;
|
|
357
466
|
}
|
|
467
|
+
return true;
|
|
358
468
|
}
|
|
359
469
|
async runTurnEval({ runId, turnId, category }) {
|
|
360
470
|
const cfg = this.configOverride ?? loadApmeConfig();
|
|
@@ -750,6 +860,12 @@ export function buildTrajectoryLines(sample, cap = 30) {
|
|
|
750
860
|
case 'model':
|
|
751
861
|
lines.push(` model ${e.model}: ${e.inputTokens}in/${e.outputTokens}out tok${e.costUsd ? ` ($${e.costUsd.toFixed(4)})` : ''}`);
|
|
752
862
|
break;
|
|
863
|
+
case 'subagent': {
|
|
864
|
+
const duration = e.durationMs == null ? '' : ` (${Math.round(e.durationMs / 1000)}s)`;
|
|
865
|
+
const summary = e.summary ? `: ${e.summary.slice(0, 160)}` : '';
|
|
866
|
+
lines.push(` subagent ${e.name} → ${e.phase}${duration}${summary}`);
|
|
867
|
+
break;
|
|
868
|
+
}
|
|
753
869
|
case 'state':
|
|
754
870
|
lines.push(` state → ${e.to}`);
|
|
755
871
|
break;
|
|
@@ -855,6 +971,12 @@ export function effectiveJudgeModelTag(cfg) {
|
|
|
855
971
|
// so analytics queries aggregate FM evals across the Node and Swift stacks.
|
|
856
972
|
if (cfg.backend === 'foundationModels')
|
|
857
973
|
return 'foundationModels:apple-intelligence';
|
|
974
|
+
// The API leg does not necessarily call `cfg.model` — `apiJudgeModel` falls
|
|
975
|
+
// back when the configured id belongs to another backend. Stamping the
|
|
976
|
+
// configured value recorded a verdict as produced by a model that never ran.
|
|
977
|
+
// Mirrored by `ApmeJudgeApi.judgeModelLabel`.
|
|
978
|
+
if (cfg.backend === 'api')
|
|
979
|
+
return `api:${apiJudgeModel(cfg)}`;
|
|
858
980
|
return `${cfg.backend}:${cfg.model}`;
|
|
859
981
|
}
|
|
860
982
|
/** Strip backend-specific fields when forcing a cfg through a different
|
|
@@ -960,7 +1082,12 @@ export async function callJudgeWithMeta(prompt, judgeCfg) {
|
|
|
960
1082
|
return { text, effectiveBackend: 'mlx', effectiveLabel: effectiveJudgeModelTag(judgeCfg) };
|
|
961
1083
|
}
|
|
962
1084
|
catch (err) {
|
|
963
|
-
|
|
1085
|
+
// "unavailable" used to cover truncation too, and those are different
|
|
1086
|
+
// facts: the server answered, its answer was cut at `max_tokens`, and the
|
|
1087
|
+
// verdict is discarded. Both fall through to the FM floor — which is a
|
|
1088
|
+
// measurably weaker judge — so the switch has to be visible in the normal
|
|
1089
|
+
// log, not only under DEBUG, or its rate is unmeasurable.
|
|
1090
|
+
log(`APME judge: mlx produced no verdict (${String(err)}) — falling back to foundationModels`);
|
|
964
1091
|
const fmCfg = sanitizeForFoundationModels(judgeCfg);
|
|
965
1092
|
const text = await callFoundationModels(prompt, fmCfg);
|
|
966
1093
|
return { text, effectiveBackend: 'foundationModels', effectiveLabel: effectiveJudgeModelTag(fmCfg) };
|
|
@@ -1047,8 +1174,9 @@ export async function probeJudgeBackend(cfg) {
|
|
|
1047
1174
|
endpoint: base, checkedAt,
|
|
1048
1175
|
};
|
|
1049
1176
|
}
|
|
1050
|
-
// Cheapest possible inference probe: max_tokens=1, temperature=0.
|
|
1051
|
-
//
|
|
1177
|
+
// Cheapest possible inference probe: max_tokens=1, temperature=0. The
|
|
1178
|
+
// server is shared with other local agents, so allow one normal request
|
|
1179
|
+
// ahead of the ping instead of marking a healthy serial backend down.
|
|
1052
1180
|
const ping = await fetch(`${base}/v1/chat/completions`, {
|
|
1053
1181
|
method: 'POST',
|
|
1054
1182
|
headers: { 'Content-Type': 'application/json' },
|
|
@@ -1057,7 +1185,7 @@ export async function probeJudgeBackend(cfg) {
|
|
|
1057
1185
|
messages: [{ role: 'user', content: 'ping' }],
|
|
1058
1186
|
max_tokens: 1, temperature: 0,
|
|
1059
1187
|
}),
|
|
1060
|
-
signal: AbortSignal.timeout(
|
|
1188
|
+
signal: AbortSignal.timeout(30_000),
|
|
1061
1189
|
}).catch((e) => ({ ok: false, status: 0, statusText: String(e).slice(0, 80) }));
|
|
1062
1190
|
if (!ping.ok) {
|
|
1063
1191
|
const detail = ping.status ? `HTTP ${ping.status}` : ping.statusText ?? 'no response';
|
|
@@ -1260,6 +1388,48 @@ export async function probeJudgeBackend(cfg) {
|
|
|
1260
1388
|
};
|
|
1261
1389
|
}
|
|
1262
1390
|
}
|
|
1391
|
+
/**
|
|
1392
|
+
* OpenAI JSON mode for the local judge legs.
|
|
1393
|
+
*
|
|
1394
|
+
* The judge prompt asks for strict JSON and the runner parses the reply as
|
|
1395
|
+
* JSON, but nothing ever ASKED the server to constrain its output — so a model
|
|
1396
|
+
* that decides to wrap the object in prose produces an unparseable verdict, the
|
|
1397
|
+
* task is retried, fails again, and parks for 30 minutes. Measured on the
|
|
1398
|
+
* author's store: one task (`de9afcc5`) parked six times in three hours and 17
|
|
1399
|
+
* times since 2026-09-03, never judged. `response_format: { type: 'json_object' }`
|
|
1400
|
+
* turns that from a hope into a request the server can honour.
|
|
1401
|
+
*
|
|
1402
|
+
* It is not universal: `apme.judge.endpoint` may be any OpenAI-compatible
|
|
1403
|
+
* server, and some answer 400/422 to the field itself. Such a server must not
|
|
1404
|
+
* lose its judge entirely, so a rejection retries once WITHOUT the field and is
|
|
1405
|
+
* remembered per endpoint — the probe then costs one request per endpoint per
|
|
1406
|
+
* process, not one per verdict.
|
|
1407
|
+
*/
|
|
1408
|
+
const judgeJsonModeUnsupported = new Set();
|
|
1409
|
+
/** Reset the per-endpoint JSON-mode memory (tests only). */
|
|
1410
|
+
export function clearJudgeJsonModeCacheForTests() {
|
|
1411
|
+
judgeJsonModeUnsupported.clear();
|
|
1412
|
+
}
|
|
1413
|
+
function judgeJsonModeEnabled(url) {
|
|
1414
|
+
return !judgeJsonModeUnsupported.has(url);
|
|
1415
|
+
}
|
|
1416
|
+
/** A 400/422 to a request carrying `response_format` is the server refusing the
|
|
1417
|
+
* FIELD. Every other status is about the request or the account (401, 429,
|
|
1418
|
+
* 5xx) and must surface unchanged — retrying those without JSON mode would
|
|
1419
|
+
* hide an auth failure behind a second identical failure. */
|
|
1420
|
+
function isJsonModeRejection(status) {
|
|
1421
|
+
return status === 400 || status === 422;
|
|
1422
|
+
}
|
|
1423
|
+
function noteJsonModeUnsupported(url, status) {
|
|
1424
|
+
if (judgeJsonModeUnsupported.has(url))
|
|
1425
|
+
return;
|
|
1426
|
+
judgeJsonModeUnsupported.add(url);
|
|
1427
|
+
log(`APME judge: ${url} rejected response_format json_object (HTTP ${status}) — retrying without JSON mode and not sending it again this process`);
|
|
1428
|
+
}
|
|
1429
|
+
/** `{ response_format: … }` or nothing, so a call site can spread it. */
|
|
1430
|
+
function jsonModeField(enabled) {
|
|
1431
|
+
return enabled ? { response_format: { type: 'json_object' } } : {};
|
|
1432
|
+
}
|
|
1263
1433
|
async function callMlx(prompt, cfg) {
|
|
1264
1434
|
// MLX server speaks OpenAI chat-completions. The llm.mlx pin (shared with
|
|
1265
1435
|
// timeline/label summarizers) is the source of truth; cfg.endpoint/model
|
|
@@ -1285,28 +1455,63 @@ async function callMlx(prompt, cfg) {
|
|
|
1285
1455
|
}
|
|
1286
1456
|
catch { /* use configured model */ }
|
|
1287
1457
|
}
|
|
1288
|
-
const
|
|
1458
|
+
const request = (userPrompt, jsonMode) => fetch(url, {
|
|
1289
1459
|
method: 'POST',
|
|
1290
1460
|
headers: { 'Content-Type': 'application/json' },
|
|
1291
1461
|
body: JSON.stringify({
|
|
1292
1462
|
model,
|
|
1293
1463
|
messages: [
|
|
1294
1464
|
{ role: 'system', content: 'You are an exacting code evaluator. Reply with strict JSON only.' },
|
|
1295
|
-
{ role: 'user', content:
|
|
1465
|
+
{ role: 'user', content: userPrompt },
|
|
1296
1466
|
],
|
|
1297
1467
|
temperature: 0.0,
|
|
1298
1468
|
max_tokens: 800,
|
|
1469
|
+
...jsonModeField(jsonMode),
|
|
1299
1470
|
}),
|
|
1300
|
-
|
|
1471
|
+
// Long task_rollup prompts can cross 60s at the tail under sustained
|
|
1472
|
+
// local load (68.7s observed with Gemma 4). Keep the timeout bounded but
|
|
1473
|
+
// leave enough room for one serial judge request to finish.
|
|
1474
|
+
signal: AbortSignal.timeout(90_000),
|
|
1301
1475
|
});
|
|
1476
|
+
let jsonMode = judgeJsonModeEnabled(url);
|
|
1477
|
+
let resp = await request(prompt, jsonMode);
|
|
1478
|
+
if (!resp.ok && isJsonModeRejection(resp.status)) {
|
|
1479
|
+
const detail = await resp.text();
|
|
1480
|
+
const overflow = detail.match(/Request needs \d+ context tokens \((\d+) prompt \+ (\d+) max generation\), but MAX_KV_SIZE is (\d+)/);
|
|
1481
|
+
if (!overflow && jsonMode) {
|
|
1482
|
+
// Not the context-overflow shape, so this server is refusing the field
|
|
1483
|
+
// rather than the prompt. Retry once without it; a genuinely bad request
|
|
1484
|
+
// fails again below with its own status.
|
|
1485
|
+
noteJsonModeUnsupported(url, resp.status);
|
|
1486
|
+
jsonMode = false;
|
|
1487
|
+
resp = await request(prompt, false);
|
|
1488
|
+
}
|
|
1489
|
+
else if (overflow) {
|
|
1490
|
+
const promptTokens = Number(overflow[1]);
|
|
1491
|
+
const maxGeneration = Number(overflow[2]);
|
|
1492
|
+
const maxKv = Number(overflow[3]);
|
|
1493
|
+
// The server gives the tokenizer-accurate count. Keep a small margin for
|
|
1494
|
+
// the chat template, then preserve the rubric/initial goal at the front
|
|
1495
|
+
// and the latest turns/trajectory/instruction at the end.
|
|
1496
|
+
const targetPromptTokens = Math.max(256, maxKv - maxGeneration - 128);
|
|
1497
|
+
const ratio = Math.min(0.95, (targetPromptTokens / promptTokens) * 0.98);
|
|
1498
|
+
const compacted = compactPromptForMlxContext(prompt, Math.max(1000, Math.floor(prompt.length * ratio)));
|
|
1499
|
+
debug('APME', `MLX context overflow (${promptTokens}+${maxGeneration}>${maxKv}); retrying with ${compacted.length}/${prompt.length} prompt chars`);
|
|
1500
|
+
resp = await request(compacted, jsonMode);
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1302
1503
|
if (!resp.ok)
|
|
1303
1504
|
throw new Error(`MLX judge HTTP ${resp.status}`);
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1505
|
+
return judgeChatContent(await resp.json(), 'MLX');
|
|
1506
|
+
}
|
|
1507
|
+
function compactPromptForMlxContext(text, maxChars) {
|
|
1508
|
+
if (text.length <= maxChars)
|
|
1509
|
+
return text;
|
|
1510
|
+
const marker = '\n\n… [middle omitted to fit local context] …\n\n';
|
|
1511
|
+
const available = Math.max(0, maxChars - marker.length);
|
|
1512
|
+
const headChars = Math.floor(available * 0.45);
|
|
1513
|
+
const tailChars = available - headChars;
|
|
1514
|
+
return text.slice(0, headChars) + marker + text.slice(text.length - tailChars);
|
|
1310
1515
|
}
|
|
1311
1516
|
/** Normalize a user-supplied base/endpoint to the chat-completions URL.
|
|
1312
1517
|
* Accepts a bare host (`http://127.0.0.1:11434`), a base with `/v1`, or the
|
|
@@ -1375,7 +1580,7 @@ async function callOpenAICompatible(prompt, cfg) {
|
|
|
1375
1580
|
const headers = { 'Content-Type': 'application/json' };
|
|
1376
1581
|
if (cfg.apiKey)
|
|
1377
1582
|
headers.Authorization = `Bearer ${cfg.apiKey}`;
|
|
1378
|
-
const
|
|
1583
|
+
const send = (jsonMode) => fetch(url, {
|
|
1379
1584
|
method: 'POST',
|
|
1380
1585
|
headers,
|
|
1381
1586
|
body: JSON.stringify({
|
|
@@ -1386,16 +1591,20 @@ async function callOpenAICompatible(prompt, cfg) {
|
|
|
1386
1591
|
],
|
|
1387
1592
|
temperature: 0,
|
|
1388
1593
|
max_tokens: 1024,
|
|
1594
|
+
...(cfg.reasoningEffort ? { reasoning_effort: cfg.reasoningEffort } : {}),
|
|
1595
|
+
...jsonModeField(jsonMode),
|
|
1389
1596
|
}),
|
|
1390
1597
|
signal: AbortSignal.timeout(90_000),
|
|
1391
1598
|
});
|
|
1599
|
+
const jsonMode = judgeJsonModeEnabled(url);
|
|
1600
|
+
let resp = await send(jsonMode);
|
|
1601
|
+
if (!resp.ok && jsonMode && isJsonModeRejection(resp.status)) {
|
|
1602
|
+
noteJsonModeUnsupported(url, resp.status);
|
|
1603
|
+
resp = await send(false);
|
|
1604
|
+
}
|
|
1392
1605
|
if (!resp.ok)
|
|
1393
1606
|
throw new Error(`openai judge HTTP ${resp.status} (${url})`);
|
|
1394
|
-
|
|
1395
|
-
const text = json.choices?.[0]?.message?.content;
|
|
1396
|
-
if (typeof text !== 'string' || text.trim().length === 0)
|
|
1397
|
-
throw new Error('openai judge returned empty content');
|
|
1398
|
-
return text;
|
|
1607
|
+
return judgeChatContent(await resp.json(), 'openai');
|
|
1399
1608
|
}
|
|
1400
1609
|
async function callOpenClaw(prompt, cfg) {
|
|
1401
1610
|
// OpenClaw Gateway exposes the user's configured models. Route through it
|
|
@@ -1512,8 +1721,19 @@ async function resolveFoundationModelsUrl() {
|
|
|
1512
1721
|
}
|
|
1513
1722
|
/** Default model for the opt-in Anthropic API judge when the configured
|
|
1514
1723
|
* `model` belongs to another backend (e.g. an MLX id left over from a
|
|
1515
|
-
* backend switch).
|
|
1516
|
-
|
|
1724
|
+
* backend switch). Mirrored by `ApmeJudgeApi.swift`: the two daemons read the
|
|
1725
|
+
* same `settings.json` and are the same judge, so a user who opted into the
|
|
1726
|
+
* API leg without naming a model must not get a different model depending on
|
|
1727
|
+
* which daemon happens to hold the port (Node said `claude-opus-4-8`, Swift
|
|
1728
|
+
* `claude-opus-4-6`, until #286). */
|
|
1729
|
+
const API_JUDGE_DEFAULT_MODEL = 'claude-opus-5';
|
|
1730
|
+
/** Output cap for the API leg, mirrored by `ApmeJudgeApi.swift`. Verdict
|
|
1731
|
+
* bodies measure p99 ~1,025 chars (~335 tokens), so this is headroom rather
|
|
1732
|
+
* than a budget — but adaptive thinking spends against the same cap, and
|
|
1733
|
+
* `max_tokens` is a ceiling, not a charge: only tokens actually produced are
|
|
1734
|
+
* billed. Swift sent 1,024, so the same task judged on the same settings was
|
|
1735
|
+
* cut 8x earlier there. */
|
|
1736
|
+
const API_JUDGE_MAX_TOKENS = 8192;
|
|
1517
1737
|
function apiJudgeModel(cfg) {
|
|
1518
1738
|
return cfg.model && cfg.model.startsWith('claude') ? cfg.model : API_JUDGE_DEFAULT_MODEL;
|
|
1519
1739
|
}
|
|
@@ -1530,34 +1750,206 @@ async function callApi(prompt, cfg) {
|
|
|
1530
1750
|
});
|
|
1531
1751
|
const response = await client.messages.create({
|
|
1532
1752
|
model: apiJudgeModel(cfg),
|
|
1533
|
-
max_tokens:
|
|
1753
|
+
max_tokens: API_JUDGE_MAX_TOKENS,
|
|
1534
1754
|
thinking: { type: 'adaptive' },
|
|
1535
1755
|
messages: [{ role: 'user', content: prompt }],
|
|
1536
1756
|
});
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1757
|
+
return apiJudgeText(response);
|
|
1758
|
+
}
|
|
1759
|
+
/** The Anthropic-shaped counterpart of `judgeChatContent`, and the same rule
|
|
1760
|
+
* under Anthropic's spelling: a refusal is not a verdict, and a body cut at
|
|
1761
|
+
* `max_tokens` is one only when its JSON object closed. The text blocks are
|
|
1762
|
+
* joined first because the object can close in one block and the cut land in
|
|
1763
|
+
* the next.
|
|
1764
|
+
*
|
|
1765
|
+
* Pure and exported so the rule has a gate: it is reached only through the
|
|
1766
|
+
* Anthropic SDK, so nothing exercised it and reverting either line left both
|
|
1767
|
+
* suites green. Mirrored by `ApmeJudgeApi.content`; behavior is pinned by
|
|
1768
|
+
* `shared/apme-judge-api-response-vectors.json`, which both suites replay. */
|
|
1769
|
+
export function apiJudgeText(response) {
|
|
1770
|
+
// `content` must be an ARRAY — the same hole just closed for `choices` in
|
|
1771
|
+
// `judgeChatContent`. Without the guard a string or object map throws a
|
|
1772
|
+
// TypeError instead of a judge error, while Swift's `as? [[String: Any]] ?? []`
|
|
1773
|
+
// degrades to empty and raises a proper one.
|
|
1774
|
+
const blocks = Array.isArray(response.content) ? response.content : [];
|
|
1775
|
+
const text = blocks
|
|
1776
|
+
.filter((b) => b?.type === 'text' && typeof b.text === 'string')
|
|
1542
1777
|
.map((b) => b.text)
|
|
1543
1778
|
.join('\n')
|
|
1544
1779
|
.trim();
|
|
1780
|
+
if (response.stop_reason === 'refusal') {
|
|
1781
|
+
throw new Error('API judge refused the request (stop_reason=refusal)');
|
|
1782
|
+
}
|
|
1783
|
+
if (response.stop_reason === 'max_tokens') {
|
|
1784
|
+
throw new Error('API judge reached output limit before completion (stop_reason=max_tokens)');
|
|
1785
|
+
}
|
|
1545
1786
|
if (!text)
|
|
1546
1787
|
throw new Error(`API judge returned no text (stop_reason=${response.stop_reason})`);
|
|
1547
1788
|
return text;
|
|
1548
1789
|
}
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1790
|
+
/**
|
|
1791
|
+
* Shared MLX / OpenAI-compatible chat response gate.
|
|
1792
|
+
*
|
|
1793
|
+
* Mirrored by Swift `ApmeJudgeChatResponse`; the cases both daemons must agree
|
|
1794
|
+
* on live in `shared/apme-judge-response-vectors.json`, which both suites
|
|
1795
|
+
* replay. Three rules, and the ORDER of the first two matters because a
|
|
1796
|
+
* truncated body is still a non-empty one:
|
|
1797
|
+
*
|
|
1798
|
+
* - `choices` must be a non-empty ARRAY. Indexing `json.choices?.[0]` happily
|
|
1799
|
+
* reads `{"choices":{"0":{…}}}`, which Swift's `as? [[String: Any]]` cast
|
|
1800
|
+
* rejects — the two daemons disagreed on that shape until #286.
|
|
1801
|
+
* - Content must be a non-empty string.
|
|
1802
|
+
* - `finish_reason: "length"` is rejected, full stop. #285's rule, restored
|
|
1803
|
+
* after an exemption for "the object closed, so the verdict finished" was
|
|
1804
|
+
* tried and removed. It produced a defect in three consecutive review
|
|
1805
|
+
* rounds — first admitting a reasoning model's scratchpad when the real
|
|
1806
|
+
* verdict was cut, then refusing complete verdicts whose trailing prose
|
|
1807
|
+
* held an unmatched brace — because brace topology cannot actually tell
|
|
1808
|
+
* whether the model finished. It also had no measured beneficiary: the one
|
|
1809
|
+
* cut mode observed on this fleet is a repetition loop INSIDE the `summary`
|
|
1810
|
+
* string, where depth can never return to zero, so the exemption and this
|
|
1811
|
+
* rule agree on every real body seen. #286 item 3 lists "keep rejecting and
|
|
1812
|
+
* alert on the rate" among its options; the park log names this failure, and
|
|
1813
|
+
* a re-attempt clears it two times in three (measured over the 24 parked
|
|
1814
|
+
* tasks, 2026-09-06).
|
|
1815
|
+
*/
|
|
1816
|
+
export function judgeChatContent(payload, label) {
|
|
1817
|
+
const choices = payload?.choices;
|
|
1818
|
+
if (!Array.isArray(choices) || choices.length === 0) {
|
|
1819
|
+
throw new Error(`${label} judge returned no choices`);
|
|
1820
|
+
}
|
|
1821
|
+
const first = choices[0];
|
|
1822
|
+
const content = first?.message?.content;
|
|
1823
|
+
if (typeof content !== 'string' || content.trim().length === 0) {
|
|
1824
|
+
throw new Error(`${label} judge returned empty content`);
|
|
1825
|
+
}
|
|
1826
|
+
if (first?.finish_reason === 'length') {
|
|
1827
|
+
throw new Error(`${label} judge reached output limit before completion`);
|
|
1828
|
+
}
|
|
1829
|
+
return content;
|
|
1830
|
+
}
|
|
1831
|
+
/** Every top-level balanced `{…}` span in `text`, then the greedy
|
|
1832
|
+
* first-`{`-to-last-`}` span when it differs from all of them.
|
|
1833
|
+
*
|
|
1834
|
+
* Balanced first, so a verdict followed by prose containing a brace reads the
|
|
1835
|
+
* same on both daemons. ALL of them, not just the first, because the first is
|
|
1836
|
+
* not necessarily the verdict — a local reasoning model emits a scratchpad
|
|
1837
|
+
* object before the real one. The greedy span last, because the balanced
|
|
1838
|
+
* scanner is string-aware and `repairJudgeJson` exists for bodies whose
|
|
1839
|
+
* quoting is itself broken: a key that lost its opening quote desyncs any such
|
|
1840
|
+
* scanner, and that body used to parse. */
|
|
1841
|
+
function jsonBlockSpans(text) {
|
|
1842
|
+
const out = [];
|
|
1843
|
+
let from = 0;
|
|
1844
|
+
for (;;) {
|
|
1845
|
+
const block = extractFirstJsonBlock(text, from);
|
|
1846
|
+
if (block === null)
|
|
1847
|
+
break;
|
|
1848
|
+
out.push(block.text);
|
|
1849
|
+
from = block.end;
|
|
1850
|
+
}
|
|
1851
|
+
const greedy = text.match(/\{[\s\S]*\}/);
|
|
1852
|
+
if (greedy && !out.includes(greedy[0]))
|
|
1853
|
+
out.push(greedy[0]);
|
|
1854
|
+
return out;
|
|
1855
|
+
}
|
|
1856
|
+
function strictParseObject(block) {
|
|
1555
1857
|
try {
|
|
1556
|
-
|
|
1858
|
+
const parsed = JSON.parse(block);
|
|
1859
|
+
return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
1860
|
+
? parsed
|
|
1861
|
+
: null;
|
|
1557
1862
|
}
|
|
1558
1863
|
catch {
|
|
1559
1864
|
return null;
|
|
1560
1865
|
}
|
|
1866
|
+
}
|
|
1867
|
+
function parseOrRepairObject(block) {
|
|
1868
|
+
return strictParseObject(block) ?? strictParseObject(repairJudgeJson(block));
|
|
1869
|
+
}
|
|
1870
|
+
/** The judge's verdict object, chosen by the one field that identifies a
|
|
1871
|
+
* verdict rather than by position.
|
|
1872
|
+
*
|
|
1873
|
+
* Taking the first span that merely PARSES is how the balanced scan turned a
|
|
1874
|
+
* loud failure into a silently wrong score: `<think>{"overall":0.5}</think>`
|
|
1875
|
+
* followed by the real `{"overall":0.9}` scored the scratchpad. An unstripped
|
|
1876
|
+
* thinking block is the exact shape `reasoningEffort: "none"` exists to
|
|
1877
|
+
* suppress, i.e. the local models this judge chain targets.
|
|
1878
|
+
*
|
|
1879
|
+
* Two spans both carrying `overall` are AMBIGUOUS and resolve to null — the
|
|
1880
|
+
* loud "unparseable verdict" the greedy match produced before, and the right
|
|
1881
|
+
* answer, because a wrong score written to `evals` is strictly worse than a
|
|
1882
|
+
* skip. */
|
|
1883
|
+
function parseJudgeObject(text) {
|
|
1884
|
+
const parsed = [];
|
|
1885
|
+
for (const block of jsonBlockSpans(text)) {
|
|
1886
|
+
const obj = parseOrRepairObject(block);
|
|
1887
|
+
if (obj)
|
|
1888
|
+
parsed.push(obj);
|
|
1889
|
+
}
|
|
1890
|
+
const verdicts = parsed.filter((o) => typeof o.overall === 'number' && isFinite(o.overall));
|
|
1891
|
+
if (verdicts.length === 1)
|
|
1892
|
+
return verdicts[0];
|
|
1893
|
+
if (verdicts.length > 1)
|
|
1894
|
+
return null;
|
|
1895
|
+
return null;
|
|
1896
|
+
}
|
|
1897
|
+
/** The first BALANCED `{…}` block, mirroring Swift `extractFirstJsonBlock`.
|
|
1898
|
+
*
|
|
1899
|
+
* This used to be a greedy `/\{[\s\S]*\}/`, which spans the first `{` to the
|
|
1900
|
+
* LAST `}` in the body — so a verdict followed by any prose containing a brace
|
|
1901
|
+
* parsed here and not on the other daemon, whose scanner stops at the object's
|
|
1902
|
+
* own closing brace. That divergence became load-bearing once a cut body is
|
|
1903
|
+
* accepted when its object closed: text after the closing brace is exactly
|
|
1904
|
+
* what a `finish_reason: "length"` body has, and a model that keeps talking
|
|
1905
|
+
* past the verdict mentioning a brace is routine. Node rejected it as cut
|
|
1906
|
+
* while Swift accepted and scored it.
|
|
1907
|
+
*
|
|
1908
|
+
* Braces inside strings do not count, so an escaped brace in a `summary`
|
|
1909
|
+
* cannot end the block early. */
|
|
1910
|
+
function extractFirstJsonBlock(text, from = 0) {
|
|
1911
|
+
const start = text.indexOf('{', from);
|
|
1912
|
+
if (start < 0)
|
|
1913
|
+
return null;
|
|
1914
|
+
let depth = 0;
|
|
1915
|
+
let inString = false;
|
|
1916
|
+
let escaped = false;
|
|
1917
|
+
for (let i = start; i < text.length; i++) {
|
|
1918
|
+
const char = text[i];
|
|
1919
|
+
if (escaped) {
|
|
1920
|
+
escaped = false;
|
|
1921
|
+
continue;
|
|
1922
|
+
}
|
|
1923
|
+
if (inString) {
|
|
1924
|
+
if (char === '\\')
|
|
1925
|
+
escaped = true;
|
|
1926
|
+
else if (char === '"')
|
|
1927
|
+
inString = false;
|
|
1928
|
+
continue;
|
|
1929
|
+
}
|
|
1930
|
+
if (char === '"')
|
|
1931
|
+
inString = true;
|
|
1932
|
+
else if (char === '{')
|
|
1933
|
+
depth++;
|
|
1934
|
+
else if (char === '}') {
|
|
1935
|
+
depth--;
|
|
1936
|
+
if (depth === 0)
|
|
1937
|
+
return { text: text.slice(start, i + 1), end: i + 1 };
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
return null;
|
|
1941
|
+
}
|
|
1942
|
+
export function parseJudgeJson(text) {
|
|
1943
|
+
// Models often wrap JSON in prose or code fences, emit a comma before `}`/`]`,
|
|
1944
|
+
// or omit the opening quote of an auxiliary object key while retaining its
|
|
1945
|
+
// closing quote (both observed with Gemma 4 on long task_rollup prompts).
|
|
1946
|
+
// `parseJudgeObject` handles the extraction and the repair — and is shared
|
|
1947
|
+
// with the transport gate so the two cannot disagree about whether this body
|
|
1948
|
+
// holds a verdict. Repair only touches structural text outside strings, so
|
|
1949
|
+
// evidence content stays byte-for-byte intact.
|
|
1950
|
+
const obj = parseJudgeObject(text);
|
|
1951
|
+
if (obj === null)
|
|
1952
|
+
return null;
|
|
1561
1953
|
// Accept any numeric axis — category-specific rubrics define their own
|
|
1562
1954
|
// (conversation: accuracy/helpfulness/conciseness; research: thoroughness/…;
|
|
1563
1955
|
// planning: completeness/feasibility/clarity; etc.) A hardcoded whitelist
|
|
@@ -1584,6 +1976,90 @@ export function parseJudgeJson(text) {
|
|
|
1584
1976
|
: undefined;
|
|
1585
1977
|
return { scores, reasoning, done, missed, summary };
|
|
1586
1978
|
}
|
|
1979
|
+
function repairJudgeJson(text) {
|
|
1980
|
+
return quoteBareClosingJsonKeys(stripTrailingJsonCommas(text));
|
|
1981
|
+
}
|
|
1982
|
+
/** Repair `...,evidence":"..."` only when the bare ASCII identifier is in an
|
|
1983
|
+
* object-key position and still has its closing quote + colon. A scanner is
|
|
1984
|
+
* used instead of a regex so the same byte sequence inside evidence strings
|
|
1985
|
+
* is never changed. */
|
|
1986
|
+
function quoteBareClosingJsonKeys(text) {
|
|
1987
|
+
let result = '';
|
|
1988
|
+
let inString = false;
|
|
1989
|
+
let escaped = false;
|
|
1990
|
+
for (let i = 0; i < text.length; i++) {
|
|
1991
|
+
const char = text[i];
|
|
1992
|
+
if (inString) {
|
|
1993
|
+
result += char;
|
|
1994
|
+
if (escaped)
|
|
1995
|
+
escaped = false;
|
|
1996
|
+
else if (char === '\\')
|
|
1997
|
+
escaped = true;
|
|
1998
|
+
else if (char === '"')
|
|
1999
|
+
inString = false;
|
|
2000
|
+
continue;
|
|
2001
|
+
}
|
|
2002
|
+
if (char === '"') {
|
|
2003
|
+
inString = true;
|
|
2004
|
+
result += char;
|
|
2005
|
+
continue;
|
|
2006
|
+
}
|
|
2007
|
+
if (/[A-Za-z_]/.test(char)) {
|
|
2008
|
+
let previous = result.length - 1;
|
|
2009
|
+
while (previous >= 0 && /\s/.test(result[previous]))
|
|
2010
|
+
previous--;
|
|
2011
|
+
if (result[previous] === '{' || result[previous] === ',') {
|
|
2012
|
+
let end = i + 1;
|
|
2013
|
+
while (end < text.length && /[A-Za-z0-9_]/.test(text[end]))
|
|
2014
|
+
end++;
|
|
2015
|
+
if (text[end] === '"') {
|
|
2016
|
+
let colon = end + 1;
|
|
2017
|
+
while (colon < text.length && /\s/.test(text[colon]))
|
|
2018
|
+
colon++;
|
|
2019
|
+
if (text[colon] === ':') {
|
|
2020
|
+
result += `"${text.slice(i, end + 1)}`;
|
|
2021
|
+
i = end;
|
|
2022
|
+
continue;
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
result += char;
|
|
2028
|
+
}
|
|
2029
|
+
return result;
|
|
2030
|
+
}
|
|
2031
|
+
function stripTrailingJsonCommas(text) {
|
|
2032
|
+
let result = '';
|
|
2033
|
+
let inString = false;
|
|
2034
|
+
let escaped = false;
|
|
2035
|
+
for (let i = 0; i < text.length; i++) {
|
|
2036
|
+
const char = text[i];
|
|
2037
|
+
if (inString) {
|
|
2038
|
+
result += char;
|
|
2039
|
+
if (escaped)
|
|
2040
|
+
escaped = false;
|
|
2041
|
+
else if (char === '\\')
|
|
2042
|
+
escaped = true;
|
|
2043
|
+
else if (char === '"')
|
|
2044
|
+
inString = false;
|
|
2045
|
+
continue;
|
|
2046
|
+
}
|
|
2047
|
+
if (char === '"') {
|
|
2048
|
+
inString = true;
|
|
2049
|
+
result += char;
|
|
2050
|
+
continue;
|
|
2051
|
+
}
|
|
2052
|
+
if (char === ',') {
|
|
2053
|
+
let next = i + 1;
|
|
2054
|
+
while (next < text.length && /\s/.test(text[next]))
|
|
2055
|
+
next++;
|
|
2056
|
+
if (text[next] === '}' || text[next] === ']')
|
|
2057
|
+
continue;
|
|
2058
|
+
}
|
|
2059
|
+
result += char;
|
|
2060
|
+
}
|
|
2061
|
+
return result;
|
|
2062
|
+
}
|
|
1587
2063
|
function clamp01(n) {
|
|
1588
2064
|
if (n > 1 && n <= 10)
|
|
1589
2065
|
n = n / 10; // accept 0-10 scale and rescale
|