@apo-ai/sdk 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-task/integrations/ai-sdk.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.js +2 -2
- package/dist/agent-task/integrations/openai.d.ts +1 -1
- package/dist/agent-task/integrations/openai.js +2 -2
- package/dist/agent-task/public.d.ts +283 -5
- package/dist/agent-task/public.js +924 -78
- package/dist/agent-task/public.js.map +1 -1
- package/dist/{chunk-P43KL5AR.js → chunk-A3UYQ7M4.js} +3 -2
- package/dist/{chunk-P43KL5AR.js.map → chunk-A3UYQ7M4.js.map} +1 -1
- package/dist/{chunk-Z64A566W.js → chunk-JTWQD27M.js} +2 -1
- package/dist/chunk-JTWQD27M.js.map +1 -0
- package/dist/{chunk-LO343UZO.js → chunk-P24ECOI3.js} +4 -2
- package/dist/chunk-P24ECOI3.js.map +1 -0
- package/dist/{chunk-6H2GACBG.js → chunk-X73JGMK3.js} +3 -2
- package/dist/{chunk-6H2GACBG.js.map → chunk-X73JGMK3.js.map} +1 -1
- package/dist/otel/index.js +1 -1
- package/dist/{tracing-DoIGvJW1.d.ts → tracing-FqFObrmt.d.ts} +5 -0
- package/package.json +1 -1
- package/dist/chunk-LO343UZO.js.map +0 -1
- package/dist/chunk-Z64A566W.js.map +0 -1
|
@@ -9,28 +9,240 @@ import {
|
|
|
9
9
|
resolveApoAuthHeaders,
|
|
10
10
|
withApoRun,
|
|
11
11
|
withApoRunSync
|
|
12
|
-
} from "../chunk-
|
|
12
|
+
} from "../chunk-P24ECOI3.js";
|
|
13
13
|
import {
|
|
14
14
|
createApoTracer
|
|
15
15
|
} from "../chunk-OZWOLCJ5.js";
|
|
16
16
|
import "../chunk-HRYXLRXJ.js";
|
|
17
17
|
import {
|
|
18
18
|
createApoOpenAI
|
|
19
|
-
} from "../chunk-
|
|
19
|
+
} from "../chunk-A3UYQ7M4.js";
|
|
20
20
|
import {
|
|
21
21
|
createApoAnthropic
|
|
22
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-X73JGMK3.js";
|
|
23
23
|
import {
|
|
24
24
|
servingHostFromBaseURL
|
|
25
|
-
} from "../chunk-
|
|
25
|
+
} from "../chunk-JTWQD27M.js";
|
|
26
|
+
|
|
27
|
+
// src/agent-task/run/steer-scheduler.ts
|
|
28
|
+
var TASK_STEER_SPAN_NAME = "task.steer";
|
|
29
|
+
function createSteerScheduler(options) {
|
|
30
|
+
const { specs, session, trace: trace2 } = options;
|
|
31
|
+
const states = specs.map((spec, index) => ({
|
|
32
|
+
spec,
|
|
33
|
+
fired: false,
|
|
34
|
+
record: {
|
|
35
|
+
number: index + 1,
|
|
36
|
+
turn: spec.turn ?? 1,
|
|
37
|
+
...spec.label !== void 0 ? { label: spec.label } : {},
|
|
38
|
+
trigger: spec.when,
|
|
39
|
+
message: spec.message,
|
|
40
|
+
status: "undelivered"
|
|
41
|
+
}
|
|
42
|
+
}));
|
|
43
|
+
const toolResults = /* @__PURE__ */ new Map();
|
|
44
|
+
const assistantReplies = /* @__PURE__ */ new Map();
|
|
45
|
+
const activeTurns = /* @__PURE__ */ new Set();
|
|
46
|
+
const runStartSeen = /* @__PURE__ */ new Set();
|
|
47
|
+
const progressSeen = /* @__PURE__ */ new Set();
|
|
48
|
+
let chain = Promise.resolve();
|
|
49
|
+
const enqueue = (step) => {
|
|
50
|
+
const next = chain.then(step, step);
|
|
51
|
+
chain = next.catch(() => {
|
|
52
|
+
});
|
|
53
|
+
return next;
|
|
54
|
+
};
|
|
55
|
+
function emitSteerEvent(record) {
|
|
56
|
+
try {
|
|
57
|
+
trace2.recordEvent({
|
|
58
|
+
step_name: TASK_STEER_SPAN_NAME,
|
|
59
|
+
observation_type: "CHAIN",
|
|
60
|
+
parent_call_id: options.turnSpanId?.(record.turn),
|
|
61
|
+
metadata: {
|
|
62
|
+
steerNumber: record.number,
|
|
63
|
+
turn: record.turn,
|
|
64
|
+
...record.label !== void 0 ? { label: record.label } : {},
|
|
65
|
+
trigger: JSON.stringify(record.trigger),
|
|
66
|
+
status: record.status,
|
|
67
|
+
...record.boundary !== void 0 ? { boundary: record.boundary } : {},
|
|
68
|
+
...record.reason !== void 0 ? { reason: record.reason } : {}
|
|
69
|
+
},
|
|
70
|
+
output: { ...record }
|
|
71
|
+
});
|
|
72
|
+
} catch {
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function notifyHost(record) {
|
|
76
|
+
try {
|
|
77
|
+
options.onSteer?.(record);
|
|
78
|
+
} catch (error) {
|
|
79
|
+
console.error(
|
|
80
|
+
"[AgentTask] onSteer callback failed:",
|
|
81
|
+
error instanceof Error ? error.message : String(error)
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function finalize(state, reason) {
|
|
86
|
+
state.record.status = "undelivered";
|
|
87
|
+
state.record.reason = reason;
|
|
88
|
+
emitSteerEvent(state.record);
|
|
89
|
+
notifyHost(state.record);
|
|
90
|
+
}
|
|
91
|
+
async function deliver(state, turnNumber) {
|
|
92
|
+
state.fired = true;
|
|
93
|
+
try {
|
|
94
|
+
const result = await session.steer?.(state.spec.message, {
|
|
95
|
+
trace: trace2,
|
|
96
|
+
turnNumber,
|
|
97
|
+
steerNumber: state.record.number
|
|
98
|
+
});
|
|
99
|
+
state.record.status = "delivered";
|
|
100
|
+
state.record.deliveredAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
101
|
+
state.record.boundary = result?.boundary ?? "tool_results";
|
|
102
|
+
} catch (error) {
|
|
103
|
+
state.record.status = "error";
|
|
104
|
+
state.record.reason = error instanceof Error ? error.message : String(error);
|
|
105
|
+
}
|
|
106
|
+
emitSteerEvent(state.record);
|
|
107
|
+
notifyHost(state.record);
|
|
108
|
+
}
|
|
109
|
+
function triggerSatisfied(trigger, turnNumber) {
|
|
110
|
+
if (trigger === "runStart") return runStartSeen.has(turnNumber);
|
|
111
|
+
if ("toolResults" in trigger) {
|
|
112
|
+
return (toolResults.get(turnNumber) ?? 0) >= trigger.toolResults;
|
|
113
|
+
}
|
|
114
|
+
return (assistantReplies.get(turnNumber) ?? 0) >= trigger.assistantReply;
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
onTurnStart(turnNumber) {
|
|
118
|
+
return enqueue(async () => {
|
|
119
|
+
toolResults.set(turnNumber, 0);
|
|
120
|
+
assistantReplies.set(turnNumber, 0);
|
|
121
|
+
activeTurns.add(turnNumber);
|
|
122
|
+
});
|
|
123
|
+
},
|
|
124
|
+
onProgressEvent(turnNumber, event) {
|
|
125
|
+
return enqueue(async () => {
|
|
126
|
+
if (!activeTurns.has(turnNumber)) return;
|
|
127
|
+
progressSeen.add(turnNumber);
|
|
128
|
+
if (event.kind === "run_start") {
|
|
129
|
+
runStartSeen.add(turnNumber);
|
|
130
|
+
} else if (event.kind === "tool_result") {
|
|
131
|
+
toolResults.set(turnNumber, (toolResults.get(turnNumber) ?? 0) + 1);
|
|
132
|
+
} else if (event.kind === "assistant_reply") {
|
|
133
|
+
assistantReplies.set(turnNumber, (assistantReplies.get(turnNumber) ?? 0) + 1);
|
|
134
|
+
}
|
|
135
|
+
for (const state of states) {
|
|
136
|
+
if (state.fired || state.record.turn !== turnNumber) continue;
|
|
137
|
+
if (triggerSatisfied(state.spec.when, turnNumber)) {
|
|
138
|
+
await deliver(state, turnNumber);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
onTurnEnd(turnNumber) {
|
|
144
|
+
return enqueue(async () => {
|
|
145
|
+
activeTurns.delete(turnNumber);
|
|
146
|
+
const reason = progressSeen.has(turnNumber) ? "turn ended before trigger fired" : "no progress events observed for this turn (adapter did not call notifyAgentEvent)";
|
|
147
|
+
for (const state of states) {
|
|
148
|
+
if (state.fired || state.record.turn !== turnNumber) continue;
|
|
149
|
+
finalize(state, reason);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
},
|
|
153
|
+
onRunEnd(maxTurnReached) {
|
|
154
|
+
return enqueue(async () => {
|
|
155
|
+
for (const state of states) {
|
|
156
|
+
if (state.fired || state.record.status !== "undelivered") continue;
|
|
157
|
+
if (state.record.turn > maxTurnReached) {
|
|
158
|
+
finalize(state, "target turn never ran");
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
},
|
|
163
|
+
recordsByTurn() {
|
|
164
|
+
const byTurn = /* @__PURE__ */ new Map();
|
|
165
|
+
for (const state of states) {
|
|
166
|
+
const list = byTurn.get(state.record.turn) ?? [];
|
|
167
|
+
list.push(state.record);
|
|
168
|
+
byTurn.set(state.record.turn, list);
|
|
169
|
+
}
|
|
170
|
+
return byTurn;
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
}
|
|
26
174
|
|
|
27
175
|
// src/agent-task/trace-projection/view.ts
|
|
28
176
|
var TASK_TURN_SPAN_NAME = "task.turn";
|
|
177
|
+
function steerRecordFrom(obs) {
|
|
178
|
+
const output = obs.output;
|
|
179
|
+
if (!output || typeof output !== "object") return void 0;
|
|
180
|
+
const asRecord2 = (v) => v !== null && typeof v === "object" && typeof v.number === "number" ? v : void 0;
|
|
181
|
+
const direct = asRecord2(output);
|
|
182
|
+
if (direct) return direct;
|
|
183
|
+
const tryParse = (text) => {
|
|
184
|
+
if (typeof text !== "string") return void 0;
|
|
185
|
+
try {
|
|
186
|
+
return asRecord2(JSON.parse(text));
|
|
187
|
+
} catch {
|
|
188
|
+
return void 0;
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
const fromText = tryParse(output.text);
|
|
192
|
+
if (fromText) return fromText;
|
|
193
|
+
const messages = output.messages;
|
|
194
|
+
if (Array.isArray(messages)) {
|
|
195
|
+
const content = messages[0]?.content;
|
|
196
|
+
const fromMessage = tryParse(content);
|
|
197
|
+
if (fromMessage) return fromMessage;
|
|
198
|
+
}
|
|
199
|
+
return void 0;
|
|
200
|
+
}
|
|
201
|
+
function traceSteerFrom(obs) {
|
|
202
|
+
const meta = obs.metadata;
|
|
203
|
+
const record = steerRecordFrom(obs);
|
|
204
|
+
const number = typeof record?.number === "number" ? record.number : typeof meta?.steerNumber === "number" ? meta.steerNumber : void 0;
|
|
205
|
+
if (number === void 0) return void 0;
|
|
206
|
+
const source = record ?? {};
|
|
207
|
+
const read = (key) => source[key] !== void 0 ? source[key] : meta?.[key];
|
|
208
|
+
const status = read("status");
|
|
209
|
+
const steer2 = {
|
|
210
|
+
number,
|
|
211
|
+
spanId: obs.spanId,
|
|
212
|
+
turn: typeof read("turn") === "number" ? read("turn") : 1,
|
|
213
|
+
trigger: triggerToString(read("trigger")),
|
|
214
|
+
message: source.message,
|
|
215
|
+
...isSteerStatus(status) ? { status } : { status: "undelivered" }
|
|
216
|
+
};
|
|
217
|
+
const label = read("label");
|
|
218
|
+
if (typeof label === "string") steer2.label = label;
|
|
219
|
+
const deliveredAt = read("deliveredAt");
|
|
220
|
+
if (typeof deliveredAt === "string") steer2.deliveredAt = deliveredAt;
|
|
221
|
+
if (obs.startedAt !== void 0) steer2.spanStartedAt = obs.startedAt;
|
|
222
|
+
const boundary = read("boundary");
|
|
223
|
+
if (typeof boundary === "string") steer2.boundary = boundary;
|
|
224
|
+
const reason = read("reason");
|
|
225
|
+
if (typeof reason === "string") steer2.reason = reason;
|
|
226
|
+
return steer2;
|
|
227
|
+
}
|
|
228
|
+
function isSteerStatus(v) {
|
|
229
|
+
return v === "delivered" || v === "undelivered" || v === "error";
|
|
230
|
+
}
|
|
231
|
+
function triggerToString(v) {
|
|
232
|
+
if (typeof v === "string") return v;
|
|
233
|
+
if (v !== null && typeof v === "object") {
|
|
234
|
+
try {
|
|
235
|
+
return JSON.stringify(v) ?? "unknown";
|
|
236
|
+
} catch {
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return "unknown";
|
|
240
|
+
}
|
|
29
241
|
function invocationOrderKey(obs) {
|
|
30
242
|
const hasTs = obs.startedAt != null ? 0 : 1;
|
|
31
243
|
return [hasTs, obs.startedAt ?? "", obs.spanId];
|
|
32
244
|
}
|
|
33
|
-
var TraceView = class {
|
|
245
|
+
var TraceView = class _TraceView {
|
|
34
246
|
constructor(snapshot) {
|
|
35
247
|
this.snapshot = snapshot;
|
|
36
248
|
}
|
|
@@ -79,6 +291,56 @@ var TraceView = class {
|
|
|
79
291
|
startedAt: o.startedAt
|
|
80
292
|
}));
|
|
81
293
|
}
|
|
294
|
+
/**
|
|
295
|
+
* Mid-run steers, derived from `task.steer` observations, in number order.
|
|
296
|
+
* Empty when the trace carries none — steering assertions fail closed on
|
|
297
|
+
* the empty case instead of passing vacuously.
|
|
298
|
+
*/
|
|
299
|
+
get steers() {
|
|
300
|
+
const out = [];
|
|
301
|
+
for (const obs of this.sortedObservations) {
|
|
302
|
+
if (obs.name !== TASK_STEER_SPAN_NAME) continue;
|
|
303
|
+
const steer2 = traceSteerFrom(obs);
|
|
304
|
+
if (steer2) out.push(steer2);
|
|
305
|
+
}
|
|
306
|
+
return out.sort((a, b) => a.number - b.number);
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Whether an observation started after the steer observation, using the
|
|
310
|
+
* same ordering key as everywhere else in this class (startedAt, then
|
|
311
|
+
* span id). The id tie-break matters on the local tee path, whose ISO
|
|
312
|
+
* timestamps truncate to milliseconds — a steer and the generation that
|
|
313
|
+
* consumes it can share one.
|
|
314
|
+
*/
|
|
315
|
+
static startedAfter(obs, steer2) {
|
|
316
|
+
if (obs.startedAt == null) return true;
|
|
317
|
+
if (steer2.spanStartedAt == null) return false;
|
|
318
|
+
if (obs.startedAt !== steer2.spanStartedAt) {
|
|
319
|
+
return obs.startedAt > steer2.spanStartedAt;
|
|
320
|
+
}
|
|
321
|
+
return obs.spanId > steer2.spanId;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* A TraceView over the run after steer n was delivered: observations that
|
|
325
|
+
* started after the steer's delivery timestamp (the first generation that
|
|
326
|
+
* consumed it through run end). Missing timestamps sort after timestamped
|
|
327
|
+
* observations per the projection contract, so they belong to the window
|
|
328
|
+
* too. An undelivered steer yields an empty view — assertions inside it
|
|
329
|
+
* fail on missing evidence rather than passing vacuously.
|
|
330
|
+
*/
|
|
331
|
+
windowAfterSteer(n) {
|
|
332
|
+
const steer2 = this.steers.find((s) => s.number === n);
|
|
333
|
+
if (!steer2) {
|
|
334
|
+
throw new Error(`No steer number ${n} in this trace.`);
|
|
335
|
+
}
|
|
336
|
+
if (steer2.spanStartedAt === void 0) {
|
|
337
|
+
return new _TraceView({ ...this.snapshot, observations: [] });
|
|
338
|
+
}
|
|
339
|
+
const filtered = this.snapshot.observations.filter(
|
|
340
|
+
(o) => o.spanId !== steer2.spanId && _TraceView.startedAfter(o, steer2)
|
|
341
|
+
);
|
|
342
|
+
return new _TraceView({ ...this.snapshot, observations: filtered });
|
|
343
|
+
}
|
|
82
344
|
/** Last assistant message content — the agent's "reply". Empty if none. */
|
|
83
345
|
get reply() {
|
|
84
346
|
const msgs = this.messages;
|
|
@@ -382,7 +644,186 @@ function levenshtein(a, b) {
|
|
|
382
644
|
// src/agent-task/checks/judge.ts
|
|
383
645
|
import { createHash } from "crypto";
|
|
384
646
|
|
|
647
|
+
// src/agent-task/checks/reasoning-loop.ts
|
|
648
|
+
var MAX_UNIT_CHARS = 64;
|
|
649
|
+
var MIN_UNIT_COPIES = 100;
|
|
650
|
+
var MIN_REPEAT_CHARS = 1e3;
|
|
651
|
+
var MAX_CHAR_RUN = 3;
|
|
652
|
+
var GRAM = 24;
|
|
653
|
+
var DIVERSITY_WINDOW = 3e3;
|
|
654
|
+
var MIN_DIVERSITY = 0.1;
|
|
655
|
+
var DIVERSITY_STRIDE = 256;
|
|
656
|
+
var MAX_DIVERSITY_LINE_CHARS = 2e3;
|
|
657
|
+
var RECYCLE_GRAM = 32;
|
|
658
|
+
var RECYCLE_WINDOW = 5e3;
|
|
659
|
+
var MAX_RECYCLED = 0.95;
|
|
660
|
+
var isFoldable = (ch) => /[\p{L}\p{N}]/u.test(ch);
|
|
661
|
+
function foldForRepetition(text) {
|
|
662
|
+
let out = "";
|
|
663
|
+
let last = "";
|
|
664
|
+
let run = 0;
|
|
665
|
+
for (const raw of text) {
|
|
666
|
+
if (!isFoldable(raw)) continue;
|
|
667
|
+
const ch = raw.toLowerCase();
|
|
668
|
+
run = ch === last ? run + 1 : 1;
|
|
669
|
+
last = ch;
|
|
670
|
+
if (run <= MAX_CHAR_RUN) out += ch;
|
|
671
|
+
}
|
|
672
|
+
return out;
|
|
673
|
+
}
|
|
674
|
+
function isTableLine(line) {
|
|
675
|
+
const trimmed = line.trim();
|
|
676
|
+
if (trimmed.startsWith("|")) return true;
|
|
677
|
+
return (trimmed.match(/\|/g)?.length ?? 0) >= 3;
|
|
678
|
+
}
|
|
679
|
+
function diversity(folded) {
|
|
680
|
+
const total = folded.length - GRAM + 1;
|
|
681
|
+
if (total <= 0) return 1;
|
|
682
|
+
const grams = /* @__PURE__ */ new Set();
|
|
683
|
+
for (let i = 0; i < total; i++) grams.add(folded.slice(i, i + GRAM));
|
|
684
|
+
return grams.size / total;
|
|
685
|
+
}
|
|
686
|
+
var RepetitionGuard = class {
|
|
687
|
+
constructor(opts) {
|
|
688
|
+
this.rawChars = 0;
|
|
689
|
+
// Repeated-unit state: the folded tail, and for each period p the number
|
|
690
|
+
// of consecutive folded chars equal to the char p before them.
|
|
691
|
+
this.tail = "";
|
|
692
|
+
this.lastChar = "";
|
|
693
|
+
this.charRun = 0;
|
|
694
|
+
this.periodRuns = new Array(MAX_UNIT_CHARS + 1).fill(0);
|
|
695
|
+
// Low-diversity state: the current raw line, and the folded window built
|
|
696
|
+
// from finished lines that are not tables.
|
|
697
|
+
this.line = "";
|
|
698
|
+
this.window = "";
|
|
699
|
+
this.sinceMeasured = 0;
|
|
700
|
+
// Recycled state: every gram of the filtered folded stream so far, and for
|
|
701
|
+
// the window's grams whether each had been seen when it arrived.
|
|
702
|
+
this.seenGrams = /* @__PURE__ */ new Set();
|
|
703
|
+
this.gramTail = "";
|
|
704
|
+
this.windowFlags = [];
|
|
705
|
+
this.flagHead = 0;
|
|
706
|
+
this.recycledInWindow = 0;
|
|
707
|
+
this.checkDiversity = opts.diversity;
|
|
708
|
+
}
|
|
709
|
+
get tripped() {
|
|
710
|
+
return this.trip;
|
|
711
|
+
}
|
|
712
|
+
push(delta) {
|
|
713
|
+
if (this.trip || !delta) return this.trip;
|
|
714
|
+
for (const raw of delta) {
|
|
715
|
+
this.rawChars++;
|
|
716
|
+
if (this.checkDiversity) this.pushLineChar(raw);
|
|
717
|
+
if (isFoldable(raw)) this.pushFolded(raw.toLowerCase());
|
|
718
|
+
if (this.trip) break;
|
|
719
|
+
}
|
|
720
|
+
return this.trip;
|
|
721
|
+
}
|
|
722
|
+
pushFolded(ch) {
|
|
723
|
+
this.charRun = ch === this.lastChar ? this.charRun + 1 : 1;
|
|
724
|
+
this.lastChar = ch;
|
|
725
|
+
if (this.charRun > MAX_CHAR_RUN) return;
|
|
726
|
+
this.tail += ch;
|
|
727
|
+
const n = this.tail.length - 1;
|
|
728
|
+
for (let p = 1; p <= MAX_UNIT_CHARS; p++) {
|
|
729
|
+
this.periodRuns[p] = n >= p && this.tail[n - p] === ch ? this.periodRuns[p] + 1 : 0;
|
|
730
|
+
const span = this.periodRuns[p] + p;
|
|
731
|
+
if (span >= MIN_REPEAT_CHARS && span / p >= MIN_UNIT_COPIES) {
|
|
732
|
+
this.trip = {
|
|
733
|
+
rule: "repeated-unit",
|
|
734
|
+
atChar: this.rawChars,
|
|
735
|
+
unit: this.tail.slice(-p),
|
|
736
|
+
copies: Math.floor(span / p)
|
|
737
|
+
};
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
if (this.tail.length > MAX_UNIT_CHARS * 2) this.tail = this.tail.slice(-MAX_UNIT_CHARS);
|
|
742
|
+
}
|
|
743
|
+
pushLineChar(raw) {
|
|
744
|
+
if (raw !== "\n") {
|
|
745
|
+
if (this.line.length <= MAX_DIVERSITY_LINE_CHARS) this.line += raw;
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
const line = this.line;
|
|
749
|
+
this.line = "";
|
|
750
|
+
if (line.length > MAX_DIVERSITY_LINE_CHARS || isTableLine(line)) return;
|
|
751
|
+
const folded = foldForRepetition(line);
|
|
752
|
+
if (!folded) return;
|
|
753
|
+
this.pushRecycled(folded);
|
|
754
|
+
this.window = (this.window + folded).slice(-DIVERSITY_WINDOW);
|
|
755
|
+
this.sinceMeasured += folded.length;
|
|
756
|
+
if (this.window.length < DIVERSITY_WINDOW || this.sinceMeasured < DIVERSITY_STRIDE) return;
|
|
757
|
+
this.sinceMeasured = 0;
|
|
758
|
+
const ratio = diversity(this.window);
|
|
759
|
+
if (ratio < MIN_DIVERSITY) {
|
|
760
|
+
this.trip = { rule: "low-diversity", atChar: this.rawChars, diversity: ratio };
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
const windowGrams = this.windowFlags.length - this.flagHead;
|
|
764
|
+
if (windowGrams < RECYCLE_WINDOW) return;
|
|
765
|
+
const recycled = this.recycledInWindow / windowGrams;
|
|
766
|
+
if (recycled >= MAX_RECYCLED) {
|
|
767
|
+
this.trip = { rule: "recycled", atChar: this.rawChars, recycled };
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
pushRecycled(folded) {
|
|
771
|
+
for (const ch of folded) {
|
|
772
|
+
this.gramTail = (this.gramTail + ch).slice(-RECYCLE_GRAM);
|
|
773
|
+
if (this.gramTail.length < RECYCLE_GRAM) continue;
|
|
774
|
+
const seen = this.seenGrams.has(this.gramTail);
|
|
775
|
+
if (!seen) this.seenGrams.add(this.gramTail);
|
|
776
|
+
this.windowFlags.push(seen);
|
|
777
|
+
if (seen) this.recycledInWindow++;
|
|
778
|
+
if (this.windowFlags.length - this.flagHead > RECYCLE_WINDOW) {
|
|
779
|
+
if (this.windowFlags[this.flagHead++]) this.recycledInWindow--;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
if (this.flagHead > RECYCLE_WINDOW * 4) {
|
|
783
|
+
this.windowFlags.splice(0, this.flagHead);
|
|
784
|
+
this.flagHead = 0;
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
};
|
|
788
|
+
function describeTrip(trip) {
|
|
789
|
+
switch (trip.rule) {
|
|
790
|
+
case "repeated-unit":
|
|
791
|
+
return `"${trip.unit}" repeated ${trip.copies}\xD7 at char ${trip.atChar}`;
|
|
792
|
+
case "low-diversity":
|
|
793
|
+
return `distinct-${GRAM}-gram ratio ${trip.diversity.toFixed(3)} at char ${trip.atChar}`;
|
|
794
|
+
case "recycled":
|
|
795
|
+
return `${Math.round(trip.recycled * 100)}% of the last ${RECYCLE_WINDOW} chars replay earlier reasoning, at char ${trip.atChar}`;
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
|
|
385
799
|
// src/agent-task/checks/second-judge.ts
|
|
800
|
+
function secondJudgeFacts(check) {
|
|
801
|
+
const sj = check.judge?.secondJudge;
|
|
802
|
+
if (!sj) return { kind: "none" };
|
|
803
|
+
if (sj.skipped) return { kind: "skipped" };
|
|
804
|
+
if (sj.error || sj.choice == null) return { kind: "error" };
|
|
805
|
+
const conf = sj.confidence ?? 0;
|
|
806
|
+
if (sj.choice === "pass" !== (check.pass === true)) {
|
|
807
|
+
return { kind: "split", confidence: conf };
|
|
808
|
+
}
|
|
809
|
+
if (conf < 0.6) return { kind: "unsure", confidence: conf };
|
|
810
|
+
return { kind: "agree", confidence: conf };
|
|
811
|
+
}
|
|
812
|
+
function secondJudgeTakeaway(check) {
|
|
813
|
+
const facts = secondJudgeFacts(check);
|
|
814
|
+
switch (facts.kind) {
|
|
815
|
+
case "split":
|
|
816
|
+
return `Verdicts differ \u2014 second judge contradicts at ${facts.confidence.toFixed(2)} confidence.`;
|
|
817
|
+
case "unsure":
|
|
818
|
+
return `Second judge unsure (${facts.confidence.toFixed(2)}) \u2014 weak corroboration.`;
|
|
819
|
+
case "error":
|
|
820
|
+
return `Second opinion failed to arrive${check.judge?.secondJudge?.error ? ` (${check.judge.secondJudge.error})` : ""}.`;
|
|
821
|
+
case "skipped":
|
|
822
|
+
return `Second opinion skipped${check.judge?.secondJudge?.skipped ? ` \u2014 ${check.judge.secondJudge.skipped}` : ""}.`;
|
|
823
|
+
default:
|
|
824
|
+
return null;
|
|
825
|
+
}
|
|
826
|
+
}
|
|
386
827
|
var SECOND_JUDGE_TIMEOUT_MS = 3e4;
|
|
387
828
|
var OPENROUTER_RESERVED_HINT = "Point APO_SECOND_JUDGE_BASE_URL at an OpenRouter-style decisions API (e.g. https://openrouter.ai/api/v1) or set APO_SECOND_JUDGE_PROVIDER=openrouter";
|
|
388
829
|
var openRouterProvider = {
|
|
@@ -461,9 +902,24 @@ function resolveSecondJudgeProvider(baseURL) {
|
|
|
461
902
|
}
|
|
462
903
|
return /(^|\.)api\.openai\.com$/.test(new URL(baseURL).hostname) ? openAIProvider : openRouterProvider;
|
|
463
904
|
}
|
|
905
|
+
var INPUT_TOO_LARGE_MARKERS = [
|
|
906
|
+
"max_tokens_exceeded",
|
|
907
|
+
"context_length_exceeded",
|
|
908
|
+
"maximum context length"
|
|
909
|
+
];
|
|
910
|
+
function isInputTooLargeRejection(status, body) {
|
|
911
|
+
if (status !== 400 && status !== 413) return false;
|
|
912
|
+
return INPUT_TOO_LARGE_MARKERS.some((marker2) => body.includes(marker2));
|
|
913
|
+
}
|
|
914
|
+
function estimateTokens(text) {
|
|
915
|
+
return Math.round(text.length / 4 / 1e3) * 1e3;
|
|
916
|
+
}
|
|
464
917
|
async function callSecondJudge(args) {
|
|
465
918
|
const startedAt = Date.now();
|
|
466
|
-
const evidence = {
|
|
919
|
+
const evidence = {
|
|
920
|
+
model: args.model,
|
|
921
|
+
...args.projected ? { projected: true } : {}
|
|
922
|
+
};
|
|
467
923
|
try {
|
|
468
924
|
const provider = resolveSecondJudgeProvider(args.baseURL);
|
|
469
925
|
if (provider.reserved) {
|
|
@@ -481,6 +937,10 @@ async function callSecondJudge(args) {
|
|
|
481
937
|
});
|
|
482
938
|
if (!response.ok) {
|
|
483
939
|
const body = await response.text().catch(() => "");
|
|
940
|
+
if (isInputTooLargeRejection(response.status, body)) {
|
|
941
|
+
evidence.skipped = `input ~${estimateTokens(args.state).toLocaleString("en-US")} tokens exceeds ${args.model}'s context limit \u2014 project a smaller view with t.judge(value, instruction, { secondJudgeValue })`;
|
|
942
|
+
return evidence;
|
|
943
|
+
}
|
|
484
944
|
evidence.error = `Second judge API ${response.status}: ${body.slice(0, 200)}`;
|
|
485
945
|
return evidence;
|
|
486
946
|
}
|
|
@@ -520,8 +980,19 @@ var JudgeUnavailableError = class extends Error {
|
|
|
520
980
|
this.name = "JudgeUnavailableError";
|
|
521
981
|
}
|
|
522
982
|
};
|
|
523
|
-
var
|
|
524
|
-
|
|
983
|
+
var JudgeReasoningLoopError = class extends JudgeUnavailableError {
|
|
984
|
+
constructor(trip, channel) {
|
|
985
|
+
super(`judge reasoning loop: ${channel} ${describeTrip(trip)}`);
|
|
986
|
+
this.name = "JudgeReasoningLoopError";
|
|
987
|
+
}
|
|
988
|
+
};
|
|
989
|
+
function isJudgeLoopGuardEnabled() {
|
|
990
|
+
const value = process.env.APO_JUDGE_LOOP_GUARD?.trim().toLowerCase();
|
|
991
|
+
return value !== "0" && value !== "false" && value !== "off";
|
|
992
|
+
}
|
|
993
|
+
var QUOTE_RULE = " Inside the reasoning, quote the output with single quotes, never with unescaped double quotes.";
|
|
994
|
+
var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}.' + QUOTE_RULE;
|
|
995
|
+
var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}.' + QUOTE_RULE;
|
|
525
996
|
function isJudgeVerdictFirstOverrideEnabled() {
|
|
526
997
|
const value = process.env.APO_JUDGE_VERDICT_FIRST?.trim().toLowerCase();
|
|
527
998
|
return value === "1" || value === "true";
|
|
@@ -532,6 +1003,23 @@ function judgeResponseContract() {
|
|
|
532
1003
|
function judgeContractInUse() {
|
|
533
1004
|
return isJudgeVerdictFirstOverrideEnabled() ? "verdict-first" : "reasoning-first";
|
|
534
1005
|
}
|
|
1006
|
+
function judgeResponseFormat() {
|
|
1007
|
+
const properties = isJudgeVerdictFirstOverrideEnabled() ? { pass: { type: "boolean" }, reasoning: { type: "string" } } : { reasoning: { type: "string" }, pass: { type: "boolean" } };
|
|
1008
|
+
return {
|
|
1009
|
+
type: "json_schema",
|
|
1010
|
+
json_schema: {
|
|
1011
|
+
name: "judge_verdict",
|
|
1012
|
+
strict: true,
|
|
1013
|
+
schema: {
|
|
1014
|
+
type: "object",
|
|
1015
|
+
properties,
|
|
1016
|
+
required: Object.keys(properties),
|
|
1017
|
+
additionalProperties: false
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
var JSON_OBJECT_FORMAT = { type: "json_object" };
|
|
535
1023
|
function judgeSystemPrompt() {
|
|
536
1024
|
return `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${judgeResponseContract()}`;
|
|
537
1025
|
}
|
|
@@ -590,9 +1078,14 @@ function parseJudgeJson(raw) {
|
|
|
590
1078
|
} catch {
|
|
591
1079
|
}
|
|
592
1080
|
}
|
|
1081
|
+
return {};
|
|
1082
|
+
}
|
|
1083
|
+
function judgeVerdict(raw) {
|
|
1084
|
+
const parsed = parseJudgeJson(raw);
|
|
1085
|
+
if (typeof parsed.pass !== "boolean") return void 0;
|
|
593
1086
|
return {
|
|
594
|
-
pass:
|
|
595
|
-
reasoning:
|
|
1087
|
+
pass: parsed.pass,
|
|
1088
|
+
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : ""
|
|
596
1089
|
};
|
|
597
1090
|
}
|
|
598
1091
|
var JUDGE_IDLE_TIMEOUT_MS = 9e4;
|
|
@@ -621,7 +1114,7 @@ function retryDelayMs(retryAfter) {
|
|
|
621
1114
|
if (!Number.isFinite(ms)) return JUDGE_RETRY_DELAY_MS;
|
|
622
1115
|
return Math.min(Math.max(ms, JUDGE_RETRY_DELAY_MS), JUDGE_RETRY_AFTER_CAP_MS);
|
|
623
1116
|
}
|
|
624
|
-
async function readCompletion(response, onData) {
|
|
1117
|
+
async function readCompletion(response, onData, guards) {
|
|
625
1118
|
const contentType = response.headers.get("content-type") ?? "";
|
|
626
1119
|
if (!contentType.includes("text/event-stream")) {
|
|
627
1120
|
const data = await response.json();
|
|
@@ -658,7 +1151,15 @@ async function readCompletion(response, onData) {
|
|
|
658
1151
|
}
|
|
659
1152
|
if (!chunk) return;
|
|
660
1153
|
const choice = chunk.choices?.[0];
|
|
661
|
-
|
|
1154
|
+
const content = choice?.delta?.content ?? "";
|
|
1155
|
+
text += content;
|
|
1156
|
+
if (guards) {
|
|
1157
|
+
const thinking = choice?.delta?.reasoning_content ?? choice?.delta?.reasoning ?? "";
|
|
1158
|
+
const reasoningTrip = guards.reasoning.push(thinking);
|
|
1159
|
+
if (reasoningTrip) throw new JudgeReasoningLoopError(reasoningTrip, "reasoning");
|
|
1160
|
+
const contentTrip = guards.content.push(content);
|
|
1161
|
+
if (contentTrip) throw new JudgeReasoningLoopError(contentTrip, "content");
|
|
1162
|
+
}
|
|
662
1163
|
if (choice?.finish_reason) finishReason = choice.finish_reason;
|
|
663
1164
|
if (chunk.usage) usage = chunk.usage;
|
|
664
1165
|
};
|
|
@@ -735,33 +1236,32 @@ function runWithSharedPrefix(key, task2) {
|
|
|
735
1236
|
);
|
|
736
1237
|
return next;
|
|
737
1238
|
}
|
|
1239
|
+
function defaultJudgeBaseURL(baseURL) {
|
|
1240
|
+
return baseURL ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1";
|
|
1241
|
+
}
|
|
1242
|
+
function defaultJudgeAPIKey(apiKey) {
|
|
1243
|
+
return apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
|
|
1244
|
+
}
|
|
738
1245
|
async function callJudge(args) {
|
|
739
|
-
const baseURL = args.baseURL
|
|
740
|
-
const apiKey = args.apiKey
|
|
741
|
-
const deliverableText
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
1246
|
+
const baseURL = defaultJudgeBaseURL(args.baseURL);
|
|
1247
|
+
const apiKey = defaultJudgeAPIKey(args.apiKey);
|
|
1248
|
+
const { briefingText, instructionText, systemPromptText, deliverableText, secondJudgeState, secondJudgeProjected } = buildJudgePromptParts({
|
|
1249
|
+
values: args.values,
|
|
1250
|
+
instruction: args.instruction,
|
|
1251
|
+
...args.prompt ? { prompt: args.prompt } : {},
|
|
1252
|
+
...args.context ? { context: args.context } : {},
|
|
1253
|
+
...args.secondJudgeValue !== void 0 ? { secondJudgeValue: args.secondJudgeValue } : {}
|
|
1254
|
+
});
|
|
747
1255
|
const secondJudgeModel = resolveSecondJudgeModel();
|
|
748
|
-
const
|
|
749
|
-
|
|
750
|
-
const secondJudgeDeliverableText = `Values to evaluate:
|
|
751
|
-
${formatJudgeValues(secondJudgeValues)}`;
|
|
752
|
-
const secondJudgePromise = secondJudgeModel ? callSecondJudge({
|
|
753
|
-
state: `${briefingText}
|
|
754
|
-
|
|
755
|
-
${secondJudgeDeliverableText}
|
|
756
|
-
|
|
757
|
-
${instructionText}`,
|
|
1256
|
+
const secondJudgePromise = args.prefetchedSecondJudge ? Promise.resolve(args.prefetchedSecondJudge) : secondJudgeModel ? callSecondJudge({
|
|
1257
|
+
state: secondJudgeState,
|
|
758
1258
|
model: secondJudgeModel,
|
|
759
1259
|
baseURL: resolveSecondJudgeBaseURL(baseURL),
|
|
760
1260
|
apiKey: resolveSecondJudgeAPIKey(apiKey),
|
|
761
1261
|
...secondJudgeProjected ? { projected: true } : {}
|
|
762
1262
|
}) : void 0;
|
|
763
1263
|
const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
|
|
764
|
-
const requestBody = (stream) => JSON.stringify({
|
|
1264
|
+
const requestBody = (stream, schema) => JSON.stringify({
|
|
765
1265
|
model: args.model,
|
|
766
1266
|
prompt_cache_key: promptCacheKey(cacheKey),
|
|
767
1267
|
messages: [
|
|
@@ -779,10 +1279,10 @@ ${instructionText}`,
|
|
|
779
1279
|
{ role: "user", content: instructionText }
|
|
780
1280
|
],
|
|
781
1281
|
temperature: 0,
|
|
782
|
-
response_format:
|
|
1282
|
+
response_format: schema ? judgeResponseFormat() : JSON_OBJECT_FORMAT,
|
|
783
1283
|
...stream ? { stream: true, stream_options: { include_usage: true } } : {}
|
|
784
1284
|
});
|
|
785
|
-
const attempt = async (stream, maxDeadline) => {
|
|
1285
|
+
const attempt = async (stream, schema, maxDeadline) => {
|
|
786
1286
|
const controller = new AbortController();
|
|
787
1287
|
const expire = (reason, name) => () => controller.abort(new DOMException(reason, name));
|
|
788
1288
|
const firstDataMs = judgeFirstDataTimeoutMs();
|
|
@@ -818,7 +1318,7 @@ ${instructionText}`,
|
|
|
818
1318
|
"Content-Type": "application/json",
|
|
819
1319
|
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
820
1320
|
},
|
|
821
|
-
body: requestBody(stream)
|
|
1321
|
+
body: requestBody(stream, schema)
|
|
822
1322
|
});
|
|
823
1323
|
} catch (error) {
|
|
824
1324
|
return transportFailure("Judge request failed", controller.signal.reason ?? error);
|
|
@@ -832,12 +1332,20 @@ ${instructionText}`,
|
|
|
832
1332
|
...isRetryableStatus(response.status) ? { retryDelayMs: retryDelayMs(response.headers.get("retry-after")) } : {},
|
|
833
1333
|
// Name the field, not the word: gateways echo request params in
|
|
834
1334
|
// unrelated 400 bodies.
|
|
835
|
-
...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {}
|
|
1335
|
+
...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {},
|
|
1336
|
+
...schema && response.status === 400 && /json_schema|response_format/.test(body) ? { schemaRejected: true } : {}
|
|
836
1337
|
};
|
|
837
1338
|
}
|
|
1339
|
+
const guards = isJudgeLoopGuardEnabled() ? {
|
|
1340
|
+
reasoning: new RepetitionGuard({ diversity: true }),
|
|
1341
|
+
content: new RepetitionGuard({ diversity: false })
|
|
1342
|
+
} : void 0;
|
|
838
1343
|
try {
|
|
839
|
-
return { kind: "completion", completion: await readCompletion(response, onData) };
|
|
1344
|
+
return { kind: "completion", completion: await readCompletion(response, onData, guards) };
|
|
840
1345
|
} catch (error) {
|
|
1346
|
+
if (error instanceof JudgeReasoningLoopError) {
|
|
1347
|
+
return { kind: "unavailable", error, retryable: true, loop: true };
|
|
1348
|
+
}
|
|
841
1349
|
return transportFailure("Judge response failed", controller.signal.reason ?? error);
|
|
842
1350
|
}
|
|
843
1351
|
} finally {
|
|
@@ -864,17 +1372,23 @@ ${instructionText}`,
|
|
|
864
1372
|
const startedAt = Date.now();
|
|
865
1373
|
const deadline = startedAt + judgeMaxDurationMs();
|
|
866
1374
|
let stream = true;
|
|
867
|
-
let
|
|
868
|
-
|
|
869
|
-
|
|
1375
|
+
let schema = true;
|
|
1376
|
+
let result = await attempt(stream, schema, deadline);
|
|
1377
|
+
while (result.kind === "unavailable" && (result.streamRejected || result.schemaRejected)) {
|
|
1378
|
+
if (result.streamRejected) stream = false;
|
|
1379
|
+
if (result.schemaRejected) schema = false;
|
|
1380
|
+
result = await attempt(stream, schema, deadline);
|
|
1381
|
+
}
|
|
1382
|
+
let loops = result.kind === "unavailable" && result.loop ? 1 : 0;
|
|
1383
|
+
const noVerdict = (r) => r.kind === "completion" && (isTruncated(r.completion) || judgeVerdict(r.completion.text) === void 0);
|
|
870
1384
|
const retryable = result.kind === "unavailable" && result.retryable;
|
|
871
|
-
if (
|
|
872
|
-
const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs :
|
|
1385
|
+
if (noVerdict(result) || retryable) {
|
|
1386
|
+
const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs : JUDGE_RETRY_DELAY_MS;
|
|
873
1387
|
if (deadline - Date.now() - delay >= JUDGE_MIN_RETRY_BUDGET_MS) {
|
|
874
|
-
if (streamRejected) stream = false;
|
|
875
1388
|
await new Promise((resolve5) => setTimeout(resolve5, delay));
|
|
876
1389
|
const first = result;
|
|
877
|
-
result = await attempt(stream, deadline);
|
|
1390
|
+
result = await attempt(stream, schema, deadline);
|
|
1391
|
+
if (result.kind === "unavailable" && result.loop) loops++;
|
|
878
1392
|
if (result.kind === "unavailable" && first.kind === "unavailable") {
|
|
879
1393
|
result = {
|
|
880
1394
|
...result,
|
|
@@ -885,17 +1399,25 @@ ${instructionText}`,
|
|
|
885
1399
|
}
|
|
886
1400
|
}
|
|
887
1401
|
}
|
|
888
|
-
|
|
889
|
-
const { text, usage } = result.completion;
|
|
890
|
-
const judge = {
|
|
1402
|
+
const judgeMetadata = async (completion) => ({
|
|
891
1403
|
model: args.model,
|
|
892
1404
|
contract: judgeContractInUse(),
|
|
893
1405
|
prompt: { system: systemPromptText, user: instructionText },
|
|
894
|
-
response: text,
|
|
895
|
-
tokens: parseJudgeUsage(usage),
|
|
1406
|
+
...completion ? { response: completion.text, tokens: parseJudgeUsage(completion.usage) } : {},
|
|
896
1407
|
latency_ms: Date.now() - startedAt,
|
|
897
|
-
secondJudge: await secondJudgePromise
|
|
898
|
-
|
|
1408
|
+
secondJudge: await secondJudgePromise,
|
|
1409
|
+
...loops > 0 ? { reasoning_loops: loops } : {}
|
|
1410
|
+
});
|
|
1411
|
+
if (result.kind === "unavailable" && result.loop) {
|
|
1412
|
+
return {
|
|
1413
|
+
pass: false,
|
|
1414
|
+
reasoning: `Judge reasoning loop \u2014 the judge's stream degenerated into repetition and was cut${loops > 1 ? ", on the retry too" : ""} (${result.error.message}). The verdict is unknown, so this check is recorded as a judge error, not a verdict.`,
|
|
1415
|
+
judge: await judgeMetadata(),
|
|
1416
|
+
unavailable: true
|
|
1417
|
+
};
|
|
1418
|
+
}
|
|
1419
|
+
if (result.kind === "unavailable") throw result.error;
|
|
1420
|
+
const judge = await judgeMetadata(result.completion);
|
|
899
1421
|
if (isTruncated(result.completion)) {
|
|
900
1422
|
return {
|
|
901
1423
|
pass: false,
|
|
@@ -904,12 +1426,16 @@ ${instructionText}`,
|
|
|
904
1426
|
unavailable: true
|
|
905
1427
|
};
|
|
906
1428
|
}
|
|
907
|
-
const
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1429
|
+
const verdict = judgeVerdict(result.completion.text);
|
|
1430
|
+
if (verdict === void 0) {
|
|
1431
|
+
return {
|
|
1432
|
+
pass: false,
|
|
1433
|
+
reasoning: "Judge reply carried no verdict (no boolean `pass`). The verdict is unknown, so this check is recorded as a judge error, not a verdict. The raw reply is on the judge metadata.",
|
|
1434
|
+
judge,
|
|
1435
|
+
unavailable: true
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
return { ...verdict, judge };
|
|
913
1439
|
});
|
|
914
1440
|
}
|
|
915
1441
|
function transportFailure(prefix, error) {
|
|
@@ -925,6 +1451,29 @@ function transportFailure(prefix, error) {
|
|
|
925
1451
|
retryable: !timedOut
|
|
926
1452
|
};
|
|
927
1453
|
}
|
|
1454
|
+
function buildJudgePromptParts(args) {
|
|
1455
|
+
const deliverableText = `Values to evaluate:
|
|
1456
|
+
${formatJudgeValues(args.values)}`;
|
|
1457
|
+
const { briefingText, instructionText } = assembleBriefing(args);
|
|
1458
|
+
const secondJudgeProjected = args.secondJudgeValue !== void 0;
|
|
1459
|
+
const secondJudgeValues = args.secondJudgeValue ?? args.values;
|
|
1460
|
+
const secondJudgeDeliverableText = `Values to evaluate:
|
|
1461
|
+
${formatJudgeValues(secondJudgeValues)}`;
|
|
1462
|
+
return {
|
|
1463
|
+
briefingText,
|
|
1464
|
+
instructionText,
|
|
1465
|
+
systemPromptText: `${briefingText}
|
|
1466
|
+
|
|
1467
|
+
${deliverableText}`,
|
|
1468
|
+
deliverableText,
|
|
1469
|
+
secondJudgeState: `${briefingText}
|
|
1470
|
+
|
|
1471
|
+
${secondJudgeDeliverableText}
|
|
1472
|
+
|
|
1473
|
+
${instructionText}`,
|
|
1474
|
+
secondJudgeProjected
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
928
1477
|
function assembleBriefing(args) {
|
|
929
1478
|
if (!args.prompt) {
|
|
930
1479
|
return {
|
|
@@ -953,6 +1502,55 @@ ${args.instruction}`
|
|
|
953
1502
|
};
|
|
954
1503
|
}
|
|
955
1504
|
|
|
1505
|
+
// src/agent-task/checks/judge-cascade.ts
|
|
1506
|
+
var CASCADE_CONFIDENCE_THRESHOLD = 0.95;
|
|
1507
|
+
function isCascadeActive(effective) {
|
|
1508
|
+
return effective?.mode === "cascade" && resolveSecondJudgeModel() !== void 0;
|
|
1509
|
+
}
|
|
1510
|
+
async function cascadePreflight(args) {
|
|
1511
|
+
const model = resolveSecondJudgeModel();
|
|
1512
|
+
if (model === void 0) return { kind: "off" };
|
|
1513
|
+
const parts = buildJudgePromptParts({
|
|
1514
|
+
values: args.values,
|
|
1515
|
+
instruction: args.instruction,
|
|
1516
|
+
...args.effective.prompt ? { prompt: args.effective.prompt } : {},
|
|
1517
|
+
...args.context ? { context: args.context } : {},
|
|
1518
|
+
...args.secondJudgeValue !== void 0 ? { secondJudgeValue: args.secondJudgeValue } : {}
|
|
1519
|
+
});
|
|
1520
|
+
const evidence = await callSecondJudge({
|
|
1521
|
+
state: parts.secondJudgeState,
|
|
1522
|
+
model,
|
|
1523
|
+
baseURL: resolveSecondJudgeBaseURL(defaultJudgeBaseURL(args.effective.baseURL)),
|
|
1524
|
+
apiKey: resolveSecondJudgeAPIKey(defaultJudgeAPIKey(args.effective.apiKey)),
|
|
1525
|
+
...parts.secondJudgeProjected ? { projected: true } : {}
|
|
1526
|
+
});
|
|
1527
|
+
return { kind: "ran", evidence, parts };
|
|
1528
|
+
}
|
|
1529
|
+
function cascadeDecides(evidence) {
|
|
1530
|
+
return evidence !== void 0 && (evidence.choice === "pass" || evidence.choice === "fail") && typeof evidence.confidence === "number" && evidence.confidence >= CASCADE_CONFIDENCE_THRESHOLD && evidence.error === void 0 && evidence.skipped === void 0 && evidence.projected !== true;
|
|
1531
|
+
}
|
|
1532
|
+
function cascadeVerdict(evidence, parts) {
|
|
1533
|
+
const pass = evidence.choice === "pass";
|
|
1534
|
+
return {
|
|
1535
|
+
pass,
|
|
1536
|
+
reasoning: `Verdict by second judge (cascade): ${evidence.choice} \xB7 confidence ${evidence.confidence}` + (evidence.passProbability !== void 0 ? ` \xB7 p(pass) ${evidence.passProbability}` : "") + `. Primary judge not called.`,
|
|
1537
|
+
judge: {
|
|
1538
|
+
model: evidence.model,
|
|
1539
|
+
prompt: { system: parts.systemPromptText, user: parts.instructionText },
|
|
1540
|
+
response: JSON.stringify({
|
|
1541
|
+
choice: evidence.choice,
|
|
1542
|
+
probabilities: { pass: evidence.passProbability },
|
|
1543
|
+
confidence: evidence.confidence
|
|
1544
|
+
}),
|
|
1545
|
+
...evidence.inputTokens !== void 0 ? { tokens: { input: evidence.inputTokens, output: 0 } } : {},
|
|
1546
|
+
...evidence.latencyMs !== void 0 ? { latency_ms: evidence.latencyMs } : {},
|
|
1547
|
+
...evidence.costUsd !== void 0 ? { cost: evidence.costUsd } : {},
|
|
1548
|
+
secondJudge: evidence,
|
|
1549
|
+
verdict_by: "second-judge"
|
|
1550
|
+
}
|
|
1551
|
+
};
|
|
1552
|
+
}
|
|
1553
|
+
|
|
956
1554
|
// src/agent-task/checks/agent-session.ts
|
|
957
1555
|
import { createHash as createHash3 } from "crypto";
|
|
958
1556
|
|
|
@@ -1362,6 +1960,25 @@ function isAbortError(error) {
|
|
|
1362
1960
|
const name = error instanceof Error ? error.name : "";
|
|
1363
1961
|
return name === "TimeoutError" || name === "AbortError" || name === "ResponseAborted";
|
|
1364
1962
|
}
|
|
1963
|
+
var LEADING_PCRE_FLAGS = /^\(\?([a-z]+)\)/;
|
|
1964
|
+
var PCRE_FLAGS_JS_SUPPORTS = /* @__PURE__ */ new Set(["i", "m", "s"]);
|
|
1965
|
+
function stripLeadingPcreFlags(pattern) {
|
|
1966
|
+
const m = LEADING_PCRE_FLAGS.exec(pattern);
|
|
1967
|
+
if (!m) return { pattern, flags: "gim" };
|
|
1968
|
+
const unsupported = [...m[1]].filter((f) => !PCRE_FLAGS_JS_SUPPORTS.has(f));
|
|
1969
|
+
if (unsupported.length > 0) {
|
|
1970
|
+
throw new Error(`leading flag group (?${m[1]}) \u2014 JS RegExp has no equivalent for ${unsupported.join(", ")}`);
|
|
1971
|
+
}
|
|
1972
|
+
return {
|
|
1973
|
+
pattern: pattern.slice(m[0].length),
|
|
1974
|
+
flags: m[1].includes("s") ? "gims" : "gim"
|
|
1975
|
+
};
|
|
1976
|
+
}
|
|
1977
|
+
var UNSUPPORTED_PCRE_CONSTRUCTS = /\\[AQZz]/;
|
|
1978
|
+
function rejectUnsupportedPcreConstructs(pattern) {
|
|
1979
|
+
const m = UNSUPPORTED_PCRE_CONSTRUCTS.exec(pattern);
|
|
1980
|
+
return m ? `unsupported PCRE construct ${m[0]} \u2014 JS RegExp has no equivalent; anchor with ^ / $ (the search is multiline) or rewrite the pattern` : void 0;
|
|
1981
|
+
}
|
|
1365
1982
|
function aggregateUsage(steps) {
|
|
1366
1983
|
return steps.reduce(
|
|
1367
1984
|
(acc, s) => ({
|
|
@@ -1377,14 +1994,15 @@ function recordedCost(steps) {
|
|
|
1377
1994
|
return known.reduce((a, s) => a + s.tokens?.cost, 0);
|
|
1378
1995
|
}
|
|
1379
1996
|
function extractVerdict(steps) {
|
|
1380
|
-
const call
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1997
|
+
for (const call of steps.flatMap((s) => s.tool_calls ?? [])) {
|
|
1998
|
+
if (call.name !== "finish_verdict" || !call.input) continue;
|
|
1999
|
+
try {
|
|
2000
|
+
const args = JSON.parse(call.input);
|
|
2001
|
+
if (typeof args.reasoning === "string" && args.reasoning.length > 0 && typeof args.pass === "boolean") {
|
|
2002
|
+
return { reasoning: args.reasoning, pass: args.pass };
|
|
2003
|
+
}
|
|
2004
|
+
} catch {
|
|
1386
2005
|
}
|
|
1387
|
-
} catch {
|
|
1388
2006
|
}
|
|
1389
2007
|
return void 0;
|
|
1390
2008
|
}
|
|
@@ -1401,7 +2019,15 @@ function buildEvidenceTools(args) {
|
|
|
1401
2019
|
});
|
|
1402
2020
|
return ledger.readBytesUsed > budget.maxReadBytes ? overflow() : null;
|
|
1403
2021
|
};
|
|
1404
|
-
const
|
|
2022
|
+
const dedupeCache = /* @__PURE__ */ new Map();
|
|
2023
|
+
const span = (name, input, fn) => {
|
|
2024
|
+
const key = `${stepIndexOf()}|${name}|${JSON.stringify(input ?? {})}`;
|
|
2025
|
+
const cached = dedupeCache.get(key);
|
|
2026
|
+
if (cached) return cached;
|
|
2027
|
+
const execution = tracer ? tracer.traceTool(name, input, fn) : fn();
|
|
2028
|
+
dedupeCache.set(key, execution);
|
|
2029
|
+
return execution;
|
|
2030
|
+
};
|
|
1405
2031
|
const tools = {
|
|
1406
2032
|
read_deliverable: tool({
|
|
1407
2033
|
description: `Read one of this run's deliverables by name, with optional offset/limit for pagination (max ${MAX_READ_WINDOW} bytes per call).`,
|
|
@@ -1432,7 +2058,7 @@ function buildEvidenceTools(args) {
|
|
|
1432
2058
|
}
|
|
1433
2059
|
}),
|
|
1434
2060
|
search_deliverable: tool({
|
|
1435
|
-
description: "Regex-search one of this run's deliverables; returns up to 8 matches with surrounding context. Cheaper than reading a large deliverable end to end \u2014 use it to locate the relevant part first.",
|
|
2061
|
+
description: "Regex-search one of this run's deliverables (case-insensitive; ^ and $ match line starts/ends); returns up to 8 matches with surrounding context. Cheaper than reading a large deliverable end to end \u2014 use it to locate the relevant part first.",
|
|
1436
2062
|
inputSchema: z.object({ name: z.string(), pattern: z.string() }),
|
|
1437
2063
|
execute: async (input) => {
|
|
1438
2064
|
const { name, pattern } = input;
|
|
@@ -1445,12 +2071,22 @@ function buildEvidenceTools(args) {
|
|
|
1445
2071
|
if (pattern.length > SEARCH_MAX_PATTERN) {
|
|
1446
2072
|
return { error: `pattern too long (max ${SEARCH_MAX_PATTERN} chars)` };
|
|
1447
2073
|
}
|
|
1448
|
-
|
|
2074
|
+
const unsupported = rejectUnsupportedPcreConstructs(pattern);
|
|
2075
|
+
if (unsupported) return { error: unsupported };
|
|
2076
|
+
let translated;
|
|
1449
2077
|
try {
|
|
1450
|
-
|
|
2078
|
+
translated = stripLeadingPcreFlags(pattern);
|
|
1451
2079
|
} catch (error) {
|
|
1452
2080
|
return { error: `invalid regex: ${error instanceof Error ? error.message : String(error)}` };
|
|
1453
2081
|
}
|
|
2082
|
+
let re;
|
|
2083
|
+
try {
|
|
2084
|
+
re = new RegExp(translated.pattern, translated.flags);
|
|
2085
|
+
} catch (error) {
|
|
2086
|
+
return {
|
|
2087
|
+
error: `invalid regex: ${error instanceof Error ? error.message : String(error)} \u2014 patterns compile as case-insensitive multiline; leading PCRE flags like (?i) are stripped automatically`
|
|
2088
|
+
};
|
|
2089
|
+
}
|
|
1454
2090
|
const content = renderValue2(evidence.deliverables[name]).slice(0, SEARCH_SCAN_LIMIT);
|
|
1455
2091
|
const hits = [];
|
|
1456
2092
|
let m;
|
|
@@ -1846,7 +2482,12 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer,
|
|
|
1846
2482
|
const detail = result.outcome === "error" ? `agentic judge session errored before verdicting: ${result.error ?? "unknown error"}` : `agentic judge session ended without a verdict (budget exhausted after ${result.session.usage?.steps ?? "?"} steps; last tool: ${result.session.steps?.at(-1)?.tool_calls?.map((c) => c.name).join(", ") ?? "none"})`;
|
|
1847
2483
|
const agreed = opts?.expect === void 0 ? result.verdict?.pass ?? false : result.verdict ? result.verdict.pass === (opts.expect === "pass") : false;
|
|
1848
2484
|
const reasoning = opts?.expect === void 0 || agreed || !result.verdict ? result.verdict?.reasoning ?? detail : `judge ${result.verdict.pass ? "PASSed" : "FAILed"} where ground truth is ${opts.expect.toUpperCase()} \u2014 judge reasoning: ${result.verdict.reasoning}`;
|
|
1849
|
-
|
|
2485
|
+
const consultedEvidence = (result.session.evidence?.length ?? 0) > 0 || exhibits.length > 0 || (result.session.steps?.some(
|
|
2486
|
+
(s) => s.tool_calls?.some((c) => c.name === "get_trace" || c.name === "list_runs" || c.name === "get_run")
|
|
2487
|
+
) ?? false);
|
|
2488
|
+
const readNothing = result.verdict !== void 0 && !consultedEvidence && evidence !== void 0 && Object.keys(evidence.deliverables).length > 0;
|
|
2489
|
+
const recorded = readNothing ? `${reasoning} \u2014 [vacuous verdict: the session consulted no run evidence before deciding]` : reasoning;
|
|
2490
|
+
rec.record(label, agreed, recorded, {
|
|
1850
2491
|
evaluator_type: "agent",
|
|
1851
2492
|
judge,
|
|
1852
2493
|
expected: instruction,
|
|
@@ -1873,7 +2514,8 @@ function resolveJudgeConfig(judgeConfig, override) {
|
|
|
1873
2514
|
model,
|
|
1874
2515
|
baseURL: override?.baseURL ?? judgeConfig?.baseURL,
|
|
1875
2516
|
apiKey: override?.apiKey ?? judgeConfig?.apiKey,
|
|
1876
|
-
prompt: override?.prompt ?? judgeConfig?.prompt
|
|
2517
|
+
prompt: override?.prompt ?? judgeConfig?.prompt,
|
|
2518
|
+
mode: override?.mode ?? judgeConfig?.mode
|
|
1877
2519
|
};
|
|
1878
2520
|
}
|
|
1879
2521
|
var TEST_METHOD_NAMES = [
|
|
@@ -1890,6 +2532,8 @@ var TEST_METHOD_NAMES = [
|
|
|
1890
2532
|
"maxDurationMs",
|
|
1891
2533
|
"maxTokens",
|
|
1892
2534
|
"minTokens",
|
|
2535
|
+
"steerDelivered",
|
|
2536
|
+
"afterSteer",
|
|
1893
2537
|
"assert",
|
|
1894
2538
|
"check",
|
|
1895
2539
|
"judge",
|
|
@@ -1912,6 +2556,58 @@ function trackPending(rec, method) {
|
|
|
1912
2556
|
return promise;
|
|
1913
2557
|
});
|
|
1914
2558
|
}
|
|
2559
|
+
function createSteerAssertions(view, rec, makeNested) {
|
|
2560
|
+
const findSteer = (n, id) => {
|
|
2561
|
+
const steers = view.steers;
|
|
2562
|
+
if (steers.length === 0) {
|
|
2563
|
+
rec.record(
|
|
2564
|
+
id,
|
|
2565
|
+
false,
|
|
2566
|
+
"steering evidence unavailable \u2014 no task.steer events in this trace",
|
|
2567
|
+
{ expected: `steer ${n} delivered and consumed`, received: "no steering evidence" }
|
|
2568
|
+
);
|
|
2569
|
+
return void 0;
|
|
2570
|
+
}
|
|
2571
|
+
const steer2 = steers.find((s) => s.number === n);
|
|
2572
|
+
if (!steer2) {
|
|
2573
|
+
rec.record(
|
|
2574
|
+
id,
|
|
2575
|
+
false,
|
|
2576
|
+
`no steer number ${n} in this trace (found: ${steers.map((s) => s.number).join(", ")})`,
|
|
2577
|
+
{ expected: `steer ${n} exists`, received: steers.map((s) => s.number).join(", ") || "none" }
|
|
2578
|
+
);
|
|
2579
|
+
}
|
|
2580
|
+
return steer2;
|
|
2581
|
+
};
|
|
2582
|
+
return {
|
|
2583
|
+
steerDelivered(n) {
|
|
2584
|
+
const steer2 = findSteer(n, `steerDelivered(${n})`);
|
|
2585
|
+
if (!steer2) return;
|
|
2586
|
+
if (steer2.status !== "delivered") {
|
|
2587
|
+
const why = steer2.reason ? ` \u2014 ${steer2.reason}` : "";
|
|
2588
|
+
rec.record(`steerDelivered(${n})`, false, `steer ${n} was ${steer2.status}${why}`, {
|
|
2589
|
+
expected: "delivered",
|
|
2590
|
+
received: steer2.status
|
|
2591
|
+
});
|
|
2592
|
+
return;
|
|
2593
|
+
}
|
|
2594
|
+
const consumed = view.snapshot.observations.some(
|
|
2595
|
+
(o) => o.type === "GENERATION" && steer2.spanStartedAt != null && o.startedAt != null && (o.startedAt > steer2.spanStartedAt || o.startedAt === steer2.spanStartedAt && o.spanId > steer2.spanId)
|
|
2596
|
+
);
|
|
2597
|
+
rec.record(
|
|
2598
|
+
`steerDelivered(${n})`,
|
|
2599
|
+
consumed,
|
|
2600
|
+
consumed ? "" : `steer ${n} was delivered but no generation observation started after it \u2014 the harness queued the message, no model call demonstrably consumed it`,
|
|
2601
|
+
{ expected: `\u22651 generation after steer ${n}`, received: consumed ? "1+" : "0" }
|
|
2602
|
+
);
|
|
2603
|
+
},
|
|
2604
|
+
afterSteer(n, fn) {
|
|
2605
|
+
const steer2 = findSteer(n, `afterSteer(${n})`);
|
|
2606
|
+
if (!steer2) return;
|
|
2607
|
+
fn(makeNested(view.windowAfterSteer(n)));
|
|
2608
|
+
}
|
|
2609
|
+
};
|
|
2610
|
+
}
|
|
1915
2611
|
function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
1916
2612
|
return async (values, instruction, opts) => {
|
|
1917
2613
|
const label = opts?.label ?? "judge";
|
|
@@ -1931,6 +2627,52 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
1931
2627
|
const context2 = judgeScopeToContext(judgeScope);
|
|
1932
2628
|
const judgeServing = servingHostFromBaseURL(effective.baseURL);
|
|
1933
2629
|
const secondJudgeValue = opts?.secondJudgeValue;
|
|
2630
|
+
let prefetchedSecondJudge;
|
|
2631
|
+
if (isCascadeActive(effective)) {
|
|
2632
|
+
let preflight;
|
|
2633
|
+
try {
|
|
2634
|
+
preflight = await cascadePreflight({
|
|
2635
|
+
values: valueArray,
|
|
2636
|
+
instruction,
|
|
2637
|
+
effective,
|
|
2638
|
+
...context2 ? { context: context2 } : {},
|
|
2639
|
+
...secondJudgeValue !== void 0 ? {
|
|
2640
|
+
secondJudgeValue: Array.isArray(secondJudgeValue) ? secondJudgeValue : [secondJudgeValue]
|
|
2641
|
+
} : {}
|
|
2642
|
+
});
|
|
2643
|
+
} catch (error) {
|
|
2644
|
+
rec.record(
|
|
2645
|
+
label,
|
|
2646
|
+
false,
|
|
2647
|
+
`judge failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2648
|
+
{
|
|
2649
|
+
evaluator_type: "llm",
|
|
2650
|
+
expected: instruction,
|
|
2651
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
2652
|
+
location,
|
|
2653
|
+
outcome: "error"
|
|
2654
|
+
}
|
|
2655
|
+
);
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
if (preflight.kind === "ran" && cascadeDecides(preflight.evidence)) {
|
|
2659
|
+
const { pass, reasoning, judge } = cascadeVerdict(
|
|
2660
|
+
preflight.evidence,
|
|
2661
|
+
preflight.parts
|
|
2662
|
+
);
|
|
2663
|
+
const agreed = opts?.expect === void 0 ? pass : pass === (opts.expect === "pass");
|
|
2664
|
+
const recordedReasoning = opts?.expect === void 0 || agreed ? reasoning : `second judge ${pass ? "PASSed" : "FAILed"} where ground truth is ${opts.expect.toUpperCase()} \u2014 ${reasoning}`;
|
|
2665
|
+
rec.record(label, agreed, recordedReasoning, {
|
|
2666
|
+
evaluator_type: "llm",
|
|
2667
|
+
judge,
|
|
2668
|
+
expected: instruction,
|
|
2669
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
2670
|
+
location
|
|
2671
|
+
});
|
|
2672
|
+
return;
|
|
2673
|
+
}
|
|
2674
|
+
if (preflight.kind === "ran") prefetchedSecondJudge = preflight.evidence;
|
|
2675
|
+
}
|
|
1934
2676
|
const call = () => callJudge({
|
|
1935
2677
|
values: valueArray,
|
|
1936
2678
|
instruction,
|
|
@@ -1939,6 +2681,7 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
1939
2681
|
apiKey: effective.apiKey,
|
|
1940
2682
|
prompt: effective.prompt,
|
|
1941
2683
|
...context2 ? { context: context2 } : {},
|
|
2684
|
+
...prefetchedSecondJudge ? { prefetchedSecondJudge } : {},
|
|
1942
2685
|
...secondJudgeValue !== void 0 ? {
|
|
1943
2686
|
secondJudgeValue: Array.isArray(secondJudgeValue) ? secondJudgeValue : [secondJudgeValue]
|
|
1944
2687
|
} : {}
|
|
@@ -2211,6 +2954,19 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
|
|
|
2211
2954
|
received: pass ? "true" : "false"
|
|
2212
2955
|
});
|
|
2213
2956
|
},
|
|
2957
|
+
...createSteerAssertions(
|
|
2958
|
+
view,
|
|
2959
|
+
rec,
|
|
2960
|
+
(window) => createTraceTestContext(
|
|
2961
|
+
window,
|
|
2962
|
+
rec,
|
|
2963
|
+
judgeConfig,
|
|
2964
|
+
judgeScope,
|
|
2965
|
+
agentEvidence,
|
|
2966
|
+
judgeTracer,
|
|
2967
|
+
judgeTools
|
|
2968
|
+
)
|
|
2969
|
+
),
|
|
2214
2970
|
check(value, matcher, label) {
|
|
2215
2971
|
const pass = matcher.test(value);
|
|
2216
2972
|
rec.record(
|
|
@@ -2887,6 +3643,29 @@ function resolveTurn(adapterTurn, taskTurn) {
|
|
|
2887
3643
|
return resolved;
|
|
2888
3644
|
}
|
|
2889
3645
|
|
|
3646
|
+
// src/agent-task/steer.ts
|
|
3647
|
+
var STEERS_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/task-steers");
|
|
3648
|
+
function steer(spec) {
|
|
3649
|
+
if (spec.turn !== void 0 && (!Number.isInteger(spec.turn) || spec.turn < 1)) {
|
|
3650
|
+
throw new Error(`steer: turn must be a positive integer, got ${spec.turn}`);
|
|
3651
|
+
}
|
|
3652
|
+
const count = spec.when === "runStart" ? void 0 : "toolResults" in spec.when ? ["toolResults", spec.when.toolResults] : ["assistantReply", spec.when.assistantReply];
|
|
3653
|
+
if (count !== void 0 && (!Number.isInteger(count[1]) || count[1] < 1)) {
|
|
3654
|
+
throw new Error(`steer: ${count[0]} must be a positive integer, got ${count[1]}`);
|
|
3655
|
+
}
|
|
3656
|
+
const registry2 = globalThis[STEERS_KEY];
|
|
3657
|
+
const list = Array.isArray(registry2) ? registry2 : [];
|
|
3658
|
+
list.push(spec);
|
|
3659
|
+
globalThis[STEERS_KEY] = list;
|
|
3660
|
+
}
|
|
3661
|
+
function getTaskSteers() {
|
|
3662
|
+
const registry2 = globalThis[STEERS_KEY];
|
|
3663
|
+
return Array.isArray(registry2) ? [...registry2] : [];
|
|
3664
|
+
}
|
|
3665
|
+
function resetTaskSteers() {
|
|
3666
|
+
delete globalThis[STEERS_KEY];
|
|
3667
|
+
}
|
|
3668
|
+
|
|
2890
3669
|
// src/agent-task/task/loadTask.ts
|
|
2891
3670
|
async function loadTask(taskDir) {
|
|
2892
3671
|
const absoluteDir = resolve2(taskDir);
|
|
@@ -2959,6 +3738,7 @@ async function loadTaskDefinition(taskFilePath) {
|
|
|
2959
3738
|
resetTaskRegistry();
|
|
2960
3739
|
resetFlowChecks();
|
|
2961
3740
|
resetTaskTurn();
|
|
3741
|
+
resetTaskSteers();
|
|
2962
3742
|
const loaded = await import(moduleUrl);
|
|
2963
3743
|
const registeredTask = getRegisteredTask();
|
|
2964
3744
|
const task2 = registeredTask ?? loaded.default;
|
|
@@ -3822,12 +4602,13 @@ function sleep(ms) {
|
|
|
3822
4602
|
|
|
3823
4603
|
// src/agent-task/checks/agent-history.ts
|
|
3824
4604
|
function createBackendReaderFromEnv(env = process.env) {
|
|
4605
|
+
const apiKey = env.APO_API_KEY;
|
|
3825
4606
|
const token = env.APO_AUTH_TOKEN;
|
|
3826
4607
|
const publicKey = env.APO_PUBLIC_KEY;
|
|
3827
4608
|
const secretKey = env.APO_SECRET_KEY;
|
|
3828
|
-
const headers = token ? { Authorization: `Bearer ${token}` } : publicKey && secretKey ? { Authorization: `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString("base64")}` } : void 0;
|
|
4609
|
+
const headers = apiKey ? { Authorization: `Bearer ${apiKey}` } : token ? { Authorization: `Bearer ${token}` } : publicKey && secretKey ? { Authorization: `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString("base64")}` } : void 0;
|
|
3829
4610
|
if (!headers) return void 0;
|
|
3830
|
-
const endpoint = env.APO_BACKEND_URL ??
|
|
4611
|
+
const endpoint = env.APO_BACKEND_URL ?? "http://127.0.0.1:8000";
|
|
3831
4612
|
return {
|
|
3832
4613
|
async get(path) {
|
|
3833
4614
|
const res = await fetch(`${endpoint.replace(/\/$/, "")}/v1${path}`, { headers });
|
|
@@ -3841,10 +4622,23 @@ function createBackendReaderFromEnv(env = process.env) {
|
|
|
3841
4622
|
function freezeHistoryPlane(args) {
|
|
3842
4623
|
const { reader, taskId, selfRunId } = args;
|
|
3843
4624
|
const detailCache = /* @__PURE__ */ new Map();
|
|
4625
|
+
const listByTaskId = async (id) => await reader.get(
|
|
4626
|
+
`/agent-task-runs?task_id=${encodeURIComponent(id)}&limit=50`
|
|
4627
|
+
);
|
|
3844
4628
|
const loadRuns = (async () => {
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
4629
|
+
let raw = await listByTaskId(taskId);
|
|
4630
|
+
if ((raw ?? []).length === 0 && selfRunId) {
|
|
4631
|
+
try {
|
|
4632
|
+
const self = await reader.get(
|
|
4633
|
+
`/agent-task-runs/${encodeURIComponent(selfRunId)}`
|
|
4634
|
+
);
|
|
4635
|
+
const canonical = self?.task_id;
|
|
4636
|
+
if (typeof canonical === "string" && canonical !== taskId) {
|
|
4637
|
+
raw = await listByTaskId(canonical);
|
|
4638
|
+
}
|
|
4639
|
+
} catch {
|
|
4640
|
+
}
|
|
4641
|
+
}
|
|
3848
4642
|
return (raw ?? []).map(
|
|
3849
4643
|
(r) => ({
|
|
3850
4644
|
id: String(r.id),
|
|
@@ -3908,7 +4702,9 @@ async function freezeHistoryPlaneFromEnv(taskId, env = process.env) {
|
|
|
3908
4702
|
taskId,
|
|
3909
4703
|
...env.AGENT_TASK_RUN_ID ? { selfRunId: env.AGENT_TASK_RUN_ID } : {}
|
|
3910
4704
|
});
|
|
3911
|
-
} catch {
|
|
4705
|
+
} catch (error) {
|
|
4706
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
4707
|
+
console.warn(`apo: judge history plane unavailable \u2014 ${reason}`);
|
|
3912
4708
|
return void 0;
|
|
3913
4709
|
}
|
|
3914
4710
|
}
|
|
@@ -5167,6 +5963,7 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5167
5963
|
const { task: task2, adapter, files, checksPath, inlineChecks } = loaded;
|
|
5168
5964
|
if (!inlineChecks && checksPath) {
|
|
5169
5965
|
resetTaskTurn();
|
|
5966
|
+
resetTaskSteers();
|
|
5170
5967
|
resetFlowChecks();
|
|
5171
5968
|
await loadChecksModule(checksPath);
|
|
5172
5969
|
}
|
|
@@ -5174,14 +5971,32 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5174
5971
|
resetTaskTurn();
|
|
5175
5972
|
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
5176
5973
|
if (!turnFn) return [];
|
|
5974
|
+
const steerSpecs = getTaskSteers();
|
|
5975
|
+
resetTaskSteers();
|
|
5976
|
+
if (steerSpecs.length > 0 && typeof session.steer !== "function") {
|
|
5977
|
+
throw new AgentTaskRunError(
|
|
5978
|
+
`Task "${task2.id}" registers ${steerSpecs.length} steer(s) but adapter "${adapter.name}" does not implement session.steer(). Implement steer() on the adapter session, or remove the steer() registrations.`,
|
|
5979
|
+
{}
|
|
5980
|
+
);
|
|
5981
|
+
}
|
|
5982
|
+
const turnSpanIds = /* @__PURE__ */ new Map();
|
|
5983
|
+
const scheduler = createSteerScheduler({
|
|
5984
|
+
specs: steerSpecs,
|
|
5985
|
+
session,
|
|
5986
|
+
trace: trace2,
|
|
5987
|
+
turnSpanId: (turnNumber) => turnSpanIds.get(turnNumber),
|
|
5988
|
+
onSteer: options?.onSteer
|
|
5989
|
+
});
|
|
5177
5990
|
const taskFiles = new TaskFiles(files);
|
|
5178
5991
|
const turnTranscript = [];
|
|
5179
5992
|
const transcriptTurns = [];
|
|
5180
5993
|
let lastTurnResponse;
|
|
5994
|
+
let lastTurnNumber = 0;
|
|
5181
5995
|
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
5182
5996
|
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
5183
5997
|
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
5184
5998
|
if (userTurn === null || userTurn === void 0) break;
|
|
5999
|
+
await scheduler.onTurnStart(turnNum);
|
|
5185
6000
|
const result = await trace2.step(
|
|
5186
6001
|
{
|
|
5187
6002
|
step_name: "task.turn",
|
|
@@ -5192,15 +6007,29 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5192
6007
|
// processor captures parent under the turn span, not the run root —
|
|
5193
6008
|
// turn-scoped assertions (t.maxTokens({ turn })) walk that chain.
|
|
5194
6009
|
async (spanId) => {
|
|
6010
|
+
turnSpanIds.set(turnNum, spanId);
|
|
5195
6011
|
const run = getActiveApoRun();
|
|
5196
6012
|
const send = () => session.sendUserTurn(userTurn, {
|
|
5197
6013
|
trace: trace2,
|
|
5198
6014
|
turnNumber: turnNum,
|
|
5199
|
-
parentSpanId: spanId
|
|
6015
|
+
parentSpanId: spanId,
|
|
6016
|
+
// Progress events drive the steer scheduler. Fire-and-forget:
|
|
6017
|
+
// the adapter never waits on steering, and scheduler errors are
|
|
6018
|
+
// contained (they must not break the agent's turn).
|
|
6019
|
+
notifyAgentEvent: (event) => {
|
|
6020
|
+
void scheduler.onProgressEvent(turnNum, event).catch((error) => {
|
|
6021
|
+
console.error(
|
|
6022
|
+
"[AgentTask] Steer scheduler error:",
|
|
6023
|
+
error instanceof Error ? error.message : String(error)
|
|
6024
|
+
);
|
|
6025
|
+
});
|
|
6026
|
+
}
|
|
5200
6027
|
});
|
|
5201
6028
|
return run ? withApoRun({ ...run, parentSpanId: spanId, turnNumber: turnNum }, send) : send();
|
|
5202
6029
|
}
|
|
5203
6030
|
);
|
|
6031
|
+
await scheduler.onTurnEnd(turnNum);
|
|
6032
|
+
const turnSteers = scheduler.recordsByTurn().get(turnNum);
|
|
5204
6033
|
turnTranscript.push({
|
|
5205
6034
|
turnNumber: turnNum,
|
|
5206
6035
|
input: userTurn,
|
|
@@ -5209,11 +6038,14 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5209
6038
|
transcriptTurns.push({
|
|
5210
6039
|
turnNumber: turnNum,
|
|
5211
6040
|
userAction: userTurn,
|
|
5212
|
-
agentResponse: result.response
|
|
6041
|
+
agentResponse: result.response,
|
|
6042
|
+
...turnSteers !== void 0 && turnSteers.length > 0 ? { steers: turnSteers } : {}
|
|
5213
6043
|
});
|
|
5214
6044
|
options?.onTurn?.(turnNum, userTurn, result.response);
|
|
5215
6045
|
lastTurnResponse = result.response;
|
|
6046
|
+
lastTurnNumber = turnNum;
|
|
5216
6047
|
}
|
|
6048
|
+
await scheduler.onRunEnd(lastTurnNumber);
|
|
5217
6049
|
if (lastTurnResponse !== void 0) {
|
|
5218
6050
|
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
5219
6051
|
}
|
|
@@ -5741,6 +6573,9 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
5741
6573
|
params.reasoning_tokens
|
|
5742
6574
|
);
|
|
5743
6575
|
}
|
|
6576
|
+
if (params.served_model) {
|
|
6577
|
+
active.span.setAttribute("gen_ai.response.model", params.served_model);
|
|
6578
|
+
}
|
|
5744
6579
|
if (params.status_message) {
|
|
5745
6580
|
active.span.setAttribute("apo.status_message", params.status_message);
|
|
5746
6581
|
}
|
|
@@ -6079,7 +6914,12 @@ function resolveJudgeFromEnv() {
|
|
|
6079
6914
|
return {
|
|
6080
6915
|
model,
|
|
6081
6916
|
baseURL: openRouterModel ? process.env.OPENROUTER_BASE_URL : process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1",
|
|
6082
|
-
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : process.env.OPENAI_API_KEY
|
|
6917
|
+
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : process.env.OPENAI_API_KEY,
|
|
6918
|
+
// "cascade" turns on second-judge-first judging (a confident decision
|
|
6919
|
+
// model verdict stands without calling the primary judge); any other
|
|
6920
|
+
// value is ignored so a typo degrades to the default dual mode, never
|
|
6921
|
+
// an error.
|
|
6922
|
+
...process.env.APO_JUDGE_MODE?.trim() === "cascade" ? { mode: "cascade" } : {}
|
|
6083
6923
|
};
|
|
6084
6924
|
}
|
|
6085
6925
|
async function runTaskDir(taskDir, options) {
|
|
@@ -6353,6 +7193,7 @@ export {
|
|
|
6353
7193
|
ApoSpanProcessor,
|
|
6354
7194
|
FlowView,
|
|
6355
7195
|
JUDGE_SEGMENT_LIMIT,
|
|
7196
|
+
NO_CHECKS_REGISTERED_MESSAGE,
|
|
6356
7197
|
RECEIVED_VALUE_LIMIT,
|
|
6357
7198
|
RejudgeError,
|
|
6358
7199
|
TEST_METHOD_NAMES,
|
|
@@ -6381,6 +7222,7 @@ export {
|
|
|
6381
7222
|
fromAnthropicMessages,
|
|
6382
7223
|
fromOpenAIMessages,
|
|
6383
7224
|
getActiveApoRun,
|
|
7225
|
+
getTaskSteers,
|
|
6384
7226
|
getTaskTurn,
|
|
6385
7227
|
includes,
|
|
6386
7228
|
isFileArtifact,
|
|
@@ -6399,6 +7241,7 @@ export {
|
|
|
6399
7241
|
resetApoTracing,
|
|
6400
7242
|
resetFlowChecks,
|
|
6401
7243
|
resetTaskRegistry,
|
|
7244
|
+
resetTaskSteers,
|
|
6402
7245
|
resetTaskTurn,
|
|
6403
7246
|
resolveMcpServerPaths,
|
|
6404
7247
|
resolveOtlpTracesUrl,
|
|
@@ -6407,8 +7250,11 @@ export {
|
|
|
6407
7250
|
runTask,
|
|
6408
7251
|
runTaskDir,
|
|
6409
7252
|
satisfies,
|
|
7253
|
+
secondJudgeFacts,
|
|
7254
|
+
secondJudgeTakeaway,
|
|
6410
7255
|
sha256Hex2 as sha256Hex,
|
|
6411
7256
|
similarity,
|
|
7257
|
+
steer,
|
|
6412
7258
|
task,
|
|
6413
7259
|
defineCheck as test,
|
|
6414
7260
|
transcriptSessionToOtlp,
|