@apo-ai/sdk 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-task/public.d.ts +273 -2
- package/dist/agent-task/public.js +693 -58
- package/dist/agent-task/public.js.map +1 -1
- package/package.json +1 -1
|
@@ -24,13 +24,225 @@ import {
|
|
|
24
24
|
servingHostFromBaseURL
|
|
25
25
|
} from "../chunk-Z64A566W.js";
|
|
26
26
|
|
|
27
|
+
// src/agent-task/run/steer-scheduler.ts
|
|
28
|
+
var TASK_STEER_SPAN_NAME = "task.steer";
|
|
29
|
+
function createSteerScheduler(options) {
|
|
30
|
+
const { specs, session, trace: trace2 } = options;
|
|
31
|
+
const states = specs.map((spec, index) => ({
|
|
32
|
+
spec,
|
|
33
|
+
fired: false,
|
|
34
|
+
record: {
|
|
35
|
+
number: index + 1,
|
|
36
|
+
turn: spec.turn ?? 1,
|
|
37
|
+
...spec.label !== void 0 ? { label: spec.label } : {},
|
|
38
|
+
trigger: spec.when,
|
|
39
|
+
message: spec.message,
|
|
40
|
+
status: "undelivered"
|
|
41
|
+
}
|
|
42
|
+
}));
|
|
43
|
+
const toolResults = /* @__PURE__ */ new Map();
|
|
44
|
+
const assistantReplies = /* @__PURE__ */ new Map();
|
|
45
|
+
const activeTurns = /* @__PURE__ */ new Set();
|
|
46
|
+
const runStartSeen = /* @__PURE__ */ new Set();
|
|
47
|
+
const progressSeen = /* @__PURE__ */ new Set();
|
|
48
|
+
let chain = Promise.resolve();
|
|
49
|
+
const enqueue = (step) => {
|
|
50
|
+
const next = chain.then(step, step);
|
|
51
|
+
chain = next.catch(() => {
|
|
52
|
+
});
|
|
53
|
+
return next;
|
|
54
|
+
};
|
|
55
|
+
function emitSteerEvent(record) {
|
|
56
|
+
try {
|
|
57
|
+
trace2.recordEvent({
|
|
58
|
+
step_name: TASK_STEER_SPAN_NAME,
|
|
59
|
+
observation_type: "CHAIN",
|
|
60
|
+
parent_call_id: options.turnSpanId?.(record.turn),
|
|
61
|
+
metadata: {
|
|
62
|
+
steerNumber: record.number,
|
|
63
|
+
turn: record.turn,
|
|
64
|
+
...record.label !== void 0 ? { label: record.label } : {},
|
|
65
|
+
trigger: JSON.stringify(record.trigger),
|
|
66
|
+
status: record.status,
|
|
67
|
+
...record.boundary !== void 0 ? { boundary: record.boundary } : {},
|
|
68
|
+
...record.reason !== void 0 ? { reason: record.reason } : {}
|
|
69
|
+
},
|
|
70
|
+
output: { ...record }
|
|
71
|
+
});
|
|
72
|
+
} catch {
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function notifyHost(record) {
|
|
76
|
+
try {
|
|
77
|
+
options.onSteer?.(record);
|
|
78
|
+
} catch (error) {
|
|
79
|
+
console.error(
|
|
80
|
+
"[AgentTask] onSteer callback failed:",
|
|
81
|
+
error instanceof Error ? error.message : String(error)
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function finalize(state, reason) {
|
|
86
|
+
state.record.status = "undelivered";
|
|
87
|
+
state.record.reason = reason;
|
|
88
|
+
emitSteerEvent(state.record);
|
|
89
|
+
notifyHost(state.record);
|
|
90
|
+
}
|
|
91
|
+
async function deliver(state, turnNumber) {
|
|
92
|
+
state.fired = true;
|
|
93
|
+
try {
|
|
94
|
+
const result = await session.steer?.(state.spec.message, {
|
|
95
|
+
trace: trace2,
|
|
96
|
+
turnNumber,
|
|
97
|
+
steerNumber: state.record.number
|
|
98
|
+
});
|
|
99
|
+
state.record.status = "delivered";
|
|
100
|
+
state.record.deliveredAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
101
|
+
state.record.boundary = result?.boundary ?? "tool_results";
|
|
102
|
+
} catch (error) {
|
|
103
|
+
state.record.status = "error";
|
|
104
|
+
state.record.reason = error instanceof Error ? error.message : String(error);
|
|
105
|
+
}
|
|
106
|
+
emitSteerEvent(state.record);
|
|
107
|
+
notifyHost(state.record);
|
|
108
|
+
}
|
|
109
|
+
function triggerSatisfied(trigger, turnNumber) {
|
|
110
|
+
if (trigger === "runStart") return runStartSeen.has(turnNumber);
|
|
111
|
+
if ("toolResults" in trigger) {
|
|
112
|
+
return (toolResults.get(turnNumber) ?? 0) >= trigger.toolResults;
|
|
113
|
+
}
|
|
114
|
+
return (assistantReplies.get(turnNumber) ?? 0) >= trigger.assistantReply;
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
onTurnStart(turnNumber) {
|
|
118
|
+
return enqueue(async () => {
|
|
119
|
+
toolResults.set(turnNumber, 0);
|
|
120
|
+
assistantReplies.set(turnNumber, 0);
|
|
121
|
+
activeTurns.add(turnNumber);
|
|
122
|
+
});
|
|
123
|
+
},
|
|
124
|
+
onProgressEvent(turnNumber, event) {
|
|
125
|
+
return enqueue(async () => {
|
|
126
|
+
if (!activeTurns.has(turnNumber)) return;
|
|
127
|
+
progressSeen.add(turnNumber);
|
|
128
|
+
if (event.kind === "run_start") {
|
|
129
|
+
runStartSeen.add(turnNumber);
|
|
130
|
+
} else if (event.kind === "tool_result") {
|
|
131
|
+
toolResults.set(turnNumber, (toolResults.get(turnNumber) ?? 0) + 1);
|
|
132
|
+
} else if (event.kind === "assistant_reply") {
|
|
133
|
+
assistantReplies.set(turnNumber, (assistantReplies.get(turnNumber) ?? 0) + 1);
|
|
134
|
+
}
|
|
135
|
+
for (const state of states) {
|
|
136
|
+
if (state.fired || state.record.turn !== turnNumber) continue;
|
|
137
|
+
if (triggerSatisfied(state.spec.when, turnNumber)) {
|
|
138
|
+
await deliver(state, turnNumber);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
},
|
|
143
|
+
onTurnEnd(turnNumber) {
|
|
144
|
+
return enqueue(async () => {
|
|
145
|
+
activeTurns.delete(turnNumber);
|
|
146
|
+
const reason = progressSeen.has(turnNumber) ? "turn ended before trigger fired" : "no progress events observed for this turn (adapter did not call notifyAgentEvent)";
|
|
147
|
+
for (const state of states) {
|
|
148
|
+
if (state.fired || state.record.turn !== turnNumber) continue;
|
|
149
|
+
finalize(state, reason);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
},
|
|
153
|
+
onRunEnd(maxTurnReached) {
|
|
154
|
+
return enqueue(async () => {
|
|
155
|
+
for (const state of states) {
|
|
156
|
+
if (state.fired || state.record.status !== "undelivered") continue;
|
|
157
|
+
if (state.record.turn > maxTurnReached) {
|
|
158
|
+
finalize(state, "target turn never ran");
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
},
|
|
163
|
+
recordsByTurn() {
|
|
164
|
+
const byTurn = /* @__PURE__ */ new Map();
|
|
165
|
+
for (const state of states) {
|
|
166
|
+
const list = byTurn.get(state.record.turn) ?? [];
|
|
167
|
+
list.push(state.record);
|
|
168
|
+
byTurn.set(state.record.turn, list);
|
|
169
|
+
}
|
|
170
|
+
return byTurn;
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
27
175
|
// src/agent-task/trace-projection/view.ts
|
|
28
176
|
var TASK_TURN_SPAN_NAME = "task.turn";
|
|
177
|
+
function steerRecordFrom(obs) {
|
|
178
|
+
const output = obs.output;
|
|
179
|
+
if (!output || typeof output !== "object") return void 0;
|
|
180
|
+
const asRecord2 = (v) => v !== null && typeof v === "object" && typeof v.number === "number" ? v : void 0;
|
|
181
|
+
const direct = asRecord2(output);
|
|
182
|
+
if (direct) return direct;
|
|
183
|
+
const tryParse = (text) => {
|
|
184
|
+
if (typeof text !== "string") return void 0;
|
|
185
|
+
try {
|
|
186
|
+
return asRecord2(JSON.parse(text));
|
|
187
|
+
} catch {
|
|
188
|
+
return void 0;
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
const fromText = tryParse(output.text);
|
|
192
|
+
if (fromText) return fromText;
|
|
193
|
+
const messages = output.messages;
|
|
194
|
+
if (Array.isArray(messages)) {
|
|
195
|
+
const content = messages[0]?.content;
|
|
196
|
+
const fromMessage = tryParse(content);
|
|
197
|
+
if (fromMessage) return fromMessage;
|
|
198
|
+
}
|
|
199
|
+
return void 0;
|
|
200
|
+
}
|
|
201
|
+
function traceSteerFrom(obs) {
|
|
202
|
+
const meta = obs.metadata;
|
|
203
|
+
const record = steerRecordFrom(obs);
|
|
204
|
+
const number = typeof record?.number === "number" ? record.number : typeof meta?.steerNumber === "number" ? meta.steerNumber : void 0;
|
|
205
|
+
if (number === void 0) return void 0;
|
|
206
|
+
const source = record ?? {};
|
|
207
|
+
const read = (key) => source[key] !== void 0 ? source[key] : meta?.[key];
|
|
208
|
+
const status = read("status");
|
|
209
|
+
const steer2 = {
|
|
210
|
+
number,
|
|
211
|
+
spanId: obs.spanId,
|
|
212
|
+
turn: typeof read("turn") === "number" ? read("turn") : 1,
|
|
213
|
+
trigger: triggerToString(read("trigger")),
|
|
214
|
+
message: source.message,
|
|
215
|
+
...isSteerStatus(status) ? { status } : { status: "undelivered" }
|
|
216
|
+
};
|
|
217
|
+
const label = read("label");
|
|
218
|
+
if (typeof label === "string") steer2.label = label;
|
|
219
|
+
const deliveredAt = read("deliveredAt");
|
|
220
|
+
if (typeof deliveredAt === "string") steer2.deliveredAt = deliveredAt;
|
|
221
|
+
if (obs.startedAt !== void 0) steer2.spanStartedAt = obs.startedAt;
|
|
222
|
+
const boundary = read("boundary");
|
|
223
|
+
if (typeof boundary === "string") steer2.boundary = boundary;
|
|
224
|
+
const reason = read("reason");
|
|
225
|
+
if (typeof reason === "string") steer2.reason = reason;
|
|
226
|
+
return steer2;
|
|
227
|
+
}
|
|
228
|
+
function isSteerStatus(v) {
|
|
229
|
+
return v === "delivered" || v === "undelivered" || v === "error";
|
|
230
|
+
}
|
|
231
|
+
function triggerToString(v) {
|
|
232
|
+
if (typeof v === "string") return v;
|
|
233
|
+
if (v !== null && typeof v === "object") {
|
|
234
|
+
try {
|
|
235
|
+
return JSON.stringify(v) ?? "unknown";
|
|
236
|
+
} catch {
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return "unknown";
|
|
240
|
+
}
|
|
29
241
|
function invocationOrderKey(obs) {
|
|
30
242
|
const hasTs = obs.startedAt != null ? 0 : 1;
|
|
31
243
|
return [hasTs, obs.startedAt ?? "", obs.spanId];
|
|
32
244
|
}
|
|
33
|
-
var TraceView = class {
|
|
245
|
+
var TraceView = class _TraceView {
|
|
34
246
|
constructor(snapshot) {
|
|
35
247
|
this.snapshot = snapshot;
|
|
36
248
|
}
|
|
@@ -79,6 +291,56 @@ var TraceView = class {
|
|
|
79
291
|
startedAt: o.startedAt
|
|
80
292
|
}));
|
|
81
293
|
}
|
|
294
|
+
/**
|
|
295
|
+
* Mid-run steers, derived from `task.steer` observations, in number order.
|
|
296
|
+
* Empty when the trace carries none — steering assertions fail closed on
|
|
297
|
+
* the empty case instead of passing vacuously.
|
|
298
|
+
*/
|
|
299
|
+
get steers() {
|
|
300
|
+
const out = [];
|
|
301
|
+
for (const obs of this.sortedObservations) {
|
|
302
|
+
if (obs.name !== TASK_STEER_SPAN_NAME) continue;
|
|
303
|
+
const steer2 = traceSteerFrom(obs);
|
|
304
|
+
if (steer2) out.push(steer2);
|
|
305
|
+
}
|
|
306
|
+
return out.sort((a, b) => a.number - b.number);
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Whether an observation started after the steer observation, using the
|
|
310
|
+
* same ordering key as everywhere else in this class (startedAt, then
|
|
311
|
+
* span id). The id tie-break matters on the local tee path, whose ISO
|
|
312
|
+
* timestamps truncate to milliseconds — a steer and the generation that
|
|
313
|
+
* consumes it can share one.
|
|
314
|
+
*/
|
|
315
|
+
static startedAfter(obs, steer2) {
|
|
316
|
+
if (obs.startedAt == null) return true;
|
|
317
|
+
if (steer2.spanStartedAt == null) return false;
|
|
318
|
+
if (obs.startedAt !== steer2.spanStartedAt) {
|
|
319
|
+
return obs.startedAt > steer2.spanStartedAt;
|
|
320
|
+
}
|
|
321
|
+
return obs.spanId > steer2.spanId;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* A TraceView over the run after steer n was delivered: observations that
|
|
325
|
+
* started after the steer's delivery timestamp (the first generation that
|
|
326
|
+
* consumed it through run end). Missing timestamps sort after timestamped
|
|
327
|
+
* observations per the projection contract, so they belong to the window
|
|
328
|
+
* too. An undelivered steer yields an empty view — assertions inside it
|
|
329
|
+
* fail on missing evidence rather than passing vacuously.
|
|
330
|
+
*/
|
|
331
|
+
windowAfterSteer(n) {
|
|
332
|
+
const steer2 = this.steers.find((s) => s.number === n);
|
|
333
|
+
if (!steer2) {
|
|
334
|
+
throw new Error(`No steer number ${n} in this trace.`);
|
|
335
|
+
}
|
|
336
|
+
if (steer2.spanStartedAt === void 0) {
|
|
337
|
+
return new _TraceView({ ...this.snapshot, observations: [] });
|
|
338
|
+
}
|
|
339
|
+
const filtered = this.snapshot.observations.filter(
|
|
340
|
+
(o) => o.spanId !== steer2.spanId && _TraceView.startedAfter(o, steer2)
|
|
341
|
+
);
|
|
342
|
+
return new _TraceView({ ...this.snapshot, observations: filtered });
|
|
343
|
+
}
|
|
82
344
|
/** Last assistant message content — the agent's "reply". Empty if none. */
|
|
83
345
|
get reply() {
|
|
84
346
|
const msgs = this.messages;
|
|
@@ -383,6 +645,33 @@ function levenshtein(a, b) {
|
|
|
383
645
|
import { createHash } from "crypto";
|
|
384
646
|
|
|
385
647
|
// src/agent-task/checks/second-judge.ts
|
|
648
|
+
function secondJudgeFacts(check) {
|
|
649
|
+
const sj = check.judge?.secondJudge;
|
|
650
|
+
if (!sj) return { kind: "none" };
|
|
651
|
+
if (sj.skipped) return { kind: "skipped" };
|
|
652
|
+
if (sj.error || sj.choice == null) return { kind: "error" };
|
|
653
|
+
const conf = sj.confidence ?? 0;
|
|
654
|
+
if (sj.choice === "pass" !== (check.pass === true)) {
|
|
655
|
+
return { kind: "split", confidence: conf };
|
|
656
|
+
}
|
|
657
|
+
if (conf < 0.6) return { kind: "unsure", confidence: conf };
|
|
658
|
+
return { kind: "agree", confidence: conf };
|
|
659
|
+
}
|
|
660
|
+
function secondJudgeTakeaway(check) {
|
|
661
|
+
const facts = secondJudgeFacts(check);
|
|
662
|
+
switch (facts.kind) {
|
|
663
|
+
case "split":
|
|
664
|
+
return `Verdicts differ \u2014 second judge contradicts at ${facts.confidence.toFixed(2)} confidence.`;
|
|
665
|
+
case "unsure":
|
|
666
|
+
return `Second judge unsure (${facts.confidence.toFixed(2)}) \u2014 weak corroboration.`;
|
|
667
|
+
case "error":
|
|
668
|
+
return `Second opinion failed to arrive${check.judge?.secondJudge?.error ? ` (${check.judge.secondJudge.error})` : ""}.`;
|
|
669
|
+
case "skipped":
|
|
670
|
+
return `Second opinion skipped${check.judge?.secondJudge?.skipped ? ` \u2014 ${check.judge.secondJudge.skipped}` : ""}.`;
|
|
671
|
+
default:
|
|
672
|
+
return null;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
386
675
|
var SECOND_JUDGE_TIMEOUT_MS = 3e4;
|
|
387
676
|
var OPENROUTER_RESERVED_HINT = "Point APO_SECOND_JUDGE_BASE_URL at an OpenRouter-style decisions API (e.g. https://openrouter.ai/api/v1) or set APO_SECOND_JUDGE_PROVIDER=openrouter";
|
|
388
677
|
var openRouterProvider = {
|
|
@@ -461,9 +750,24 @@ function resolveSecondJudgeProvider(baseURL) {
|
|
|
461
750
|
}
|
|
462
751
|
return /(^|\.)api\.openai\.com$/.test(new URL(baseURL).hostname) ? openAIProvider : openRouterProvider;
|
|
463
752
|
}
|
|
753
|
+
var INPUT_TOO_LARGE_MARKERS = [
|
|
754
|
+
"max_tokens_exceeded",
|
|
755
|
+
"context_length_exceeded",
|
|
756
|
+
"maximum context length"
|
|
757
|
+
];
|
|
758
|
+
function isInputTooLargeRejection(status, body) {
|
|
759
|
+
if (status !== 400 && status !== 413) return false;
|
|
760
|
+
return INPUT_TOO_LARGE_MARKERS.some((marker2) => body.includes(marker2));
|
|
761
|
+
}
|
|
762
|
+
function estimateTokens(text) {
|
|
763
|
+
return Math.round(text.length / 4 / 1e3) * 1e3;
|
|
764
|
+
}
|
|
464
765
|
async function callSecondJudge(args) {
|
|
465
766
|
const startedAt = Date.now();
|
|
466
|
-
const evidence = {
|
|
767
|
+
const evidence = {
|
|
768
|
+
model: args.model,
|
|
769
|
+
...args.projected ? { projected: true } : {}
|
|
770
|
+
};
|
|
467
771
|
try {
|
|
468
772
|
const provider = resolveSecondJudgeProvider(args.baseURL);
|
|
469
773
|
if (provider.reserved) {
|
|
@@ -481,6 +785,10 @@ async function callSecondJudge(args) {
|
|
|
481
785
|
});
|
|
482
786
|
if (!response.ok) {
|
|
483
787
|
const body = await response.text().catch(() => "");
|
|
788
|
+
if (isInputTooLargeRejection(response.status, body)) {
|
|
789
|
+
evidence.skipped = `input ~${estimateTokens(args.state).toLocaleString("en-US")} tokens exceeds ${args.model}'s context limit \u2014 project a smaller view with t.judge(value, instruction, { secondJudgeValue })`;
|
|
790
|
+
return evidence;
|
|
791
|
+
}
|
|
484
792
|
evidence.error = `Second judge API ${response.status}: ${body.slice(0, 200)}`;
|
|
485
793
|
return evidence;
|
|
486
794
|
}
|
|
@@ -520,8 +828,9 @@ var JudgeUnavailableError = class extends Error {
|
|
|
520
828
|
this.name = "JudgeUnavailableError";
|
|
521
829
|
}
|
|
522
830
|
};
|
|
523
|
-
var
|
|
524
|
-
var
|
|
831
|
+
var QUOTE_RULE = " Inside the reasoning, quote the output with single quotes, never with unescaped double quotes.";
|
|
832
|
+
var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}.' + QUOTE_RULE;
|
|
833
|
+
var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}.' + QUOTE_RULE;
|
|
525
834
|
function isJudgeVerdictFirstOverrideEnabled() {
|
|
526
835
|
const value = process.env.APO_JUDGE_VERDICT_FIRST?.trim().toLowerCase();
|
|
527
836
|
return value === "1" || value === "true";
|
|
@@ -532,6 +841,23 @@ function judgeResponseContract() {
|
|
|
532
841
|
function judgeContractInUse() {
|
|
533
842
|
return isJudgeVerdictFirstOverrideEnabled() ? "verdict-first" : "reasoning-first";
|
|
534
843
|
}
|
|
844
|
+
function judgeResponseFormat() {
|
|
845
|
+
const properties = isJudgeVerdictFirstOverrideEnabled() ? { pass: { type: "boolean" }, reasoning: { type: "string" } } : { reasoning: { type: "string" }, pass: { type: "boolean" } };
|
|
846
|
+
return {
|
|
847
|
+
type: "json_schema",
|
|
848
|
+
json_schema: {
|
|
849
|
+
name: "judge_verdict",
|
|
850
|
+
strict: true,
|
|
851
|
+
schema: {
|
|
852
|
+
type: "object",
|
|
853
|
+
properties,
|
|
854
|
+
required: Object.keys(properties),
|
|
855
|
+
additionalProperties: false
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
var JSON_OBJECT_FORMAT = { type: "json_object" };
|
|
535
861
|
function judgeSystemPrompt() {
|
|
536
862
|
return `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${judgeResponseContract()}`;
|
|
537
863
|
}
|
|
@@ -590,9 +916,14 @@ function parseJudgeJson(raw) {
|
|
|
590
916
|
} catch {
|
|
591
917
|
}
|
|
592
918
|
}
|
|
919
|
+
return {};
|
|
920
|
+
}
|
|
921
|
+
function judgeVerdict(raw) {
|
|
922
|
+
const parsed = parseJudgeJson(raw);
|
|
923
|
+
if (typeof parsed.pass !== "boolean") return void 0;
|
|
593
924
|
return {
|
|
594
|
-
pass:
|
|
595
|
-
reasoning:
|
|
925
|
+
pass: parsed.pass,
|
|
926
|
+
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : ""
|
|
596
927
|
};
|
|
597
928
|
}
|
|
598
929
|
var JUDGE_IDLE_TIMEOUT_MS = 9e4;
|
|
@@ -735,33 +1066,32 @@ function runWithSharedPrefix(key, task2) {
|
|
|
735
1066
|
);
|
|
736
1067
|
return next;
|
|
737
1068
|
}
|
|
1069
|
+
function defaultJudgeBaseURL(baseURL) {
|
|
1070
|
+
return baseURL ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1";
|
|
1071
|
+
}
|
|
1072
|
+
function defaultJudgeAPIKey(apiKey) {
|
|
1073
|
+
return apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
|
|
1074
|
+
}
|
|
738
1075
|
async function callJudge(args) {
|
|
739
|
-
const baseURL = args.baseURL
|
|
740
|
-
const apiKey = args.apiKey
|
|
741
|
-
const deliverableText
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
1076
|
+
const baseURL = defaultJudgeBaseURL(args.baseURL);
|
|
1077
|
+
const apiKey = defaultJudgeAPIKey(args.apiKey);
|
|
1078
|
+
const { briefingText, instructionText, systemPromptText, deliverableText, secondJudgeState, secondJudgeProjected } = buildJudgePromptParts({
|
|
1079
|
+
values: args.values,
|
|
1080
|
+
instruction: args.instruction,
|
|
1081
|
+
...args.prompt ? { prompt: args.prompt } : {},
|
|
1082
|
+
...args.context ? { context: args.context } : {},
|
|
1083
|
+
...args.secondJudgeValue !== void 0 ? { secondJudgeValue: args.secondJudgeValue } : {}
|
|
1084
|
+
});
|
|
747
1085
|
const secondJudgeModel = resolveSecondJudgeModel();
|
|
748
|
-
const
|
|
749
|
-
|
|
750
|
-
const secondJudgeDeliverableText = `Values to evaluate:
|
|
751
|
-
${formatJudgeValues(secondJudgeValues)}`;
|
|
752
|
-
const secondJudgePromise = secondJudgeModel ? callSecondJudge({
|
|
753
|
-
state: `${briefingText}
|
|
754
|
-
|
|
755
|
-
${secondJudgeDeliverableText}
|
|
756
|
-
|
|
757
|
-
${instructionText}`,
|
|
1086
|
+
const secondJudgePromise = args.prefetchedSecondJudge ? Promise.resolve(args.prefetchedSecondJudge) : secondJudgeModel ? callSecondJudge({
|
|
1087
|
+
state: secondJudgeState,
|
|
758
1088
|
model: secondJudgeModel,
|
|
759
1089
|
baseURL: resolveSecondJudgeBaseURL(baseURL),
|
|
760
1090
|
apiKey: resolveSecondJudgeAPIKey(apiKey),
|
|
761
1091
|
...secondJudgeProjected ? { projected: true } : {}
|
|
762
1092
|
}) : void 0;
|
|
763
1093
|
const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
|
|
764
|
-
const requestBody = (stream) => JSON.stringify({
|
|
1094
|
+
const requestBody = (stream, schema) => JSON.stringify({
|
|
765
1095
|
model: args.model,
|
|
766
1096
|
prompt_cache_key: promptCacheKey(cacheKey),
|
|
767
1097
|
messages: [
|
|
@@ -779,10 +1109,10 @@ ${instructionText}`,
|
|
|
779
1109
|
{ role: "user", content: instructionText }
|
|
780
1110
|
],
|
|
781
1111
|
temperature: 0,
|
|
782
|
-
response_format:
|
|
1112
|
+
response_format: schema ? judgeResponseFormat() : JSON_OBJECT_FORMAT,
|
|
783
1113
|
...stream ? { stream: true, stream_options: { include_usage: true } } : {}
|
|
784
1114
|
});
|
|
785
|
-
const attempt = async (stream, maxDeadline) => {
|
|
1115
|
+
const attempt = async (stream, schema, maxDeadline) => {
|
|
786
1116
|
const controller = new AbortController();
|
|
787
1117
|
const expire = (reason, name) => () => controller.abort(new DOMException(reason, name));
|
|
788
1118
|
const firstDataMs = judgeFirstDataTimeoutMs();
|
|
@@ -818,7 +1148,7 @@ ${instructionText}`,
|
|
|
818
1148
|
"Content-Type": "application/json",
|
|
819
1149
|
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
820
1150
|
},
|
|
821
|
-
body: requestBody(stream)
|
|
1151
|
+
body: requestBody(stream, schema)
|
|
822
1152
|
});
|
|
823
1153
|
} catch (error) {
|
|
824
1154
|
return transportFailure("Judge request failed", controller.signal.reason ?? error);
|
|
@@ -832,7 +1162,8 @@ ${instructionText}`,
|
|
|
832
1162
|
...isRetryableStatus(response.status) ? { retryDelayMs: retryDelayMs(response.headers.get("retry-after")) } : {},
|
|
833
1163
|
// Name the field, not the word: gateways echo request params in
|
|
834
1164
|
// unrelated 400 bodies.
|
|
835
|
-
...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {}
|
|
1165
|
+
...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {},
|
|
1166
|
+
...schema && response.status === 400 && /json_schema|response_format/.test(body) ? { schemaRejected: true } : {}
|
|
836
1167
|
};
|
|
837
1168
|
}
|
|
838
1169
|
try {
|
|
@@ -864,17 +1195,21 @@ ${instructionText}`,
|
|
|
864
1195
|
const startedAt = Date.now();
|
|
865
1196
|
const deadline = startedAt + judgeMaxDurationMs();
|
|
866
1197
|
let stream = true;
|
|
867
|
-
let
|
|
868
|
-
|
|
869
|
-
|
|
1198
|
+
let schema = true;
|
|
1199
|
+
let result = await attempt(stream, schema, deadline);
|
|
1200
|
+
while (result.kind === "unavailable" && (result.streamRejected || result.schemaRejected)) {
|
|
1201
|
+
if (result.streamRejected) stream = false;
|
|
1202
|
+
if (result.schemaRejected) schema = false;
|
|
1203
|
+
result = await attempt(stream, schema, deadline);
|
|
1204
|
+
}
|
|
1205
|
+
const noVerdict = (r) => r.kind === "completion" && (isTruncated(r.completion) || judgeVerdict(r.completion.text) === void 0);
|
|
870
1206
|
const retryable = result.kind === "unavailable" && result.retryable;
|
|
871
|
-
if (
|
|
872
|
-
const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs :
|
|
1207
|
+
if (noVerdict(result) || retryable) {
|
|
1208
|
+
const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs : JUDGE_RETRY_DELAY_MS;
|
|
873
1209
|
if (deadline - Date.now() - delay >= JUDGE_MIN_RETRY_BUDGET_MS) {
|
|
874
|
-
if (streamRejected) stream = false;
|
|
875
1210
|
await new Promise((resolve5) => setTimeout(resolve5, delay));
|
|
876
1211
|
const first = result;
|
|
877
|
-
result = await attempt(stream, deadline);
|
|
1212
|
+
result = await attempt(stream, schema, deadline);
|
|
878
1213
|
if (result.kind === "unavailable" && first.kind === "unavailable") {
|
|
879
1214
|
result = {
|
|
880
1215
|
...result,
|
|
@@ -904,12 +1239,16 @@ ${instructionText}`,
|
|
|
904
1239
|
unavailable: true
|
|
905
1240
|
};
|
|
906
1241
|
}
|
|
907
|
-
const
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
1242
|
+
const verdict = judgeVerdict(text);
|
|
1243
|
+
if (verdict === void 0) {
|
|
1244
|
+
return {
|
|
1245
|
+
pass: false,
|
|
1246
|
+
reasoning: "Judge reply carried no verdict (no boolean `pass`) on two draws. The verdict is unknown, so this check is recorded as a judge error, not a verdict. The raw reply is on the judge metadata.",
|
|
1247
|
+
judge,
|
|
1248
|
+
unavailable: true
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
return { ...verdict, judge };
|
|
913
1252
|
});
|
|
914
1253
|
}
|
|
915
1254
|
function transportFailure(prefix, error) {
|
|
@@ -925,6 +1264,29 @@ function transportFailure(prefix, error) {
|
|
|
925
1264
|
retryable: !timedOut
|
|
926
1265
|
};
|
|
927
1266
|
}
|
|
1267
|
+
function buildJudgePromptParts(args) {
|
|
1268
|
+
const deliverableText = `Values to evaluate:
|
|
1269
|
+
${formatJudgeValues(args.values)}`;
|
|
1270
|
+
const { briefingText, instructionText } = assembleBriefing(args);
|
|
1271
|
+
const secondJudgeProjected = args.secondJudgeValue !== void 0;
|
|
1272
|
+
const secondJudgeValues = args.secondJudgeValue ?? args.values;
|
|
1273
|
+
const secondJudgeDeliverableText = `Values to evaluate:
|
|
1274
|
+
${formatJudgeValues(secondJudgeValues)}`;
|
|
1275
|
+
return {
|
|
1276
|
+
briefingText,
|
|
1277
|
+
instructionText,
|
|
1278
|
+
systemPromptText: `${briefingText}
|
|
1279
|
+
|
|
1280
|
+
${deliverableText}`,
|
|
1281
|
+
deliverableText,
|
|
1282
|
+
secondJudgeState: `${briefingText}
|
|
1283
|
+
|
|
1284
|
+
${secondJudgeDeliverableText}
|
|
1285
|
+
|
|
1286
|
+
${instructionText}`,
|
|
1287
|
+
secondJudgeProjected
|
|
1288
|
+
};
|
|
1289
|
+
}
|
|
928
1290
|
function assembleBriefing(args) {
|
|
929
1291
|
if (!args.prompt) {
|
|
930
1292
|
return {
|
|
@@ -953,6 +1315,55 @@ ${args.instruction}`
|
|
|
953
1315
|
};
|
|
954
1316
|
}
|
|
955
1317
|
|
|
1318
|
+
// src/agent-task/checks/judge-cascade.ts
|
|
1319
|
+
var CASCADE_CONFIDENCE_THRESHOLD = 0.95;
|
|
1320
|
+
function isCascadeActive(effective) {
|
|
1321
|
+
return effective?.mode === "cascade" && resolveSecondJudgeModel() !== void 0;
|
|
1322
|
+
}
|
|
1323
|
+
async function cascadePreflight(args) {
|
|
1324
|
+
const model = resolveSecondJudgeModel();
|
|
1325
|
+
if (model === void 0) return { kind: "off" };
|
|
1326
|
+
const parts = buildJudgePromptParts({
|
|
1327
|
+
values: args.values,
|
|
1328
|
+
instruction: args.instruction,
|
|
1329
|
+
...args.effective.prompt ? { prompt: args.effective.prompt } : {},
|
|
1330
|
+
...args.context ? { context: args.context } : {},
|
|
1331
|
+
...args.secondJudgeValue !== void 0 ? { secondJudgeValue: args.secondJudgeValue } : {}
|
|
1332
|
+
});
|
|
1333
|
+
const evidence = await callSecondJudge({
|
|
1334
|
+
state: parts.secondJudgeState,
|
|
1335
|
+
model,
|
|
1336
|
+
baseURL: resolveSecondJudgeBaseURL(defaultJudgeBaseURL(args.effective.baseURL)),
|
|
1337
|
+
apiKey: resolveSecondJudgeAPIKey(defaultJudgeAPIKey(args.effective.apiKey)),
|
|
1338
|
+
...parts.secondJudgeProjected ? { projected: true } : {}
|
|
1339
|
+
});
|
|
1340
|
+
return { kind: "ran", evidence, parts };
|
|
1341
|
+
}
|
|
1342
|
+
function cascadeDecides(evidence) {
|
|
1343
|
+
return evidence !== void 0 && (evidence.choice === "pass" || evidence.choice === "fail") && typeof evidence.confidence === "number" && evidence.confidence >= CASCADE_CONFIDENCE_THRESHOLD && evidence.error === void 0 && evidence.skipped === void 0 && evidence.projected !== true;
|
|
1344
|
+
}
|
|
1345
|
+
function cascadeVerdict(evidence, parts) {
|
|
1346
|
+
const pass = evidence.choice === "pass";
|
|
1347
|
+
return {
|
|
1348
|
+
pass,
|
|
1349
|
+
reasoning: `Verdict by second judge (cascade): ${evidence.choice} \xB7 confidence ${evidence.confidence}` + (evidence.passProbability !== void 0 ? ` \xB7 p(pass) ${evidence.passProbability}` : "") + `. Primary judge not called.`,
|
|
1350
|
+
judge: {
|
|
1351
|
+
model: evidence.model,
|
|
1352
|
+
prompt: { system: parts.systemPromptText, user: parts.instructionText },
|
|
1353
|
+
response: JSON.stringify({
|
|
1354
|
+
choice: evidence.choice,
|
|
1355
|
+
probabilities: { pass: evidence.passProbability },
|
|
1356
|
+
confidence: evidence.confidence
|
|
1357
|
+
}),
|
|
1358
|
+
...evidence.inputTokens !== void 0 ? { tokens: { input: evidence.inputTokens, output: 0 } } : {},
|
|
1359
|
+
...evidence.latencyMs !== void 0 ? { latency_ms: evidence.latencyMs } : {},
|
|
1360
|
+
...evidence.costUsd !== void 0 ? { cost: evidence.costUsd } : {},
|
|
1361
|
+
secondJudge: evidence,
|
|
1362
|
+
verdict_by: "second-judge"
|
|
1363
|
+
}
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
|
|
956
1367
|
// src/agent-task/checks/agent-session.ts
|
|
957
1368
|
import { createHash as createHash3 } from "crypto";
|
|
958
1369
|
|
|
@@ -1362,6 +1773,25 @@ function isAbortError(error) {
|
|
|
1362
1773
|
const name = error instanceof Error ? error.name : "";
|
|
1363
1774
|
return name === "TimeoutError" || name === "AbortError" || name === "ResponseAborted";
|
|
1364
1775
|
}
|
|
1776
|
+
var LEADING_PCRE_FLAGS = /^\(\?([a-z]+)\)/;
|
|
1777
|
+
var PCRE_FLAGS_JS_SUPPORTS = /* @__PURE__ */ new Set(["i", "m", "s"]);
|
|
1778
|
+
function stripLeadingPcreFlags(pattern) {
|
|
1779
|
+
const m = LEADING_PCRE_FLAGS.exec(pattern);
|
|
1780
|
+
if (!m) return { pattern, flags: "gim" };
|
|
1781
|
+
const unsupported = [...m[1]].filter((f) => !PCRE_FLAGS_JS_SUPPORTS.has(f));
|
|
1782
|
+
if (unsupported.length > 0) {
|
|
1783
|
+
throw new Error(`leading flag group (?${m[1]}) \u2014 JS RegExp has no equivalent for ${unsupported.join(", ")}`);
|
|
1784
|
+
}
|
|
1785
|
+
return {
|
|
1786
|
+
pattern: pattern.slice(m[0].length),
|
|
1787
|
+
flags: m[1].includes("s") ? "gims" : "gim"
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
var UNSUPPORTED_PCRE_CONSTRUCTS = /\\[AQZz]/;
|
|
1791
|
+
function rejectUnsupportedPcreConstructs(pattern) {
|
|
1792
|
+
const m = UNSUPPORTED_PCRE_CONSTRUCTS.exec(pattern);
|
|
1793
|
+
return m ? `unsupported PCRE construct ${m[0]} \u2014 JS RegExp has no equivalent; anchor with ^ / $ (the search is multiline) or rewrite the pattern` : void 0;
|
|
1794
|
+
}
|
|
1365
1795
|
function aggregateUsage(steps) {
|
|
1366
1796
|
return steps.reduce(
|
|
1367
1797
|
(acc, s) => ({
|
|
@@ -1377,14 +1807,15 @@ function recordedCost(steps) {
|
|
|
1377
1807
|
return known.reduce((a, s) => a + s.tokens?.cost, 0);
|
|
1378
1808
|
}
|
|
1379
1809
|
function extractVerdict(steps) {
|
|
1380
|
-
const call
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1810
|
+
for (const call of steps.flatMap((s) => s.tool_calls ?? [])) {
|
|
1811
|
+
if (call.name !== "finish_verdict" || !call.input) continue;
|
|
1812
|
+
try {
|
|
1813
|
+
const args = JSON.parse(call.input);
|
|
1814
|
+
if (typeof args.reasoning === "string" && args.reasoning.length > 0 && typeof args.pass === "boolean") {
|
|
1815
|
+
return { reasoning: args.reasoning, pass: args.pass };
|
|
1816
|
+
}
|
|
1817
|
+
} catch {
|
|
1386
1818
|
}
|
|
1387
|
-
} catch {
|
|
1388
1819
|
}
|
|
1389
1820
|
return void 0;
|
|
1390
1821
|
}
|
|
@@ -1401,7 +1832,15 @@ function buildEvidenceTools(args) {
|
|
|
1401
1832
|
});
|
|
1402
1833
|
return ledger.readBytesUsed > budget.maxReadBytes ? overflow() : null;
|
|
1403
1834
|
};
|
|
1404
|
-
const
|
|
1835
|
+
const dedupeCache = /* @__PURE__ */ new Map();
|
|
1836
|
+
const span = (name, input, fn) => {
|
|
1837
|
+
const key = `${stepIndexOf()}|${name}|${JSON.stringify(input ?? {})}`;
|
|
1838
|
+
const cached = dedupeCache.get(key);
|
|
1839
|
+
if (cached) return cached;
|
|
1840
|
+
const execution = tracer ? tracer.traceTool(name, input, fn) : fn();
|
|
1841
|
+
dedupeCache.set(key, execution);
|
|
1842
|
+
return execution;
|
|
1843
|
+
};
|
|
1405
1844
|
const tools = {
|
|
1406
1845
|
read_deliverable: tool({
|
|
1407
1846
|
description: `Read one of this run's deliverables by name, with optional offset/limit for pagination (max ${MAX_READ_WINDOW} bytes per call).`,
|
|
@@ -1432,7 +1871,7 @@ function buildEvidenceTools(args) {
|
|
|
1432
1871
|
}
|
|
1433
1872
|
}),
|
|
1434
1873
|
search_deliverable: tool({
|
|
1435
|
-
description: "Regex-search one of this run's deliverables; returns up to 8 matches with surrounding context. Cheaper than reading a large deliverable end to end \u2014 use it to locate the relevant part first.",
|
|
1874
|
+
description: "Regex-search one of this run's deliverables (case-insensitive; ^ and $ match line starts/ends); returns up to 8 matches with surrounding context. Cheaper than reading a large deliverable end to end \u2014 use it to locate the relevant part first.",
|
|
1436
1875
|
inputSchema: z.object({ name: z.string(), pattern: z.string() }),
|
|
1437
1876
|
execute: async (input) => {
|
|
1438
1877
|
const { name, pattern } = input;
|
|
@@ -1445,11 +1884,16 @@ function buildEvidenceTools(args) {
|
|
|
1445
1884
|
if (pattern.length > SEARCH_MAX_PATTERN) {
|
|
1446
1885
|
return { error: `pattern too long (max ${SEARCH_MAX_PATTERN} chars)` };
|
|
1447
1886
|
}
|
|
1887
|
+
const unsupported = rejectUnsupportedPcreConstructs(pattern);
|
|
1888
|
+
if (unsupported) return { error: unsupported };
|
|
1448
1889
|
let re;
|
|
1449
1890
|
try {
|
|
1450
|
-
|
|
1891
|
+
const stripped = stripLeadingPcreFlags(pattern);
|
|
1892
|
+
re = new RegExp(stripped.pattern, stripped.flags);
|
|
1451
1893
|
} catch (error) {
|
|
1452
|
-
return {
|
|
1894
|
+
return {
|
|
1895
|
+
error: `invalid regex: ${error instanceof Error ? error.message : String(error)} \u2014 patterns compile as case-insensitive multiline; leading PCRE flags like (?i) are stripped automatically`
|
|
1896
|
+
};
|
|
1453
1897
|
}
|
|
1454
1898
|
const content = renderValue2(evidence.deliverables[name]).slice(0, SEARCH_SCAN_LIMIT);
|
|
1455
1899
|
const hits = [];
|
|
@@ -1846,7 +2290,12 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer,
|
|
|
1846
2290
|
const detail = result.outcome === "error" ? `agentic judge session errored before verdicting: ${result.error ?? "unknown error"}` : `agentic judge session ended without a verdict (budget exhausted after ${result.session.usage?.steps ?? "?"} steps; last tool: ${result.session.steps?.at(-1)?.tool_calls?.map((c) => c.name).join(", ") ?? "none"})`;
|
|
1847
2291
|
const agreed = opts?.expect === void 0 ? result.verdict?.pass ?? false : result.verdict ? result.verdict.pass === (opts.expect === "pass") : false;
|
|
1848
2292
|
const reasoning = opts?.expect === void 0 || agreed || !result.verdict ? result.verdict?.reasoning ?? detail : `judge ${result.verdict.pass ? "PASSed" : "FAILed"} where ground truth is ${opts.expect.toUpperCase()} \u2014 judge reasoning: ${result.verdict.reasoning}`;
|
|
1849
|
-
|
|
2293
|
+
const consultedEvidence = (result.session.evidence?.length ?? 0) > 0 || exhibits.length > 0 || (result.session.steps?.some(
|
|
2294
|
+
(s) => s.tool_calls?.some((c) => c.name === "get_trace" || c.name === "list_runs" || c.name === "get_run")
|
|
2295
|
+
) ?? false);
|
|
2296
|
+
const readNothing = result.verdict !== void 0 && !consultedEvidence && evidence !== void 0 && Object.keys(evidence.deliverables).length > 0;
|
|
2297
|
+
const recorded = readNothing ? `${reasoning} \u2014 [vacuous verdict: the session consulted no run evidence before deciding]` : reasoning;
|
|
2298
|
+
rec.record(label, agreed, recorded, {
|
|
1850
2299
|
evaluator_type: "agent",
|
|
1851
2300
|
judge,
|
|
1852
2301
|
expected: instruction,
|
|
@@ -1873,7 +2322,8 @@ function resolveJudgeConfig(judgeConfig, override) {
|
|
|
1873
2322
|
model,
|
|
1874
2323
|
baseURL: override?.baseURL ?? judgeConfig?.baseURL,
|
|
1875
2324
|
apiKey: override?.apiKey ?? judgeConfig?.apiKey,
|
|
1876
|
-
prompt: override?.prompt ?? judgeConfig?.prompt
|
|
2325
|
+
prompt: override?.prompt ?? judgeConfig?.prompt,
|
|
2326
|
+
mode: override?.mode ?? judgeConfig?.mode
|
|
1877
2327
|
};
|
|
1878
2328
|
}
|
|
1879
2329
|
var TEST_METHOD_NAMES = [
|
|
@@ -1890,6 +2340,8 @@ var TEST_METHOD_NAMES = [
|
|
|
1890
2340
|
"maxDurationMs",
|
|
1891
2341
|
"maxTokens",
|
|
1892
2342
|
"minTokens",
|
|
2343
|
+
"steerDelivered",
|
|
2344
|
+
"afterSteer",
|
|
1893
2345
|
"assert",
|
|
1894
2346
|
"check",
|
|
1895
2347
|
"judge",
|
|
@@ -1912,6 +2364,58 @@ function trackPending(rec, method) {
|
|
|
1912
2364
|
return promise;
|
|
1913
2365
|
});
|
|
1914
2366
|
}
|
|
2367
|
+
function createSteerAssertions(view, rec, makeNested) {
|
|
2368
|
+
const findSteer = (n, id) => {
|
|
2369
|
+
const steers = view.steers;
|
|
2370
|
+
if (steers.length === 0) {
|
|
2371
|
+
rec.record(
|
|
2372
|
+
id,
|
|
2373
|
+
false,
|
|
2374
|
+
"steering evidence unavailable \u2014 no task.steer events in this trace",
|
|
2375
|
+
{ expected: `steer ${n} delivered and consumed`, received: "no steering evidence" }
|
|
2376
|
+
);
|
|
2377
|
+
return void 0;
|
|
2378
|
+
}
|
|
2379
|
+
const steer2 = steers.find((s) => s.number === n);
|
|
2380
|
+
if (!steer2) {
|
|
2381
|
+
rec.record(
|
|
2382
|
+
id,
|
|
2383
|
+
false,
|
|
2384
|
+
`no steer number ${n} in this trace (found: ${steers.map((s) => s.number).join(", ")})`,
|
|
2385
|
+
{ expected: `steer ${n} exists`, received: steers.map((s) => s.number).join(", ") || "none" }
|
|
2386
|
+
);
|
|
2387
|
+
}
|
|
2388
|
+
return steer2;
|
|
2389
|
+
};
|
|
2390
|
+
return {
|
|
2391
|
+
steerDelivered(n) {
|
|
2392
|
+
const steer2 = findSteer(n, `steerDelivered(${n})`);
|
|
2393
|
+
if (!steer2) return;
|
|
2394
|
+
if (steer2.status !== "delivered") {
|
|
2395
|
+
const why = steer2.reason ? ` \u2014 ${steer2.reason}` : "";
|
|
2396
|
+
rec.record(`steerDelivered(${n})`, false, `steer ${n} was ${steer2.status}${why}`, {
|
|
2397
|
+
expected: "delivered",
|
|
2398
|
+
received: steer2.status
|
|
2399
|
+
});
|
|
2400
|
+
return;
|
|
2401
|
+
}
|
|
2402
|
+
const consumed = view.snapshot.observations.some(
|
|
2403
|
+
(o) => o.type === "GENERATION" && steer2.spanStartedAt != null && o.startedAt != null && (o.startedAt > steer2.spanStartedAt || o.startedAt === steer2.spanStartedAt && o.spanId > steer2.spanId)
|
|
2404
|
+
);
|
|
2405
|
+
rec.record(
|
|
2406
|
+
`steerDelivered(${n})`,
|
|
2407
|
+
consumed,
|
|
2408
|
+
consumed ? "" : `steer ${n} was delivered but no generation observation started after it \u2014 the harness queued the message, no model call demonstrably consumed it`,
|
|
2409
|
+
{ expected: `\u22651 generation after steer ${n}`, received: consumed ? "1+" : "0" }
|
|
2410
|
+
);
|
|
2411
|
+
},
|
|
2412
|
+
afterSteer(n, fn) {
|
|
2413
|
+
const steer2 = findSteer(n, `afterSteer(${n})`);
|
|
2414
|
+
if (!steer2) return;
|
|
2415
|
+
fn(makeNested(view.windowAfterSteer(n)));
|
|
2416
|
+
}
|
|
2417
|
+
};
|
|
2418
|
+
}
|
|
1915
2419
|
function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
1916
2420
|
return async (values, instruction, opts) => {
|
|
1917
2421
|
const label = opts?.label ?? "judge";
|
|
@@ -1931,6 +2435,52 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
1931
2435
|
const context2 = judgeScopeToContext(judgeScope);
|
|
1932
2436
|
const judgeServing = servingHostFromBaseURL(effective.baseURL);
|
|
1933
2437
|
const secondJudgeValue = opts?.secondJudgeValue;
|
|
2438
|
+
let prefetchedSecondJudge;
|
|
2439
|
+
if (isCascadeActive(effective)) {
|
|
2440
|
+
let preflight;
|
|
2441
|
+
try {
|
|
2442
|
+
preflight = await cascadePreflight({
|
|
2443
|
+
values: valueArray,
|
|
2444
|
+
instruction,
|
|
2445
|
+
effective,
|
|
2446
|
+
...context2 ? { context: context2 } : {},
|
|
2447
|
+
...secondJudgeValue !== void 0 ? {
|
|
2448
|
+
secondJudgeValue: Array.isArray(secondJudgeValue) ? secondJudgeValue : [secondJudgeValue]
|
|
2449
|
+
} : {}
|
|
2450
|
+
});
|
|
2451
|
+
} catch (error) {
|
|
2452
|
+
rec.record(
|
|
2453
|
+
label,
|
|
2454
|
+
false,
|
|
2455
|
+
`judge failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2456
|
+
{
|
|
2457
|
+
evaluator_type: "llm",
|
|
2458
|
+
expected: instruction,
|
|
2459
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
2460
|
+
location,
|
|
2461
|
+
outcome: "error"
|
|
2462
|
+
}
|
|
2463
|
+
);
|
|
2464
|
+
return;
|
|
2465
|
+
}
|
|
2466
|
+
if (preflight.kind === "ran" && cascadeDecides(preflight.evidence)) {
|
|
2467
|
+
const { pass, reasoning, judge } = cascadeVerdict(
|
|
2468
|
+
preflight.evidence,
|
|
2469
|
+
preflight.parts
|
|
2470
|
+
);
|
|
2471
|
+
const agreed = opts?.expect === void 0 ? pass : pass === (opts.expect === "pass");
|
|
2472
|
+
const recordedReasoning = opts?.expect === void 0 || agreed ? reasoning : `second judge ${pass ? "PASSed" : "FAILed"} where ground truth is ${opts.expect.toUpperCase()} \u2014 ${reasoning}`;
|
|
2473
|
+
rec.record(label, agreed, recordedReasoning, {
|
|
2474
|
+
evaluator_type: "llm",
|
|
2475
|
+
judge,
|
|
2476
|
+
expected: instruction,
|
|
2477
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
2478
|
+
location
|
|
2479
|
+
});
|
|
2480
|
+
return;
|
|
2481
|
+
}
|
|
2482
|
+
if (preflight.kind === "ran") prefetchedSecondJudge = preflight.evidence;
|
|
2483
|
+
}
|
|
1934
2484
|
const call = () => callJudge({
|
|
1935
2485
|
values: valueArray,
|
|
1936
2486
|
instruction,
|
|
@@ -1939,6 +2489,7 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
1939
2489
|
apiKey: effective.apiKey,
|
|
1940
2490
|
prompt: effective.prompt,
|
|
1941
2491
|
...context2 ? { context: context2 } : {},
|
|
2492
|
+
...prefetchedSecondJudge ? { prefetchedSecondJudge } : {},
|
|
1942
2493
|
...secondJudgeValue !== void 0 ? {
|
|
1943
2494
|
secondJudgeValue: Array.isArray(secondJudgeValue) ? secondJudgeValue : [secondJudgeValue]
|
|
1944
2495
|
} : {}
|
|
@@ -2211,6 +2762,19 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
|
|
|
2211
2762
|
received: pass ? "true" : "false"
|
|
2212
2763
|
});
|
|
2213
2764
|
},
|
|
2765
|
+
...createSteerAssertions(
|
|
2766
|
+
view,
|
|
2767
|
+
rec,
|
|
2768
|
+
(window) => createTraceTestContext(
|
|
2769
|
+
window,
|
|
2770
|
+
rec,
|
|
2771
|
+
judgeConfig,
|
|
2772
|
+
judgeScope,
|
|
2773
|
+
agentEvidence,
|
|
2774
|
+
judgeTracer,
|
|
2775
|
+
judgeTools
|
|
2776
|
+
)
|
|
2777
|
+
),
|
|
2214
2778
|
check(value, matcher, label) {
|
|
2215
2779
|
const pass = matcher.test(value);
|
|
2216
2780
|
rec.record(
|
|
@@ -2887,6 +3451,29 @@ function resolveTurn(adapterTurn, taskTurn) {
|
|
|
2887
3451
|
return resolved;
|
|
2888
3452
|
}
|
|
2889
3453
|
|
|
3454
|
+
// src/agent-task/steer.ts
|
|
3455
|
+
var STEERS_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/task-steers");
|
|
3456
|
+
function steer(spec) {
|
|
3457
|
+
if (spec.turn !== void 0 && (!Number.isInteger(spec.turn) || spec.turn < 1)) {
|
|
3458
|
+
throw new Error(`steer: turn must be a positive integer, got ${spec.turn}`);
|
|
3459
|
+
}
|
|
3460
|
+
const count = spec.when === "runStart" ? void 0 : "toolResults" in spec.when ? ["toolResults", spec.when.toolResults] : ["assistantReply", spec.when.assistantReply];
|
|
3461
|
+
if (count !== void 0 && (!Number.isInteger(count[1]) || count[1] < 1)) {
|
|
3462
|
+
throw new Error(`steer: ${count[0]} must be a positive integer, got ${count[1]}`);
|
|
3463
|
+
}
|
|
3464
|
+
const registry2 = globalThis[STEERS_KEY];
|
|
3465
|
+
const list = Array.isArray(registry2) ? registry2 : [];
|
|
3466
|
+
list.push(spec);
|
|
3467
|
+
globalThis[STEERS_KEY] = list;
|
|
3468
|
+
}
|
|
3469
|
+
function getTaskSteers() {
|
|
3470
|
+
const registry2 = globalThis[STEERS_KEY];
|
|
3471
|
+
return Array.isArray(registry2) ? [...registry2] : [];
|
|
3472
|
+
}
|
|
3473
|
+
function resetTaskSteers() {
|
|
3474
|
+
delete globalThis[STEERS_KEY];
|
|
3475
|
+
}
|
|
3476
|
+
|
|
2890
3477
|
// src/agent-task/task/loadTask.ts
|
|
2891
3478
|
async function loadTask(taskDir) {
|
|
2892
3479
|
const absoluteDir = resolve2(taskDir);
|
|
@@ -2959,6 +3546,7 @@ async function loadTaskDefinition(taskFilePath) {
|
|
|
2959
3546
|
resetTaskRegistry();
|
|
2960
3547
|
resetFlowChecks();
|
|
2961
3548
|
resetTaskTurn();
|
|
3549
|
+
resetTaskSteers();
|
|
2962
3550
|
const loaded = await import(moduleUrl);
|
|
2963
3551
|
const registeredTask = getRegisteredTask();
|
|
2964
3552
|
const task2 = registeredTask ?? loaded.default;
|
|
@@ -5167,6 +5755,7 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5167
5755
|
const { task: task2, adapter, files, checksPath, inlineChecks } = loaded;
|
|
5168
5756
|
if (!inlineChecks && checksPath) {
|
|
5169
5757
|
resetTaskTurn();
|
|
5758
|
+
resetTaskSteers();
|
|
5170
5759
|
resetFlowChecks();
|
|
5171
5760
|
await loadChecksModule(checksPath);
|
|
5172
5761
|
}
|
|
@@ -5174,14 +5763,32 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5174
5763
|
resetTaskTurn();
|
|
5175
5764
|
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
5176
5765
|
if (!turnFn) return [];
|
|
5766
|
+
const steerSpecs = getTaskSteers();
|
|
5767
|
+
resetTaskSteers();
|
|
5768
|
+
if (steerSpecs.length > 0 && typeof session.steer !== "function") {
|
|
5769
|
+
throw new AgentTaskRunError(
|
|
5770
|
+
`Task "${task2.id}" registers ${steerSpecs.length} steer(s) but adapter "${adapter.name}" does not implement session.steer(). Implement steer() on the adapter session, or remove the steer() registrations.`,
|
|
5771
|
+
{}
|
|
5772
|
+
);
|
|
5773
|
+
}
|
|
5774
|
+
const turnSpanIds = /* @__PURE__ */ new Map();
|
|
5775
|
+
const scheduler = createSteerScheduler({
|
|
5776
|
+
specs: steerSpecs,
|
|
5777
|
+
session,
|
|
5778
|
+
trace: trace2,
|
|
5779
|
+
turnSpanId: (turnNumber) => turnSpanIds.get(turnNumber),
|
|
5780
|
+
onSteer: options?.onSteer
|
|
5781
|
+
});
|
|
5177
5782
|
const taskFiles = new TaskFiles(files);
|
|
5178
5783
|
const turnTranscript = [];
|
|
5179
5784
|
const transcriptTurns = [];
|
|
5180
5785
|
let lastTurnResponse;
|
|
5786
|
+
let lastTurnNumber = 0;
|
|
5181
5787
|
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
5182
5788
|
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
5183
5789
|
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
5184
5790
|
if (userTurn === null || userTurn === void 0) break;
|
|
5791
|
+
await scheduler.onTurnStart(turnNum);
|
|
5185
5792
|
const result = await trace2.step(
|
|
5186
5793
|
{
|
|
5187
5794
|
step_name: "task.turn",
|
|
@@ -5192,15 +5799,29 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5192
5799
|
// processor captures parent under the turn span, not the run root —
|
|
5193
5800
|
// turn-scoped assertions (t.maxTokens({ turn })) walk that chain.
|
|
5194
5801
|
async (spanId) => {
|
|
5802
|
+
turnSpanIds.set(turnNum, spanId);
|
|
5195
5803
|
const run = getActiveApoRun();
|
|
5196
5804
|
const send = () => session.sendUserTurn(userTurn, {
|
|
5197
5805
|
trace: trace2,
|
|
5198
5806
|
turnNumber: turnNum,
|
|
5199
|
-
parentSpanId: spanId
|
|
5807
|
+
parentSpanId: spanId,
|
|
5808
|
+
// Progress events drive the steer scheduler. Fire-and-forget:
|
|
5809
|
+
// the adapter never waits on steering, and scheduler errors are
|
|
5810
|
+
// contained (they must not break the agent's turn).
|
|
5811
|
+
notifyAgentEvent: (event) => {
|
|
5812
|
+
void scheduler.onProgressEvent(turnNum, event).catch((error) => {
|
|
5813
|
+
console.error(
|
|
5814
|
+
"[AgentTask] Steer scheduler error:",
|
|
5815
|
+
error instanceof Error ? error.message : String(error)
|
|
5816
|
+
);
|
|
5817
|
+
});
|
|
5818
|
+
}
|
|
5200
5819
|
});
|
|
5201
5820
|
return run ? withApoRun({ ...run, parentSpanId: spanId, turnNumber: turnNum }, send) : send();
|
|
5202
5821
|
}
|
|
5203
5822
|
);
|
|
5823
|
+
await scheduler.onTurnEnd(turnNum);
|
|
5824
|
+
const turnSteers = scheduler.recordsByTurn().get(turnNum);
|
|
5204
5825
|
turnTranscript.push({
|
|
5205
5826
|
turnNumber: turnNum,
|
|
5206
5827
|
input: userTurn,
|
|
@@ -5209,11 +5830,14 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
|
5209
5830
|
transcriptTurns.push({
|
|
5210
5831
|
turnNumber: turnNum,
|
|
5211
5832
|
userAction: userTurn,
|
|
5212
|
-
agentResponse: result.response
|
|
5833
|
+
agentResponse: result.response,
|
|
5834
|
+
...turnSteers !== void 0 && turnSteers.length > 0 ? { steers: turnSteers } : {}
|
|
5213
5835
|
});
|
|
5214
5836
|
options?.onTurn?.(turnNum, userTurn, result.response);
|
|
5215
5837
|
lastTurnResponse = result.response;
|
|
5838
|
+
lastTurnNumber = turnNum;
|
|
5216
5839
|
}
|
|
5840
|
+
await scheduler.onRunEnd(lastTurnNumber);
|
|
5217
5841
|
if (lastTurnResponse !== void 0) {
|
|
5218
5842
|
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
5219
5843
|
}
|
|
@@ -6079,7 +6703,12 @@ function resolveJudgeFromEnv() {
|
|
|
6079
6703
|
return {
|
|
6080
6704
|
model,
|
|
6081
6705
|
baseURL: openRouterModel ? process.env.OPENROUTER_BASE_URL : process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1",
|
|
6082
|
-
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : process.env.OPENAI_API_KEY
|
|
6706
|
+
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : process.env.OPENAI_API_KEY,
|
|
6707
|
+
// "cascade" turns on second-judge-first judging (a confident decision
|
|
6708
|
+
// model verdict stands without calling the primary judge); any other
|
|
6709
|
+
// value is ignored so a typo degrades to the default dual mode, never
|
|
6710
|
+
// an error.
|
|
6711
|
+
...process.env.APO_JUDGE_MODE?.trim() === "cascade" ? { mode: "cascade" } : {}
|
|
6083
6712
|
};
|
|
6084
6713
|
}
|
|
6085
6714
|
async function runTaskDir(taskDir, options) {
|
|
@@ -6353,6 +6982,7 @@ export {
|
|
|
6353
6982
|
ApoSpanProcessor,
|
|
6354
6983
|
FlowView,
|
|
6355
6984
|
JUDGE_SEGMENT_LIMIT,
|
|
6985
|
+
NO_CHECKS_REGISTERED_MESSAGE,
|
|
6356
6986
|
RECEIVED_VALUE_LIMIT,
|
|
6357
6987
|
RejudgeError,
|
|
6358
6988
|
TEST_METHOD_NAMES,
|
|
@@ -6381,6 +7011,7 @@ export {
|
|
|
6381
7011
|
fromAnthropicMessages,
|
|
6382
7012
|
fromOpenAIMessages,
|
|
6383
7013
|
getActiveApoRun,
|
|
7014
|
+
getTaskSteers,
|
|
6384
7015
|
getTaskTurn,
|
|
6385
7016
|
includes,
|
|
6386
7017
|
isFileArtifact,
|
|
@@ -6399,6 +7030,7 @@ export {
|
|
|
6399
7030
|
resetApoTracing,
|
|
6400
7031
|
resetFlowChecks,
|
|
6401
7032
|
resetTaskRegistry,
|
|
7033
|
+
resetTaskSteers,
|
|
6402
7034
|
resetTaskTurn,
|
|
6403
7035
|
resolveMcpServerPaths,
|
|
6404
7036
|
resolveOtlpTracesUrl,
|
|
@@ -6407,8 +7039,11 @@ export {
|
|
|
6407
7039
|
runTask,
|
|
6408
7040
|
runTaskDir,
|
|
6409
7041
|
satisfies,
|
|
7042
|
+
secondJudgeFacts,
|
|
7043
|
+
secondJudgeTakeaway,
|
|
6410
7044
|
sha256Hex2 as sha256Hex,
|
|
6411
7045
|
similarity,
|
|
7046
|
+
steer,
|
|
6412
7047
|
task,
|
|
6413
7048
|
defineCheck as test,
|
|
6414
7049
|
transcriptSessionToOtlp,
|