@apo-ai/sdk 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/agent-task/integrations/ai-sdk.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.js +2 -2
- package/dist/agent-task/integrations/openai.d.ts +4 -1
- package/dist/agent-task/integrations/openai.js +2 -2
- package/dist/agent-task/public.d.ts +17 -1
- package/dist/agent-task/public.js +238 -52
- package/dist/agent-task/public.js.map +1 -1
- package/dist/{chunk-X73JGMK3.js → chunk-4BQSCD6T.js} +2 -2
- package/dist/{chunk-JTWQD27M.js → chunk-JN7RHBTS.js} +36 -3
- package/dist/chunk-JN7RHBTS.js.map +1 -0
- package/dist/{chunk-A3UYQ7M4.js → chunk-NSMKSPBU.js} +20 -3
- package/dist/chunk-NSMKSPBU.js.map +1 -0
- package/dist/{tracing-FqFObrmt.d.ts → tracing-CEucJqQI.d.ts} +22 -0
- package/package.json +2 -1
- package/dist/chunk-A3UYQ7M4.js.map +0 -1
- package/dist/chunk-JTWQD27M.js.map +0 -1
- /package/dist/{chunk-X73JGMK3.js.map → chunk-4BQSCD6T.js.map} +0 -0
package/README.md
CHANGED
|
@@ -250,8 +250,11 @@ For apo tasks the flow is built automatically from the run's trace
|
|
|
250
250
|
projection — no extra wiring.
|
|
251
251
|
|
|
252
252
|
**`t.judge(value, instruction)` — LLM-backed assertions.** Configure the judge
|
|
253
|
-
with `runTask(dir, { judge: { model, apiKey?, baseURL? } })`, or set
|
|
254
|
-
`OPENROUTER_MODEL`/`OPENAI_MODEL` for the CLI runtime.
|
|
253
|
+
with `runTask(dir, { judge: { model, apiKey?, baseURL?, provider? } })`, or set
|
|
254
|
+
`OPENROUTER_MODEL`/`OPENAI_MODEL`/`ANTHROPIC_MODEL` for the CLI runtime. A bare
|
|
255
|
+
`claude-*` model id (or an Anthropic base URL) switches the call to the
|
|
256
|
+
Anthropic Messages wire, authenticated with `ANTHROPIC_API_KEY` or
|
|
257
|
+
`ANTHROPIC_AUTH_TOKEN`. Judge verdicts use the
|
|
255
258
|
same recorder and result format as code assertions, including model, prompt,
|
|
256
259
|
response, token usage, and latency metadata.
|
|
257
260
|
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Do not edit by hand — edit the src/ originals and rebuild.
|
|
3
3
|
import {
|
|
4
4
|
createApoAnthropic
|
|
5
|
-
} from "../../chunk-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
} from "../../chunk-4BQSCD6T.js";
|
|
6
|
+
import "../../chunk-JN7RHBTS.js";
|
|
7
7
|
export {
|
|
8
8
|
createApoAnthropic
|
|
9
9
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as AgentTaskTraceContext } from '../../tracing-
|
|
1
|
+
import { A as AgentTaskTraceContext } from '../../tracing-CEucJqQI.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* OpenAI JS SDK tracing integration for apo.
|
|
@@ -56,6 +56,9 @@ interface OpenAIClientLike {
|
|
|
56
56
|
chat: {
|
|
57
57
|
completions: {
|
|
58
58
|
create: (...args: any[]) => Promise<any>;
|
|
59
|
+
withRawResponse?: {
|
|
60
|
+
create: (...args: any[]) => Promise<any>;
|
|
61
|
+
};
|
|
59
62
|
};
|
|
60
63
|
};
|
|
61
64
|
[key: string]: any;
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Do not edit by hand — edit the src/ originals and rebuild.
|
|
3
3
|
import {
|
|
4
4
|
createApoOpenAI
|
|
5
|
-
} from "../../chunk-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
} from "../../chunk-NSMKSPBU.js";
|
|
6
|
+
import "../../chunk-JN7RHBTS.js";
|
|
7
7
|
export {
|
|
8
8
|
createApoOpenAI
|
|
9
9
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-
|
|
1
|
+
import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-CEucJqQI.js';
|
|
2
2
|
export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
|
|
3
3
|
export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
|
|
4
4
|
export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
|
|
@@ -83,6 +83,7 @@ interface TraceProjectionObservation {
|
|
|
83
83
|
errorMessage?: string;
|
|
84
84
|
input?: unknown;
|
|
85
85
|
output?: unknown;
|
|
86
|
+
/** The model the caller requested (often a gateway alias). */
|
|
86
87
|
model?: string;
|
|
87
88
|
toolName?: string;
|
|
88
89
|
toolParameters?: unknown;
|
|
@@ -615,6 +616,12 @@ type AssertionOutcome = "pass" | "fail" | "unsupported" | "error";
|
|
|
615
616
|
type JudgeMetadata = {
|
|
616
617
|
/** Model identifier, e.g. ``"deepseek/deepseek-v4.1-flash"``. */
|
|
617
618
|
model?: string;
|
|
619
|
+
/**
|
|
620
|
+
* The model the provider reports as having served the judge call (the
|
|
621
|
+
* response's `model` field). Diverges from `model` when the judge's
|
|
622
|
+
* gateway fell back — judge spend and attribution key on this.
|
|
623
|
+
*/
|
|
624
|
+
served_model?: string;
|
|
618
625
|
/**
|
|
619
626
|
* Which response contract elicited this judgment (#163): reasoning-first
|
|
620
627
|
* (the default) or the legacy verdict-first (``pass`` before
|
|
@@ -1004,6 +1011,8 @@ declare function similarity(expected: string, threshold?: number): Matcher<unkno
|
|
|
1004
1011
|
* LLM-as-judge call. Used by `t.judge(values, instruction)` to evaluate
|
|
1005
1012
|
* deliverables against a natural-language rubric. Calls an OpenAI-compatible
|
|
1006
1013
|
* endpoint (OpenRouter, OpenAI, etc.) via fetch and parses the verdict.
|
|
1014
|
+
* Anthropic-direct (`api.anthropic.com` Messages API) speaks its own wire —
|
|
1015
|
+
* see {@link resolveJudgeWire}.
|
|
1007
1016
|
*/
|
|
1008
1017
|
|
|
1009
1018
|
/**
|
|
@@ -1294,6 +1303,13 @@ type JudgeConfig = {
|
|
|
1294
1303
|
baseURL?: string;
|
|
1295
1304
|
/** Override the API key (defaults to env). */
|
|
1296
1305
|
apiKey?: string;
|
|
1306
|
+
/**
|
|
1307
|
+
* Force the Anthropic Messages wire (`api.anthropic.com/v1/messages`) for a
|
|
1308
|
+
* base URL whose hostname does not identify it — an Anthropic-protocol
|
|
1309
|
+
* gateway. Unset = auto-detection (Anthropic host, or a bare `claude-*`
|
|
1310
|
+
* model id); see `resolveJudgeWire` in judge.ts.
|
|
1311
|
+
*/
|
|
1312
|
+
provider?: "anthropic";
|
|
1297
1313
|
/**
|
|
1298
1314
|
* Build a custom judge briefing from the threaded context (#161). The SDK
|
|
1299
1315
|
* keeps ownership of the JSON response contract — see
|
|
@@ -16,13 +16,13 @@ import {
|
|
|
16
16
|
import "../chunk-HRYXLRXJ.js";
|
|
17
17
|
import {
|
|
18
18
|
createApoOpenAI
|
|
19
|
-
} from "../chunk-
|
|
19
|
+
} from "../chunk-NSMKSPBU.js";
|
|
20
20
|
import {
|
|
21
21
|
createApoAnthropic
|
|
22
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-4BQSCD6T.js";
|
|
23
23
|
import {
|
|
24
24
|
servingHostFromBaseURL
|
|
25
|
-
} from "../chunk-
|
|
25
|
+
} from "../chunk-JN7RHBTS.js";
|
|
26
26
|
|
|
27
27
|
// src/agent-task/run/steer-scheduler.ts
|
|
28
28
|
var TASK_STEER_SPAN_NAME = "task.steer";
|
|
@@ -691,7 +691,7 @@ var RepetitionGuard = class {
|
|
|
691
691
|
this.tail = "";
|
|
692
692
|
this.lastChar = "";
|
|
693
693
|
this.charRun = 0;
|
|
694
|
-
this.periodRuns =
|
|
694
|
+
this.periodRuns = Array.from({ length: MAX_UNIT_CHARS + 1 }, () => 0);
|
|
695
695
|
// Low-diversity state: the current raw line, and the folded window built
|
|
696
696
|
// from finished lines that are not tables.
|
|
697
697
|
this.line = "";
|
|
@@ -872,9 +872,19 @@ var openAIProvider = {
|
|
|
872
872
|
apiRootExample: "https://api.openai.com/v1",
|
|
873
873
|
reserved: "The OpenAI Decisions API provider is not implemented yet: the endpoint is limited-preview and its schema is unpublished. " + OPENROUTER_RESERVED_HINT + "."
|
|
874
874
|
};
|
|
875
|
+
var anthropicProvider = {
|
|
876
|
+
decisionsURL: (chatBaseURL) => `${chatBaseURL.replace(/\/v1\/?$/, "")}/decisions`,
|
|
877
|
+
requestBody: () => {
|
|
878
|
+
throw new Error("unreachable \u2014 reserved guard returns before the call");
|
|
879
|
+
},
|
|
880
|
+
parseResponse: () => ({ error: "unreachable \u2014 reserved guard returns before the call" }),
|
|
881
|
+
apiRootExample: "https://api.anthropic.com",
|
|
882
|
+
reserved: "Anthropic-direct serves no decisions API \u2014 the second judge cannot run there. " + OPENROUTER_RESERVED_HINT + "."
|
|
883
|
+
};
|
|
875
884
|
var PROVIDERS = {
|
|
876
885
|
openrouter: openRouterProvider,
|
|
877
|
-
openai: openAIProvider
|
|
886
|
+
openai: openAIProvider,
|
|
887
|
+
anthropic: anthropicProvider
|
|
878
888
|
};
|
|
879
889
|
function resolveSecondJudgeModel() {
|
|
880
890
|
const value = process.env.APO_SECOND_JUDGE_MODEL?.trim();
|
|
@@ -900,7 +910,10 @@ function resolveSecondJudgeProvider(baseURL) {
|
|
|
900
910
|
}
|
|
901
911
|
return provider;
|
|
902
912
|
}
|
|
903
|
-
|
|
913
|
+
const hostname = new URL(baseURL).hostname;
|
|
914
|
+
if (/(^|\.)api\.openai\.com$/.test(hostname)) return openAIProvider;
|
|
915
|
+
if (/(^|\.)api\.anthropic\.com$/.test(hostname)) return anthropicProvider;
|
|
916
|
+
return openRouterProvider;
|
|
904
917
|
}
|
|
905
918
|
var INPUT_TOO_LARGE_MARKERS = [
|
|
906
919
|
"max_tokens_exceeded",
|
|
@@ -1049,10 +1062,12 @@ ${formatValue(v)}`).join("\n\n");
|
|
|
1049
1062
|
}
|
|
1050
1063
|
function parseJudgeUsage(usage) {
|
|
1051
1064
|
if (!usage) return void 0;
|
|
1065
|
+
const anthropicShape = usage.cache_creation_input_tokens !== void 0 || usage.cache_read_input_tokens !== void 0;
|
|
1052
1066
|
const cacheCreation = usage.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_write_tokens;
|
|
1053
1067
|
const cacheRead = usage.cache_read_input_tokens ?? usage.prompt_tokens_details?.cached_tokens;
|
|
1068
|
+
const prompt = usage.prompt_tokens ?? 0;
|
|
1054
1069
|
const tokens = {
|
|
1055
|
-
input:
|
|
1070
|
+
input: anthropicShape ? prompt + (cacheCreation ?? 0) + (cacheRead ?? 0) : prompt,
|
|
1056
1071
|
output: usage.completion_tokens ?? 0
|
|
1057
1072
|
};
|
|
1058
1073
|
if (typeof cacheCreation === "number") tokens.cache_creation = cacheCreation;
|
|
@@ -1089,6 +1104,7 @@ function judgeVerdict(raw) {
|
|
|
1089
1104
|
};
|
|
1090
1105
|
}
|
|
1091
1106
|
var JUDGE_IDLE_TIMEOUT_MS = 9e4;
|
|
1107
|
+
var ANTHROPIC_JUDGE_MAX_TOKENS = 8192;
|
|
1092
1108
|
function envMs(name, fallback) {
|
|
1093
1109
|
const raw = process.env[name]?.trim();
|
|
1094
1110
|
if (!raw) return fallback;
|
|
@@ -1121,12 +1137,14 @@ async function readCompletion(response, onData, guards) {
|
|
|
1121
1137
|
return {
|
|
1122
1138
|
text: data.choices?.[0]?.message?.content ?? "",
|
|
1123
1139
|
usage: data.usage,
|
|
1124
|
-
finishReason: data.choices?.[0]?.finish_reason ?? void 0
|
|
1140
|
+
finishReason: data.choices?.[0]?.finish_reason ?? void 0,
|
|
1141
|
+
servedModel: typeof data.model === "string" && data.model ? data.model : void 0
|
|
1125
1142
|
};
|
|
1126
1143
|
}
|
|
1127
1144
|
let text = "";
|
|
1128
1145
|
let usage;
|
|
1129
1146
|
let finishReason;
|
|
1147
|
+
let servedModel;
|
|
1130
1148
|
let eventType = "message";
|
|
1131
1149
|
let dataLines = [];
|
|
1132
1150
|
let finished = false;
|
|
@@ -1162,6 +1180,7 @@ async function readCompletion(response, onData, guards) {
|
|
|
1162
1180
|
}
|
|
1163
1181
|
if (choice?.finish_reason) finishReason = choice.finish_reason;
|
|
1164
1182
|
if (chunk.usage) usage = chunk.usage;
|
|
1183
|
+
if (typeof chunk.model === "string" && chunk.model) servedModel = chunk.model;
|
|
1165
1184
|
};
|
|
1166
1185
|
const handleLine = (line) => {
|
|
1167
1186
|
if (line === "") return dispatch();
|
|
@@ -1174,7 +1193,7 @@ async function readCompletion(response, onData, guards) {
|
|
|
1174
1193
|
else if (field === "event") eventType = value;
|
|
1175
1194
|
};
|
|
1176
1195
|
const reader = response.body?.getReader();
|
|
1177
|
-
if (!reader) return { text, usage, finishReason };
|
|
1196
|
+
if (!reader) return { text, usage, finishReason, servedModel };
|
|
1178
1197
|
const decoder = new TextDecoder();
|
|
1179
1198
|
let buffer = "";
|
|
1180
1199
|
for (; ; ) {
|
|
@@ -1185,13 +1204,29 @@ async function readCompletion(response, onData, guards) {
|
|
|
1185
1204
|
buffer = lines.pop() ?? "";
|
|
1186
1205
|
for (const line of lines) {
|
|
1187
1206
|
handleLine(line);
|
|
1188
|
-
if (finished) return { text, usage, finishReason };
|
|
1207
|
+
if (finished) return { text, usage, finishReason, servedModel };
|
|
1189
1208
|
}
|
|
1190
1209
|
}
|
|
1191
1210
|
buffer += decoder.decode();
|
|
1192
1211
|
for (const line of buffer.split(/\r\n|\r|\n/)) if (line) handleLine(line);
|
|
1193
1212
|
dispatch();
|
|
1194
|
-
return { text, usage, finishReason };
|
|
1213
|
+
return { text, usage, finishReason, servedModel };
|
|
1214
|
+
}
|
|
1215
|
+
async function readAnthropicCompletion(response) {
|
|
1216
|
+
const data = await response.json();
|
|
1217
|
+
const text = (data.content ?? []).filter((block) => block.type === "text" && typeof block.text === "string").map((block) => block.text).join("");
|
|
1218
|
+
const finishReason = data.stop_reason === "max_tokens" ? "length" : data.stop_reason === "refusal" ? "content_filter" : data.stop_reason;
|
|
1219
|
+
return {
|
|
1220
|
+
text,
|
|
1221
|
+
finishReason,
|
|
1222
|
+
servedModel: typeof data.model === "string" && data.model ? data.model : void 0,
|
|
1223
|
+
usage: data.usage ? {
|
|
1224
|
+
prompt_tokens: data.usage.input_tokens,
|
|
1225
|
+
completion_tokens: data.usage.output_tokens,
|
|
1226
|
+
...data.usage.cache_creation_input_tokens !== void 0 ? { cache_creation_input_tokens: data.usage.cache_creation_input_tokens } : {},
|
|
1227
|
+
...data.usage.cache_read_input_tokens !== void 0 ? { cache_read_input_tokens: data.usage.cache_read_input_tokens } : {}
|
|
1228
|
+
} : void 0
|
|
1229
|
+
};
|
|
1195
1230
|
}
|
|
1196
1231
|
var prefixLanes = /* @__PURE__ */ new Map();
|
|
1197
1232
|
function judgeConcurrency() {
|
|
@@ -1262,26 +1297,58 @@ function runWithSharedPrefix(key, task2) {
|
|
|
1262
1297
|
};
|
|
1263
1298
|
return lane.warmed.then(acquire).then(() => task2().finally(release));
|
|
1264
1299
|
}
|
|
1265
|
-
function judgeTemperature(configured) {
|
|
1300
|
+
function judgeTemperature(configured, wire = "openai-compatible") {
|
|
1301
|
+
const max = wire === "anthropic" ? 1 : 2;
|
|
1266
1302
|
const raw = process.env.APO_JUDGE_TEMPERATURE?.trim();
|
|
1267
1303
|
const fromEnv = raw ? Number(raw) : void 0;
|
|
1268
1304
|
const value = configured ?? fromEnv ?? 0;
|
|
1269
|
-
if (!Number.isFinite(value) || value < 0 || value >
|
|
1305
|
+
if (!Number.isFinite(value) || value < 0 || value > max) {
|
|
1270
1306
|
const source = configured !== void 0 ? "judge.temperature" : "APO_JUDGE_TEMPERATURE";
|
|
1271
|
-
throw new Error(`${source} must be a number between 0 and
|
|
1307
|
+
throw new Error(`${source} must be a number between 0 and ${max}, got ${configured ?? raw}`);
|
|
1272
1308
|
}
|
|
1273
1309
|
return value;
|
|
1274
1310
|
}
|
|
1275
|
-
function
|
|
1311
|
+
function isAnthropicHost(baseURL) {
|
|
1312
|
+
try {
|
|
1313
|
+
return /(^|\.)api\.anthropic\.com$/.test(new URL(baseURL).hostname);
|
|
1314
|
+
} catch {
|
|
1315
|
+
return false;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
function resolveJudgeWire(opts) {
|
|
1319
|
+
if (opts.provider === "anthropic") return "anthropic";
|
|
1320
|
+
if (opts.baseURL !== void 0 && isAnthropicHost(opts.baseURL)) return "anthropic";
|
|
1321
|
+
if (opts.baseURL === void 0 && /^claude-[a-z0-9]/.test(opts.model)) return "anthropic";
|
|
1322
|
+
return "openai-compatible";
|
|
1323
|
+
}
|
|
1324
|
+
function anthropicAuth(apiKey) {
|
|
1325
|
+
if (apiKey) return { apiKey, asBearer: false };
|
|
1326
|
+
if (process.env.ANTHROPIC_API_KEY) return { apiKey: process.env.ANTHROPIC_API_KEY, asBearer: false };
|
|
1327
|
+
if (process.env.ANTHROPIC_AUTH_TOKEN) return { apiKey: process.env.ANTHROPIC_AUTH_TOKEN, asBearer: true };
|
|
1328
|
+
return { apiKey: void 0, asBearer: false };
|
|
1329
|
+
}
|
|
1330
|
+
function defaultJudgeBaseURL(baseURL, wire = "openai-compatible") {
|
|
1331
|
+
if (wire === "anthropic") {
|
|
1332
|
+
return baseURL ?? process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com";
|
|
1333
|
+
}
|
|
1276
1334
|
return baseURL ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1";
|
|
1277
1335
|
}
|
|
1278
|
-
function defaultJudgeAPIKey(apiKey) {
|
|
1336
|
+
function defaultJudgeAPIKey(apiKey, wire = "openai-compatible") {
|
|
1337
|
+
if (wire === "anthropic") {
|
|
1338
|
+
return anthropicAuth(apiKey).apiKey;
|
|
1339
|
+
}
|
|
1279
1340
|
return apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
|
|
1280
1341
|
}
|
|
1281
1342
|
async function callJudge(args) {
|
|
1282
|
-
const
|
|
1283
|
-
|
|
1284
|
-
|
|
1343
|
+
const wire = resolveJudgeWire({
|
|
1344
|
+
baseURL: args.baseURL,
|
|
1345
|
+
model: args.model,
|
|
1346
|
+
provider: args.provider
|
|
1347
|
+
});
|
|
1348
|
+
const baseURL = defaultJudgeBaseURL(args.baseURL, wire);
|
|
1349
|
+
const auth = wire === "anthropic" ? anthropicAuth(args.apiKey) : { apiKey: void 0, asBearer: false };
|
|
1350
|
+
const apiKey = wire === "anthropic" ? auth.apiKey : defaultJudgeAPIKey(args.apiKey);
|
|
1351
|
+
const temperature = judgeTemperature(args.temperature, wire);
|
|
1285
1352
|
const { briefingText, instructionText, systemPromptText, deliverableText, secondJudgeState, secondJudgeProjected } = buildJudgePromptParts({
|
|
1286
1353
|
values: args.values,
|
|
1287
1354
|
instruction: args.instruction,
|
|
@@ -1298,6 +1365,21 @@ async function callJudge(args) {
|
|
|
1298
1365
|
...secondJudgeProjected ? { projected: true } : {}
|
|
1299
1366
|
}) : void 0;
|
|
1300
1367
|
const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
|
|
1368
|
+
const anthropicRequestBody = () => JSON.stringify({
|
|
1369
|
+
model: args.model,
|
|
1370
|
+
max_tokens: ANTHROPIC_JUDGE_MAX_TOKENS,
|
|
1371
|
+
temperature,
|
|
1372
|
+
system: [
|
|
1373
|
+
{ type: "text", text: briefingText },
|
|
1374
|
+
{ type: "text", text: deliverableText, cache_control: { type: "ephemeral" } }
|
|
1375
|
+
],
|
|
1376
|
+
messages: [{ role: "user", content: instructionText }]
|
|
1377
|
+
});
|
|
1378
|
+
const anthropicHeaders = {
|
|
1379
|
+
"Content-Type": "application/json",
|
|
1380
|
+
"anthropic-version": "2023-06-01",
|
|
1381
|
+
...apiKey !== void 0 ? auth.asBearer ? { Authorization: `Bearer ${apiKey}` } : { "x-api-key": apiKey } : {}
|
|
1382
|
+
};
|
|
1301
1383
|
const requestBody = (stream, schema) => JSON.stringify({
|
|
1302
1384
|
model: args.model,
|
|
1303
1385
|
prompt_cache_key: promptCacheKey(cacheKey),
|
|
@@ -1348,7 +1430,16 @@ async function callJudge(args) {
|
|
|
1348
1430
|
try {
|
|
1349
1431
|
let response;
|
|
1350
1432
|
try {
|
|
1351
|
-
response =
|
|
1433
|
+
response = wire === "anthropic" ? (
|
|
1434
|
+
// Plain fetch: the Anthropic body never carries prompt_cache_key,
|
|
1435
|
+
// so the resend wrapper would be a pass-through.
|
|
1436
|
+
await fetch(`${baseURL}/v1/messages`, {
|
|
1437
|
+
method: "POST",
|
|
1438
|
+
signal: controller.signal,
|
|
1439
|
+
headers: anthropicHeaders,
|
|
1440
|
+
body: anthropicRequestBody()
|
|
1441
|
+
})
|
|
1442
|
+
) : await fetchWithPromptCacheKey(`${baseURL}/chat/completions`, {
|
|
1352
1443
|
method: "POST",
|
|
1353
1444
|
signal: controller.signal,
|
|
1354
1445
|
headers: {
|
|
@@ -1378,7 +1469,10 @@ async function callJudge(args) {
|
|
|
1378
1469
|
content: new RepetitionGuard({ diversity: false })
|
|
1379
1470
|
} : void 0;
|
|
1380
1471
|
try {
|
|
1381
|
-
return {
|
|
1472
|
+
return {
|
|
1473
|
+
kind: "completion",
|
|
1474
|
+
completion: wire === "anthropic" ? await readAnthropicCompletion(response) : await readCompletion(response, onData, guards)
|
|
1475
|
+
};
|
|
1382
1476
|
} catch (error) {
|
|
1383
1477
|
if (error instanceof JudgeReasoningLoopError) {
|
|
1384
1478
|
return { kind: "unavailable", error, retryable: true, loop: true };
|
|
@@ -1438,6 +1532,9 @@ async function callJudge(args) {
|
|
|
1438
1532
|
}
|
|
1439
1533
|
const judgeMetadata = async (completion) => ({
|
|
1440
1534
|
model: args.model,
|
|
1535
|
+
// The model that actually judged, when the gateway fell back to a
|
|
1536
|
+
// different one than requested.
|
|
1537
|
+
...completion?.servedModel && completion.servedModel !== args.model ? { served_model: completion.servedModel } : {},
|
|
1441
1538
|
temperature,
|
|
1442
1539
|
contract: judgeContractInUse(),
|
|
1443
1540
|
prompt: { system: systemPromptText, user: instructionText },
|
|
@@ -1555,11 +1652,16 @@ async function cascadePreflight(args) {
|
|
|
1555
1652
|
...args.context ? { context: args.context } : {},
|
|
1556
1653
|
...args.secondJudgeValue !== void 0 ? { secondJudgeValue: args.secondJudgeValue } : {}
|
|
1557
1654
|
});
|
|
1655
|
+
const wire = resolveJudgeWire({
|
|
1656
|
+
baseURL: args.effective.baseURL,
|
|
1657
|
+
model: args.effective.model,
|
|
1658
|
+
provider: args.effective.provider
|
|
1659
|
+
});
|
|
1558
1660
|
const evidence = await callSecondJudge({
|
|
1559
1661
|
state: parts.secondJudgeState,
|
|
1560
1662
|
model,
|
|
1561
|
-
baseURL: resolveSecondJudgeBaseURL(defaultJudgeBaseURL(args.effective.baseURL)),
|
|
1562
|
-
apiKey: resolveSecondJudgeAPIKey(defaultJudgeAPIKey(args.effective.apiKey)),
|
|
1663
|
+
baseURL: resolveSecondJudgeBaseURL(defaultJudgeBaseURL(args.effective.baseURL, wire)),
|
|
1664
|
+
apiKey: resolveSecondJudgeAPIKey(defaultJudgeAPIKey(args.effective.apiKey, wire)),
|
|
1563
1665
|
...parts.secondJudgeProjected ? { projected: true } : {}
|
|
1564
1666
|
});
|
|
1565
1667
|
return { kind: "ran", evidence, parts };
|
|
@@ -1962,6 +2064,7 @@ var DEFAULT_BUDGET = {
|
|
|
1962
2064
|
timeoutMs: 3e5,
|
|
1963
2065
|
maxReadBytes: 2 * 1024 * 1024
|
|
1964
2066
|
};
|
|
2067
|
+
var ANTHROPIC_AGENT_MAX_TOKENS = 8192;
|
|
1965
2068
|
var MAX_READ_WINDOW = 12e3;
|
|
1966
2069
|
var SEARCH_HITS = 8;
|
|
1967
2070
|
var SEARCH_CONTEXT = 400;
|
|
@@ -2297,29 +2400,50 @@ async function runAgentSession(spec) {
|
|
|
2297
2400
|
const budget = { ...DEFAULT_BUDGET, ...spec.budget };
|
|
2298
2401
|
const started = Date.now();
|
|
2299
2402
|
const ai = await import("ai");
|
|
2300
|
-
const { createOpenAICompatible } = await import("@ai-sdk/openai-compatible");
|
|
2301
2403
|
const { z } = await import("zod");
|
|
2302
2404
|
const { generateText } = ai;
|
|
2303
2405
|
const stepCountIs = (n) => ai.stepCountIs(n);
|
|
2304
2406
|
const tool = (def) => ai.tool(def);
|
|
2305
|
-
const
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
apiKey: spec.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY,
|
|
2310
|
-
// Mark the briefing prefix cacheable — repeated samples (rejudge) re-bill
|
|
2311
|
-
// nothing for the shared turn-0 context — and keep every step of one
|
|
2312
|
-
// session on the replica holding its growing prefix. The key is per
|
|
2313
|
-
// session (briefing + criterion): sessions run concurrently, unlike
|
|
2314
|
-
// t.judge calls, and one key per task would pile them onto one replica.
|
|
2315
|
-
transformRequestBody: (body) => ({
|
|
2316
|
-
...body,
|
|
2317
|
-
prompt_cache_key: promptCacheKey(sessionPrefix(body)),
|
|
2318
|
-
messages: body.messages?.map(
|
|
2319
|
-
(m) => m.role === "system" && typeof m.content === "string" ? { ...m, content: [{ type: "text", text: m.content, cache_control: { type: "ephemeral" } }] } : m
|
|
2320
|
-
)
|
|
2321
|
-
})
|
|
2407
|
+
const wire = resolveJudgeWire({
|
|
2408
|
+
baseURL: spec.baseURL,
|
|
2409
|
+
model: spec.model,
|
|
2410
|
+
provider: spec.provider
|
|
2322
2411
|
});
|
|
2412
|
+
let model;
|
|
2413
|
+
let wireOptions;
|
|
2414
|
+
if (wire === "anthropic") {
|
|
2415
|
+
const { createAnthropic } = await import("@ai-sdk/anthropic");
|
|
2416
|
+
const auth = anthropicAuth(spec.apiKey);
|
|
2417
|
+
const base = (spec.baseURL ?? process.env.ANTHROPIC_BASE_URL)?.replace(/\/+$/, "");
|
|
2418
|
+
const anthropicEngine = createAnthropic({
|
|
2419
|
+
...base ? { baseURL: /\/v\d+$/.test(base) ? base : `${base}/v1` } : {},
|
|
2420
|
+
...auth.apiKey ? auth.asBearer ? { authToken: auth.apiKey } : { apiKey: auth.apiKey } : {}
|
|
2421
|
+
});
|
|
2422
|
+
model = anthropicEngine(spec.model);
|
|
2423
|
+
wireOptions = { maxOutputTokens: ANTHROPIC_AGENT_MAX_TOKENS };
|
|
2424
|
+
} else {
|
|
2425
|
+
const { createOpenAICompatible } = await import("@ai-sdk/openai-compatible");
|
|
2426
|
+
const provider = createOpenAICompatible({
|
|
2427
|
+
name: "openrouter",
|
|
2428
|
+
baseURL: spec.baseURL ?? "https://openrouter.ai/api/v1",
|
|
2429
|
+
fetch: fetchWithPromptCacheKey,
|
|
2430
|
+
apiKey: spec.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY,
|
|
2431
|
+
// Mark the briefing prefix cacheable — repeated samples (rejudge) re-bill
|
|
2432
|
+
// nothing for the shared turn-0 context — and keep every step of one
|
|
2433
|
+
// session on the replica holding its growing prefix. The key is per
|
|
2434
|
+
// session (briefing + criterion): sessions run concurrently, unlike
|
|
2435
|
+
// t.judge calls, and one key per task would pile them onto one replica.
|
|
2436
|
+
transformRequestBody: (body) => ({
|
|
2437
|
+
...body,
|
|
2438
|
+
prompt_cache_key: promptCacheKey(sessionPrefix(body)),
|
|
2439
|
+
messages: body.messages?.map(
|
|
2440
|
+
(m) => m.role === "system" && typeof m.content === "string" ? { ...m, content: [{ type: "text", text: m.content, cache_control: { type: "ephemeral" } }] } : m
|
|
2441
|
+
)
|
|
2442
|
+
})
|
|
2443
|
+
});
|
|
2444
|
+
model = provider(spec.model);
|
|
2445
|
+
wireOptions = {};
|
|
2446
|
+
}
|
|
2323
2447
|
const ledger = { toolCallsUsed: 0, readBytesUsed: 0, manifest: [] };
|
|
2324
2448
|
const budgetGuard = () => {
|
|
2325
2449
|
ledger.toolCallsUsed += 1;
|
|
@@ -2382,8 +2506,17 @@ async function runAgentSession(spec) {
|
|
|
2382
2506
|
try {
|
|
2383
2507
|
try {
|
|
2384
2508
|
const result = await generateText({
|
|
2385
|
-
model
|
|
2386
|
-
|
|
2509
|
+
model,
|
|
2510
|
+
// On the Anthropic wire the briefing's cache breakpoint is a per-part
|
|
2511
|
+
// provider option on the system message (the request-body rewrite the
|
|
2512
|
+
// OpenAI-compatible path uses has no equivalent there).
|
|
2513
|
+
system: wire === "anthropic" ? [{
|
|
2514
|
+
role: "system",
|
|
2515
|
+
content: system,
|
|
2516
|
+
// Message-level providerOptions: the Anthropic engine reads
|
|
2517
|
+
// the cache breakpoint from here, not from content parts.
|
|
2518
|
+
providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }
|
|
2519
|
+
}] : system,
|
|
2387
2520
|
prompt: spec.instruction + exhibitsBlock,
|
|
2388
2521
|
temperature: 0,
|
|
2389
2522
|
toolChoice: "required",
|
|
@@ -2391,7 +2524,8 @@ async function runAgentSession(spec) {
|
|
|
2391
2524
|
tools,
|
|
2392
2525
|
abortSignal: AbortSignal.timeout(budget.timeoutMs),
|
|
2393
2526
|
maxRetries: 2,
|
|
2394
|
-
onStepFinish: onStep
|
|
2527
|
+
onStepFinish: onStep,
|
|
2528
|
+
...wireOptions
|
|
2395
2529
|
});
|
|
2396
2530
|
engineUsage = result.usage;
|
|
2397
2531
|
} catch (error) {
|
|
@@ -2449,7 +2583,7 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer,
|
|
|
2449
2583
|
rec.record(
|
|
2450
2584
|
label,
|
|
2451
2585
|
false,
|
|
2452
|
-
"No judge model configured for t.agent. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 needs a tool-calling-capable model)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
|
|
2586
|
+
"No judge model configured for t.agent. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 needs a tool-calling-capable model)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\n\u2022 ANTHROPIC_MODEL + ANTHROPIC_API_KEY (Anthropic direct \u2014 needs a tool-calling-capable model; or ANTHROPIC_AUTH_TOKEN for plan credits)\nOr pass { judge } to runTask() programmatically.",
|
|
2453
2587
|
{ evaluator_type: "agent", location, outcome: "error" }
|
|
2454
2588
|
);
|
|
2455
2589
|
return;
|
|
@@ -2459,6 +2593,7 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer,
|
|
|
2459
2593
|
model: effective.model,
|
|
2460
2594
|
baseURL: effective.baseURL,
|
|
2461
2595
|
apiKey: effective.apiKey,
|
|
2596
|
+
provider: effective.provider,
|
|
2462
2597
|
evidence: evidence ?? { deliverables: {} },
|
|
2463
2598
|
scope: judgeScope,
|
|
2464
2599
|
exhibits,
|
|
@@ -2552,6 +2687,7 @@ function resolveJudgeConfig(judgeConfig, override) {
|
|
|
2552
2687
|
model,
|
|
2553
2688
|
baseURL: override?.baseURL ?? judgeConfig?.baseURL,
|
|
2554
2689
|
apiKey: override?.apiKey ?? judgeConfig?.apiKey,
|
|
2690
|
+
provider: override?.provider ?? judgeConfig?.provider,
|
|
2555
2691
|
prompt: override?.prompt ?? judgeConfig?.prompt,
|
|
2556
2692
|
mode: override?.mode ?? judgeConfig?.mode,
|
|
2557
2693
|
temperature: override?.temperature ?? judgeConfig?.temperature
|
|
@@ -2657,14 +2793,21 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
2657
2793
|
rec.record(
|
|
2658
2794
|
label,
|
|
2659
2795
|
false,
|
|
2660
|
-
"No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
|
|
2796
|
+
"No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\n\u2022 ANTHROPIC_MODEL + ANTHROPIC_API_KEY (Anthropic direct \u2014 or ANTHROPIC_AUTH_TOKEN for plan credits)\nOr pass { judge } to runTask() programmatically.",
|
|
2661
2797
|
// No judge means no verdict — the same bucket as an unreachable judge.
|
|
2662
2798
|
{ evaluator_type: "llm", location, outcome: "error" }
|
|
2663
2799
|
);
|
|
2664
2800
|
return;
|
|
2665
2801
|
}
|
|
2666
2802
|
try {
|
|
2667
|
-
judgeTemperature(
|
|
2803
|
+
judgeTemperature(
|
|
2804
|
+
effective.temperature,
|
|
2805
|
+
resolveJudgeWire({
|
|
2806
|
+
baseURL: effective.baseURL,
|
|
2807
|
+
model: effective.model,
|
|
2808
|
+
provider: effective.provider
|
|
2809
|
+
})
|
|
2810
|
+
);
|
|
2668
2811
|
} catch (error) {
|
|
2669
2812
|
rec.record(label, false, `judge failed: ${error instanceof Error ? error.message : String(error)}`, {
|
|
2670
2813
|
evaluator_type: "llm",
|
|
@@ -2730,6 +2873,7 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
2730
2873
|
model: effective.model,
|
|
2731
2874
|
baseURL: effective.baseURL,
|
|
2732
2875
|
apiKey: effective.apiKey,
|
|
2876
|
+
provider: effective.provider,
|
|
2733
2877
|
prompt: effective.prompt,
|
|
2734
2878
|
...effective.temperature !== void 0 ? { temperature: effective.temperature } : {},
|
|
2735
2879
|
...context2 ? { context: context2 } : {},
|
|
@@ -2761,8 +2905,21 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
|
2761
2905
|
};
|
|
2762
2906
|
},
|
|
2763
2907
|
usage: (r) => {
|
|
2764
|
-
const
|
|
2765
|
-
|
|
2908
|
+
const judgeMeta = r?.judge;
|
|
2909
|
+
const tokens = judgeMeta?.tokens;
|
|
2910
|
+
return {
|
|
2911
|
+
...tokens ? {
|
|
2912
|
+
prompt_tokens: tokens.input,
|
|
2913
|
+
completion_tokens: tokens.output,
|
|
2914
|
+
// Without these the backend prices every cached
|
|
2915
|
+
// prefix read at the full input rate.
|
|
2916
|
+
...tokens.cache_read !== void 0 ? { cache_read_tokens: tokens.cache_read } : {},
|
|
2917
|
+
...tokens.cache_creation !== void 0 ? { cache_creation_tokens: tokens.cache_creation } : {}
|
|
2918
|
+
} : {},
|
|
2919
|
+
// The model that actually judged — a gateway fallback
|
|
2920
|
+
// diverges from the requested judge model.
|
|
2921
|
+
...judgeMeta?.served_model ? { served_model: judgeMeta.served_model } : {}
|
|
2922
|
+
};
|
|
2766
2923
|
}
|
|
2767
2924
|
},
|
|
2768
2925
|
async (spanId) => {
|
|
@@ -6619,6 +6776,18 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
6619
6776
|
if (params.completion_tokens !== void 0) {
|
|
6620
6777
|
active.span.setAttribute("gen_ai.usage.output_tokens", params.completion_tokens);
|
|
6621
6778
|
}
|
|
6779
|
+
if (params.cache_read_tokens !== void 0 || params.cache_creation_tokens !== void 0) {
|
|
6780
|
+
active.span.setAttribute("apo.usage.input_includes_cache", true);
|
|
6781
|
+
if (params.cache_read_tokens !== void 0) {
|
|
6782
|
+
active.span.setAttribute("gen_ai.usage.cache_read.input_tokens", params.cache_read_tokens);
|
|
6783
|
+
}
|
|
6784
|
+
if (params.cache_creation_tokens !== void 0) {
|
|
6785
|
+
active.span.setAttribute(
|
|
6786
|
+
"gen_ai.usage.cache_creation.input_tokens",
|
|
6787
|
+
params.cache_creation_tokens
|
|
6788
|
+
);
|
|
6789
|
+
}
|
|
6790
|
+
}
|
|
6622
6791
|
if (params.reasoning_tokens !== void 0) {
|
|
6623
6792
|
active.span.setAttribute(
|
|
6624
6793
|
"gen_ai.usage.reasoning.output_tokens",
|
|
@@ -6628,6 +6797,12 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
6628
6797
|
if (params.served_model) {
|
|
6629
6798
|
active.span.setAttribute("gen_ai.response.model", params.served_model);
|
|
6630
6799
|
}
|
|
6800
|
+
if (params.provider) {
|
|
6801
|
+
active.span.setAttribute("gen_ai.provider.name", params.provider);
|
|
6802
|
+
}
|
|
6803
|
+
if (params.route) {
|
|
6804
|
+
active.span.setAttribute("apo.llm.route", params.route);
|
|
6805
|
+
}
|
|
6631
6806
|
if (params.status_message) {
|
|
6632
6807
|
active.span.setAttribute("apo.status_message", params.status_message);
|
|
6633
6808
|
}
|
|
@@ -6961,12 +7136,23 @@ async function loadTaskRuntime(_taskDir) {
|
|
|
6961
7136
|
}
|
|
6962
7137
|
function resolveJudgeFromEnv() {
|
|
6963
7138
|
const openRouterModel = process.env.OPENROUTER_MODEL;
|
|
6964
|
-
const
|
|
7139
|
+
const openAIModel = process.env.OPENAI_MODEL;
|
|
7140
|
+
const anthropicModel = process.env.ANTHROPIC_MODEL;
|
|
7141
|
+
const model = openRouterModel ?? openAIModel ?? anthropicModel;
|
|
6965
7142
|
if (!model) return void 0;
|
|
6966
7143
|
return {
|
|
6967
7144
|
model,
|
|
6968
|
-
baseURL: openRouterModel ? process.env.OPENROUTER_BASE_URL : process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1",
|
|
6969
|
-
|
|
7145
|
+
baseURL: openRouterModel ? process.env.OPENROUTER_BASE_URL : openAIModel ? process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1" : process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com",
|
|
7146
|
+
// ANTHROPIC_AUTH_TOKEN deliberately stays out of apiKey: an explicit
|
|
7147
|
+
// config key is always sent as x-api-key, which 401s for the OAuth-style
|
|
7148
|
+
// token. Leaving apiKey unset lets anthropicAuth pick the token up from
|
|
7149
|
+
// the env and send it as Authorization: Bearer.
|
|
7150
|
+
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : openAIModel ? process.env.OPENAI_API_KEY : process.env.ANTHROPIC_API_KEY,
|
|
7151
|
+
// Anthropic is last in the precedence chain (OPENROUTER > OPENAI >
|
|
7152
|
+
// ANTHROPIC): a workspace that already configures one of the others
|
|
7153
|
+
// keeps its behavior, so the arm can be reached only deliberately.
|
|
7154
|
+
// Reaching this return with neither set means ANTHROPIC_MODEL won.
|
|
7155
|
+
...openRouterModel || openAIModel ? {} : { provider: "anthropic" },
|
|
6970
7156
|
// "cascade" turns on second-judge-first judging (a confident decision
|
|
6971
7157
|
// model verdict stands without calling the primary judge); any other
|
|
6972
7158
|
// value is ignored so a typo degrades to the default dual mode, never
|