agent-tool-forge 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +209 -0
- package/lib/agent-registry.js +170 -0
- package/lib/api-client.js +792 -0
- package/lib/api-loader.js +260 -0
- package/lib/auth.d.ts +25 -0
- package/lib/auth.js +158 -0
- package/lib/checks/check-adapter.js +172 -0
- package/lib/checks/compose.js +42 -0
- package/lib/checks/content-match.js +14 -0
- package/lib/checks/cost-budget.js +11 -0
- package/lib/checks/index.js +18 -0
- package/lib/checks/json-valid.js +15 -0
- package/lib/checks/latency.js +11 -0
- package/lib/checks/length-bounds.js +17 -0
- package/lib/checks/negative-match.js +14 -0
- package/lib/checks/no-hallucinated-numbers.js +63 -0
- package/lib/checks/non-empty.js +34 -0
- package/lib/checks/regex-match.js +12 -0
- package/lib/checks/run-checks.js +84 -0
- package/lib/checks/schema-match.js +26 -0
- package/lib/checks/tool-call-count.js +16 -0
- package/lib/checks/tool-selection.js +34 -0
- package/lib/checks/types.js +45 -0
- package/lib/comparison/compare.js +86 -0
- package/lib/comparison/format.js +104 -0
- package/lib/comparison/index.js +6 -0
- package/lib/comparison/statistics.js +59 -0
- package/lib/comparison/types.js +41 -0
- package/lib/config-schema.js +200 -0
- package/lib/config.d.ts +66 -0
- package/lib/conversation-store.d.ts +77 -0
- package/lib/conversation-store.js +443 -0
- package/lib/db.d.ts +6 -0
- package/lib/db.js +1112 -0
- package/lib/dep-check.js +99 -0
- package/lib/drift-background.js +61 -0
- package/lib/drift-monitor.js +187 -0
- package/lib/eval-runner.js +566 -0
- package/lib/fixtures/fixture-store.js +161 -0
- package/lib/fixtures/index.js +11 -0
- package/lib/forge-engine.js +982 -0
- package/lib/forge-eval-generator.js +417 -0
- package/lib/forge-file-writer.js +386 -0
- package/lib/forge-service-client.js +190 -0
- package/lib/forge-service.d.ts +4 -0
- package/lib/forge-service.js +655 -0
- package/lib/forge-verifier-generator.js +271 -0
- package/lib/handlers/admin.js +151 -0
- package/lib/handlers/agents.js +229 -0
- package/lib/handlers/chat-resume.js +334 -0
- package/lib/handlers/chat-sync.js +320 -0
- package/lib/handlers/chat.js +320 -0
- package/lib/handlers/conversations.js +92 -0
- package/lib/handlers/preferences.js +88 -0
- package/lib/handlers/tools-list.js +58 -0
- package/lib/hitl-engine.d.ts +60 -0
- package/lib/hitl-engine.js +261 -0
- package/lib/http-utils.js +92 -0
- package/lib/index.d.ts +20 -0
- package/lib/index.js +141 -0
- package/lib/init.js +636 -0
- package/lib/manual-entry.js +59 -0
- package/lib/mcp-server.js +252 -0
- package/lib/output-groups.js +54 -0
- package/lib/postgres-store.d.ts +31 -0
- package/lib/postgres-store.js +465 -0
- package/lib/preference-store.d.ts +47 -0
- package/lib/preference-store.js +79 -0
- package/lib/prompt-store.d.ts +42 -0
- package/lib/prompt-store.js +60 -0
- package/lib/rate-limiter.d.ts +30 -0
- package/lib/rate-limiter.js +104 -0
- package/lib/react-engine.d.ts +110 -0
- package/lib/react-engine.js +337 -0
- package/lib/runner/cli.js +156 -0
- package/lib/runner/cost-estimator.js +71 -0
- package/lib/runner/gate.js +46 -0
- package/lib/runner/index.js +165 -0
- package/lib/sidecar.d.ts +83 -0
- package/lib/sidecar.js +161 -0
- package/lib/sse.d.ts +15 -0
- package/lib/sse.js +30 -0
- package/lib/tools-scanner.js +91 -0
- package/lib/tui.js +253 -0
- package/lib/verifier-report.js +78 -0
- package/lib/verifier-runner.js +338 -0
- package/lib/verifier-scanner.js +70 -0
- package/lib/verifier-worker-pool.js +196 -0
- package/lib/views/chat.js +340 -0
- package/lib/views/endpoints.js +203 -0
- package/lib/views/eval-run.js +206 -0
- package/lib/views/forge-agent.js +538 -0
- package/lib/views/forge.js +410 -0
- package/lib/views/main-menu.js +275 -0
- package/lib/views/mediation.js +381 -0
- package/lib/views/model-compare.js +430 -0
- package/lib/views/model-comparison.js +333 -0
- package/lib/views/onboarding.js +470 -0
- package/lib/views/performance.js +237 -0
- package/lib/views/run-evals.js +205 -0
- package/lib/views/settings.js +829 -0
- package/lib/views/tools-evals.js +514 -0
- package/lib/views/verifier-coverage.js +617 -0
- package/lib/workers/verifier-worker.js +52 -0
- package/package.json +123 -0
- package/widget/forge-chat.js +789 -0
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared LLM client — Anthropic + OpenAI.
|
|
3
|
+
* Used by forge-engine, forge-file-writer, forge-eval-generator, forge-verifier-generator, chat.js, eval-runner.js
|
|
4
|
+
*
|
|
5
|
+
* No external dependencies — uses built-in `https` module only.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { request as httpsRequest } from 'https';
|
|
9
|
+
|
|
10
|
+
// ── Transport ──────────────────────────────────────────────────────────────
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Perform an HTTPS POST request.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} hostname - e.g. 'api.anthropic.com'
|
|
16
|
+
* @param {string} path - e.g. '/v1/messages'
|
|
17
|
+
* @param {object} headers - HTTP headers (Content-Type etc.)
|
|
18
|
+
* @param {object} body - Request body (will be JSON-serialised)
|
|
19
|
+
* @param {number} [timeoutMs] - Request timeout in ms (default 60 000)
|
|
20
|
+
* @returns {Promise<{ status: number, body: string }>}
|
|
21
|
+
*/
|
|
22
|
+
export function httpsPost(hostname, path, headers, body, timeoutMs = 60_000) {
|
|
23
|
+
return new Promise((res, rej) => {
|
|
24
|
+
const payload = JSON.stringify(body);
|
|
25
|
+
const req = httpsRequest(
|
|
26
|
+
{
|
|
27
|
+
hostname,
|
|
28
|
+
path,
|
|
29
|
+
method: 'POST',
|
|
30
|
+
headers: { ...headers, 'Content-Length': Buffer.byteLength(payload) }
|
|
31
|
+
},
|
|
32
|
+
(resp) => {
|
|
33
|
+
let data = '';
|
|
34
|
+
resp.on('error', rej);
|
|
35
|
+
resp.on('data', (d) => { data += d; });
|
|
36
|
+
resp.on('end', () => res({ status: resp.statusCode, body: data }));
|
|
37
|
+
}
|
|
38
|
+
);
|
|
39
|
+
req.setTimeout(timeoutMs, () => req.destroy(new Error('API timeout')));
|
|
40
|
+
req.on('error', rej);
|
|
41
|
+
req.write(payload);
|
|
42
|
+
req.end();
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ── Tool format converters ─────────────────────────────────────────────────
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Convert a forge-format tool to Anthropic tool format.
|
|
50
|
+
*
|
|
51
|
+
* @param {{ name: string, description?: string, inputSchema?: object }} t
|
|
52
|
+
* @returns {{ name: string, description: string, input_schema: object }}
|
|
53
|
+
*/
|
|
54
|
+
export function toAnthropicTool(t) {
|
|
55
|
+
return {
|
|
56
|
+
name: t.name,
|
|
57
|
+
description: t.description || '',
|
|
58
|
+
input_schema: t.inputSchema || { type: 'object', properties: {} }
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Convert a forge-format tool to OpenAI tool format.
|
|
64
|
+
*
|
|
65
|
+
* @param {{ name: string, description?: string, inputSchema?: object }} t
|
|
66
|
+
* @returns {{ type: 'function', function: { name: string, description: string, parameters: object } }}
|
|
67
|
+
*/
|
|
68
|
+
export function toOpenAiTool(t) {
|
|
69
|
+
return {
|
|
70
|
+
type: 'function',
|
|
71
|
+
function: {
|
|
72
|
+
name: t.name,
|
|
73
|
+
description: t.description || '',
|
|
74
|
+
parameters: t.inputSchema || { type: 'object', properties: {} }
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ── Unified LLM turn ───────────────────────────────────────────────────────
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Perform a single LLM turn against Anthropic or OpenAI.
|
|
83
|
+
*
|
|
84
|
+
* @param {object} opts
|
|
85
|
+
* @param {'anthropic'|'openai'} opts.provider
|
|
86
|
+
* @param {string} opts.apiKey
|
|
87
|
+
* @param {string} opts.model
|
|
88
|
+
* @param {string} [opts.system] - System prompt (optional)
|
|
89
|
+
* @param {object[]} opts.messages - Provider-format message history
|
|
90
|
+
* @param {object[]} [opts.tools] - Forge-format tools (auto-converted per provider)
|
|
91
|
+
* @param {number} [opts.maxTokens] - Default 4096
|
|
92
|
+
* @param {number} [opts.timeoutMs] - Default 60 000
|
|
93
|
+
* @returns {Promise<{
|
|
94
|
+
* text: string,
|
|
95
|
+
* toolCalls: Array<{ id: string, name: string, input: object }>,
|
|
96
|
+
* rawContent: any,
|
|
97
|
+
* stopReason: string|null,
|
|
98
|
+
* usage: object|null
|
|
99
|
+
* }>}
|
|
100
|
+
*/
|
|
101
|
+
export async function llmTurn({
|
|
102
|
+
provider,
|
|
103
|
+
apiKey,
|
|
104
|
+
model,
|
|
105
|
+
system,
|
|
106
|
+
messages,
|
|
107
|
+
tools = [],
|
|
108
|
+
maxTokens = 4096,
|
|
109
|
+
timeoutMs = 60_000
|
|
110
|
+
}) {
|
|
111
|
+
if (provider === 'anthropic') {
|
|
112
|
+
return _anthropicTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
113
|
+
}
|
|
114
|
+
if (provider === 'openai') {
|
|
115
|
+
return _openaiTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
116
|
+
}
|
|
117
|
+
if (provider === 'google') {
|
|
118
|
+
return _geminiTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
119
|
+
}
|
|
120
|
+
if (provider === 'deepseek') {
|
|
121
|
+
return _deepseekTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
122
|
+
}
|
|
123
|
+
throw new Error(`llmTurn: unknown provider "${provider}". Expected 'anthropic', 'openai', 'google', or 'deepseek'.`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Normalise provider-specific usage objects to a common shape.
|
|
128
|
+
* Anthropic: { input_tokens, output_tokens }
|
|
129
|
+
* OpenAI/DeepSeek/Gemini-compat: { prompt_tokens, completion_tokens }
|
|
130
|
+
*
|
|
131
|
+
* @param {object|null} usage - Raw usage object from API response
|
|
132
|
+
* @param {string} provider
|
|
133
|
+
* @returns {{ inputTokens: number, outputTokens: number }}
|
|
134
|
+
*/
|
|
135
|
+
export function normalizeUsage(usage, provider) {
|
|
136
|
+
if (!usage) return { inputTokens: 0, outputTokens: 0 };
|
|
137
|
+
if (provider === 'anthropic') {
|
|
138
|
+
return {
|
|
139
|
+
inputTokens: usage.input_tokens || 0,
|
|
140
|
+
outputTokens: usage.output_tokens || 0
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
// OpenAI-compatible (openai, google, deepseek)
|
|
144
|
+
return {
|
|
145
|
+
inputTokens: usage.prompt_tokens || 0,
|
|
146
|
+
outputTokens: usage.completion_tokens || 0
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ── Internal: Anthropic ────────────────────────────────────────────────────
|
|
151
|
+
|
|
152
|
+
async function _anthropicTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
153
|
+
const body = {
|
|
154
|
+
model,
|
|
155
|
+
max_tokens: maxTokens,
|
|
156
|
+
...(system ? { system } : {}),
|
|
157
|
+
messages,
|
|
158
|
+
...(tools.length ? { tools: tools.map(toAnthropicTool) } : {})
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
const raw = await httpsPost(
|
|
162
|
+
'api.anthropic.com',
|
|
163
|
+
'/v1/messages',
|
|
164
|
+
{
|
|
165
|
+
'Content-Type': 'application/json',
|
|
166
|
+
'anthropic-version': '2023-06-01',
|
|
167
|
+
'x-api-key': apiKey
|
|
168
|
+
},
|
|
169
|
+
body,
|
|
170
|
+
timeoutMs
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
let data;
|
|
174
|
+
try {
|
|
175
|
+
data = JSON.parse(raw.body);
|
|
176
|
+
} catch (_) {
|
|
177
|
+
throw new Error(
|
|
178
|
+
`Anthropic API returned non-JSON (status ${raw.status}): ${raw.body.slice(0, 120)}`
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (data.error) throw new Error(`Anthropic API: ${data.error.message}`);
|
|
183
|
+
|
|
184
|
+
const content = data.content || [];
|
|
185
|
+
const textBlocks = content.filter((b) => b.type === 'text');
|
|
186
|
+
const toolUseBlocks = content.filter((b) => b.type === 'tool_use');
|
|
187
|
+
|
|
188
|
+
return {
|
|
189
|
+
text: textBlocks.map((b) => b.text).join('\n'),
|
|
190
|
+
toolCalls: toolUseBlocks.map((b) => ({ id: b.id, name: b.name, input: b.input })),
|
|
191
|
+
rawContent: content,
|
|
192
|
+
stopReason: data.stop_reason ?? null,
|
|
193
|
+
usage: data.usage ?? null
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// ── Internal: OpenAI ───────────────────────────────────────────────────────
|
|
198
|
+
|
|
199
|
+
async function _openaiTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
200
|
+
const msgs = system
|
|
201
|
+
? [{ role: 'system', content: system }, ...messages]
|
|
202
|
+
: [...messages];
|
|
203
|
+
|
|
204
|
+
const body = {
|
|
205
|
+
model,
|
|
206
|
+
max_tokens: maxTokens,
|
|
207
|
+
messages: msgs,
|
|
208
|
+
...(tools.length ? { tools: tools.map(toOpenAiTool), tool_choice: 'auto' } : {})
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
const raw = await httpsPost(
|
|
212
|
+
'api.openai.com',
|
|
213
|
+
'/v1/chat/completions',
|
|
214
|
+
{
|
|
215
|
+
'Content-Type': 'application/json',
|
|
216
|
+
'Authorization': `Bearer ${apiKey}`
|
|
217
|
+
},
|
|
218
|
+
body,
|
|
219
|
+
timeoutMs
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
let data;
|
|
223
|
+
try {
|
|
224
|
+
data = JSON.parse(raw.body);
|
|
225
|
+
} catch (_) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`OpenAI API returned non-JSON (status ${raw.status}): ${raw.body.slice(0, 120)}`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (data.error) throw new Error(`OpenAI API: ${data.error.message}`);
|
|
232
|
+
|
|
233
|
+
const msg = data.choices?.[0]?.message || {};
|
|
234
|
+
const toolCalls = (msg.tool_calls || []).map((tc) => ({
|
|
235
|
+
id: tc.id,
|
|
236
|
+
name: tc.function?.name,
|
|
237
|
+
input: (() => {
|
|
238
|
+
try { return JSON.parse(tc.function?.arguments || '{}'); } catch (_) { return {}; }
|
|
239
|
+
})()
|
|
240
|
+
}));
|
|
241
|
+
|
|
242
|
+
return {
|
|
243
|
+
text: msg.content || '',
|
|
244
|
+
toolCalls,
|
|
245
|
+
rawContent: msg,
|
|
246
|
+
stopReason: data.choices?.[0]?.finish_reason ?? null,
|
|
247
|
+
usage: data.usage ?? null
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ── Internal: Google Gemini (OpenAI-compatible endpoint) ──────────────────
|
|
252
|
+
|
|
253
|
+
async function _geminiTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
254
|
+
// Gemini exposes an OpenAI-compatible endpoint — same JSON shape, different host + auth
|
|
255
|
+
const msgs = system
|
|
256
|
+
? [{ role: 'system', content: system }, ...messages]
|
|
257
|
+
: [...messages];
|
|
258
|
+
|
|
259
|
+
const body = {
|
|
260
|
+
model,
|
|
261
|
+
max_tokens: maxTokens,
|
|
262
|
+
messages: msgs,
|
|
263
|
+
...(tools.length ? { tools: tools.map(toOpenAiTool), tool_choice: 'auto' } : {})
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
const raw = await httpsPost(
|
|
267
|
+
'generativelanguage.googleapis.com',
|
|
268
|
+
'/v1beta/openai/chat/completions',
|
|
269
|
+
{
|
|
270
|
+
'Content-Type': 'application/json',
|
|
271
|
+
'Authorization': `Bearer ${apiKey}`
|
|
272
|
+
},
|
|
273
|
+
body,
|
|
274
|
+
timeoutMs
|
|
275
|
+
);
|
|
276
|
+
|
|
277
|
+
let data;
|
|
278
|
+
try {
|
|
279
|
+
data = JSON.parse(raw.body);
|
|
280
|
+
} catch (_) {
|
|
281
|
+
throw new Error(
|
|
282
|
+
`Gemini API returned non-JSON (status ${raw.status}): ${raw.body.slice(0, 120)}`
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (data.error) throw new Error(`Gemini API: ${data.error.message || JSON.stringify(data.error)}`);
|
|
287
|
+
|
|
288
|
+
const msg = data.choices?.[0]?.message || {};
|
|
289
|
+
const toolCalls = (msg.tool_calls || []).map((tc) => ({
|
|
290
|
+
id: tc.id,
|
|
291
|
+
name: tc.function?.name,
|
|
292
|
+
input: (() => {
|
|
293
|
+
try { return JSON.parse(tc.function?.arguments || '{}'); } catch (_) { return {}; }
|
|
294
|
+
})()
|
|
295
|
+
}));
|
|
296
|
+
|
|
297
|
+
return {
|
|
298
|
+
text: msg.content || '',
|
|
299
|
+
toolCalls,
|
|
300
|
+
rawContent: msg,
|
|
301
|
+
stopReason: data.choices?.[0]?.finish_reason ?? null,
|
|
302
|
+
usage: data.usage ?? null
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// ── Internal: DeepSeek (OpenAI-compatible) ────────────────────────────────
|
|
307
|
+
|
|
308
|
+
async function _deepseekTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
309
|
+
const msgs = system
|
|
310
|
+
? [{ role: 'system', content: system }, ...messages]
|
|
311
|
+
: [...messages];
|
|
312
|
+
|
|
313
|
+
const body = {
|
|
314
|
+
model,
|
|
315
|
+
max_tokens: maxTokens,
|
|
316
|
+
messages: msgs,
|
|
317
|
+
...(tools.length ? { tools: tools.map(toOpenAiTool), tool_choice: 'auto' } : {})
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
const raw = await httpsPost(
|
|
321
|
+
'api.deepseek.com',
|
|
322
|
+
'/v1/chat/completions',
|
|
323
|
+
{
|
|
324
|
+
'Content-Type': 'application/json',
|
|
325
|
+
'Authorization': `Bearer ${apiKey}`
|
|
326
|
+
},
|
|
327
|
+
body,
|
|
328
|
+
timeoutMs
|
|
329
|
+
);
|
|
330
|
+
|
|
331
|
+
let data;
|
|
332
|
+
try {
|
|
333
|
+
data = JSON.parse(raw.body);
|
|
334
|
+
} catch (_) {
|
|
335
|
+
throw new Error(
|
|
336
|
+
`DeepSeek API returned non-JSON (status ${raw.status}): ${raw.body.slice(0, 120)}`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (data.error) throw new Error(`DeepSeek API: ${data.error.message || JSON.stringify(data.error)}`);
|
|
341
|
+
|
|
342
|
+
const msg = data.choices?.[0]?.message || {};
|
|
343
|
+
const toolCalls = (msg.tool_calls || []).map((tc) => ({
|
|
344
|
+
id: tc.id,
|
|
345
|
+
name: tc.function?.name,
|
|
346
|
+
input: (() => {
|
|
347
|
+
try { return JSON.parse(tc.function?.arguments || '{}'); } catch (_) { return {}; }
|
|
348
|
+
})()
|
|
349
|
+
}));
|
|
350
|
+
|
|
351
|
+
return {
|
|
352
|
+
text: msg.content || '',
|
|
353
|
+
toolCalls,
|
|
354
|
+
rawContent: msg,
|
|
355
|
+
stopReason: data.choices?.[0]?.finish_reason ?? null,
|
|
356
|
+
usage: data.usage ?? null
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// ── Model config resolver ──────────────────────────────────────────────────
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Hardcoded default models per role.
|
|
364
|
+
* Callers that need a different default should pass it in config.models[role].
|
|
365
|
+
*/
|
|
366
|
+
const ROLE_DEFAULTS = {
|
|
367
|
+
generation: 'claude-sonnet-4-6',
|
|
368
|
+
eval: 'claude-sonnet-4-6',
|
|
369
|
+
verifier: 'claude-sonnet-4-6',
|
|
370
|
+
secondary: null
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Detect provider from model name.
|
|
375
|
+
*
|
|
376
|
+
* @param {string} model
|
|
377
|
+
* @returns {'anthropic'|'openai'|'google'|'deepseek'}
|
|
378
|
+
*/
|
|
379
|
+
export function detectProvider(model) {
|
|
380
|
+
if (!model) return 'anthropic';
|
|
381
|
+
if (model.startsWith('claude-')) return 'anthropic';
|
|
382
|
+
if (model.startsWith('gemini-')) return 'google';
|
|
383
|
+
if (model.startsWith('deepseek-')) return 'deepseek';
|
|
384
|
+
if (
|
|
385
|
+
model.startsWith('gpt-') ||
|
|
386
|
+
model.startsWith('o1') ||
|
|
387
|
+
model.startsWith('o3') ||
|
|
388
|
+
model.startsWith('o4')
|
|
389
|
+
) return 'openai';
|
|
390
|
+
// Fallback: assume anthropic for unknown model names
|
|
391
|
+
return 'anthropic';
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Resolve the API key for a given provider from environment variables.
|
|
396
|
+
*
|
|
397
|
+
* @param {string} provider
|
|
398
|
+
* @param {object} env — process.env or equivalent
|
|
399
|
+
* @returns {string|null}
|
|
400
|
+
*/
|
|
401
|
+
export function resolveApiKey(provider, env) {
|
|
402
|
+
switch (provider) {
|
|
403
|
+
case 'anthropic': return env?.ANTHROPIC_API_KEY ?? null;
|
|
404
|
+
case 'openai': return env?.OPENAI_API_KEY ?? null;
|
|
405
|
+
case 'google': return env?.GOOGLE_API_KEY ?? env?.GEMINI_API_KEY ?? null;
|
|
406
|
+
case 'deepseek': return env?.DEEPSEEK_API_KEY ?? null;
|
|
407
|
+
default: return env?.ANTHROPIC_API_KEY ?? null;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Resolve provider, model, and API key from forge config + environment variables.
|
|
413
|
+
*
|
|
414
|
+
* Priority for model:
|
|
415
|
+
* 1. config.models?.[role]
|
|
416
|
+
* 2. config.model
|
|
417
|
+
* 3. Hardcoded default for the role
|
|
418
|
+
*
|
|
419
|
+
* Priority for API key (by provider):
|
|
420
|
+
* anthropic → ANTHROPIC_API_KEY
|
|
421
|
+
* openai → OPENAI_API_KEY
|
|
422
|
+
* google → GOOGLE_API_KEY or GEMINI_API_KEY
|
|
423
|
+
* deepseek → DEEPSEEK_API_KEY
|
|
424
|
+
* Returns null apiKey if key is not present (callers must check).
|
|
425
|
+
*
|
|
426
|
+
* @param {object} config - Forge config object (may be null/undefined)
|
|
427
|
+
* @param {object} env - Key/value env object (e.g. from process.env or loadEnv())
|
|
428
|
+
* @param {string} [role] - 'generation' | 'eval' | 'verifier' | 'secondary'
|
|
429
|
+
* @returns {{ provider: 'anthropic'|'openai'|'google'|'deepseek', apiKey: string|null, model: string|null }}
|
|
430
|
+
*/
|
|
431
|
+
export function resolveModelConfig(config, env, role = 'generation') {
|
|
432
|
+
const model =
|
|
433
|
+
config?.models?.[role] ??
|
|
434
|
+
config?.model ??
|
|
435
|
+
ROLE_DEFAULTS[role] ??
|
|
436
|
+
null;
|
|
437
|
+
|
|
438
|
+
const provider = detectProvider(model);
|
|
439
|
+
const apiKey = resolveApiKey(provider, env);
|
|
440
|
+
|
|
441
|
+
return { provider, apiKey, model };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Build a modelConfig object for a specific model string, resolving provider + key from env.
|
|
446
|
+
* Convenience wrapper used by the model matrix runner.
|
|
447
|
+
*
|
|
448
|
+
* @param {string} modelName
|
|
449
|
+
* @param {object} env
|
|
450
|
+
* @returns {{ provider: string, apiKey: string|null, model: string }}
|
|
451
|
+
*/
|
|
452
|
+
export function modelConfigForName(modelName, env) {
|
|
453
|
+
if (!modelName) throw new Error('modelConfigForName: modelName is required');
|
|
454
|
+
const provider = detectProvider(modelName);
|
|
455
|
+
const apiKey = resolveApiKey(provider, env);
|
|
456
|
+
return { provider, apiKey, model: modelName };
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// ── Streaming Transport ───────────────────────────────────────────────────
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Perform an HTTPS POST request that resolves with the raw IncomingMessage stream
|
|
463
|
+
* instead of buffering. On non-2xx, buffers the error body and throws.
|
|
464
|
+
*
|
|
465
|
+
* @param {string} hostname
|
|
466
|
+
* @param {string} path
|
|
467
|
+
* @param {object} headers
|
|
468
|
+
* @param {object} body
|
|
469
|
+
* @param {number} [timeoutMs=120000]
|
|
470
|
+
* @returns {Promise<import('http').IncomingMessage>}
|
|
471
|
+
*/
|
|
472
|
+
export function httpsPostStream(hostname, path, headers, body, timeoutMs = 120_000) {
|
|
473
|
+
return new Promise((res, rej) => {
|
|
474
|
+
const payload = JSON.stringify(body);
|
|
475
|
+
const req = httpsRequest(
|
|
476
|
+
{
|
|
477
|
+
hostname,
|
|
478
|
+
path,
|
|
479
|
+
method: 'POST',
|
|
480
|
+
headers: { ...headers, 'Content-Length': Buffer.byteLength(payload) }
|
|
481
|
+
},
|
|
482
|
+
(resp) => {
|
|
483
|
+
resp.on('error', rej);
|
|
484
|
+
if (resp.statusCode >= 200 && resp.statusCode < 300) {
|
|
485
|
+
res(resp);
|
|
486
|
+
} else {
|
|
487
|
+
// Buffer error body and throw
|
|
488
|
+
let data = '';
|
|
489
|
+
resp.on('data', (d) => { data += d; });
|
|
490
|
+
resp.on('end', () => rej(new Error(`HTTP ${resp.statusCode}: ${data.slice(0, 300)}`)));
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
);
|
|
494
|
+
req.setTimeout(timeoutMs, () => req.destroy(new Error('API stream timeout')));
|
|
495
|
+
req.on('error', rej);
|
|
496
|
+
req.write(payload);
|
|
497
|
+
req.end();
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// ── SSE Frame Parser ──────────────────────────────────────────────────────
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Shared async generator that splits an IncomingMessage into SSE frames.
|
|
505
|
+
* Yields { event: string|null, data: string }.
|
|
506
|
+
*
|
|
507
|
+
* @param {import('http').IncomingMessage} stream
|
|
508
|
+
* @yields {{ event: string|null, data: string }}
|
|
509
|
+
*/
|
|
510
|
+
export async function* parseSSEFrames(stream) {
|
|
511
|
+
let buffer = '';
|
|
512
|
+
let currentEvent = null;
|
|
513
|
+
let dataLines = [];
|
|
514
|
+
|
|
515
|
+
for await (const chunk of stream) {
|
|
516
|
+
buffer += chunk.toString();
|
|
517
|
+
const lines = buffer.split('\n');
|
|
518
|
+
buffer = lines.pop(); // keep incomplete last line
|
|
519
|
+
|
|
520
|
+
for (const line of lines) {
|
|
521
|
+
if (line.startsWith(':')) continue; // comment
|
|
522
|
+
if (line.startsWith('event: ')) {
|
|
523
|
+
currentEvent = line.slice(7).trim();
|
|
524
|
+
} else if (line.startsWith('data: ')) {
|
|
525
|
+
dataLines.push(line.slice(6));
|
|
526
|
+
} else if (line === '') {
|
|
527
|
+
// Frame separator — emit if we have data
|
|
528
|
+
if (dataLines.length > 0) {
|
|
529
|
+
yield { event: currentEvent, data: dataLines.join('\n') };
|
|
530
|
+
}
|
|
531
|
+
currentEvent = null;
|
|
532
|
+
dataLines = [];
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
// Flush any remaining frame
|
|
537
|
+
if (dataLines.length > 0) {
|
|
538
|
+
yield { event: currentEvent, data: dataLines.join('\n') };
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
// ── Anthropic Stream Parser ───────────────────────────────────────────────
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Parse an Anthropic SSE stream. Yields text_delta events and a final done event.
|
|
546
|
+
*
|
|
547
|
+
* @param {import('http').IncomingMessage} stream
|
|
548
|
+
* @yields {{ type: 'text_delta', text: string } | { type: 'done', text: string, toolCalls: Array, usage: object, stopReason: string|null }}
|
|
549
|
+
*/
|
|
550
|
+
export async function* parseAnthropicStream(stream) {
|
|
551
|
+
let textBlocks = [];
|
|
552
|
+
let currentTextIndex = -1;
|
|
553
|
+
const toolCallBlocks = []; // { id, name, inputFragments: [] }
|
|
554
|
+
let stopReason = null;
|
|
555
|
+
let usage = { input_tokens: 0, output_tokens: 0 };
|
|
556
|
+
|
|
557
|
+
for await (const frame of parseSSEFrames(stream)) {
|
|
558
|
+
if (frame.data === '[DONE]') break;
|
|
559
|
+
|
|
560
|
+
let parsed;
|
|
561
|
+
try { parsed = JSON.parse(frame.data); } catch { continue; }
|
|
562
|
+
|
|
563
|
+
const eventType = frame.event || parsed.type;
|
|
564
|
+
|
|
565
|
+
if (eventType === 'message_start' && parsed.message?.usage) {
|
|
566
|
+
usage.input_tokens = parsed.message.usage.input_tokens || 0;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if (eventType === 'content_block_start') {
|
|
570
|
+
const block = parsed.content_block;
|
|
571
|
+
if (block?.type === 'text') {
|
|
572
|
+
currentTextIndex = textBlocks.length;
|
|
573
|
+
textBlocks.push('');
|
|
574
|
+
} else if (block?.type === 'tool_use') {
|
|
575
|
+
toolCallBlocks.push({ id: block.id, name: block.name, inputFragments: [] });
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
if (eventType === 'content_block_delta') {
|
|
580
|
+
const delta = parsed.delta;
|
|
581
|
+
if (delta?.type === 'text_delta' && delta.text) {
|
|
582
|
+
if (currentTextIndex >= 0) textBlocks[currentTextIndex] += delta.text;
|
|
583
|
+
yield { type: 'text_delta', text: delta.text };
|
|
584
|
+
} else if (delta?.type === 'input_json_delta' && delta.partial_json !== undefined) {
|
|
585
|
+
const lastTool = toolCallBlocks[toolCallBlocks.length - 1];
|
|
586
|
+
if (lastTool) lastTool.inputFragments.push(delta.partial_json);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
if (eventType === 'content_block_stop') {
|
|
591
|
+
// Reset current text index so subsequent text blocks start fresh
|
|
592
|
+
currentTextIndex = -1;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
if (eventType === 'message_delta') {
|
|
596
|
+
if (parsed.delta?.stop_reason) stopReason = parsed.delta.stop_reason;
|
|
597
|
+
if (parsed.usage?.output_tokens) usage.output_tokens = parsed.usage.output_tokens;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (eventType === 'message_stop') {
|
|
601
|
+
break;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const fullText = textBlocks.join('');
|
|
606
|
+
const toolCalls = toolCallBlocks.map(tc => {
|
|
607
|
+
let input = {};
|
|
608
|
+
if (tc.inputFragments.length > 0) {
|
|
609
|
+
try { input = JSON.parse(tc.inputFragments.join('')); } catch { /* malformed */ }
|
|
610
|
+
}
|
|
611
|
+
return { id: tc.id, name: tc.name, input };
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
yield { type: 'done', text: fullText, toolCalls, usage, stopReason };
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// ── OpenAI-Compatible Stream Parser ───────────────────────────────────────
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Parse an OpenAI-compatible SSE stream (OpenAI, Gemini, DeepSeek).
|
|
621
|
+
* Yields text_delta events and a final done event.
|
|
622
|
+
*
|
|
623
|
+
* @param {import('http').IncomingMessage} stream
|
|
624
|
+
* @param {string} providerName — for error messages
|
|
625
|
+
* @yields {{ type: 'text_delta', text: string } | { type: 'done', text: string, toolCalls: Array, usage: object|null, stopReason: string|null }}
|
|
626
|
+
*/
|
|
627
|
+
export async function* parseOpenAICompatStream(stream, providerName) {
|
|
628
|
+
let fullText = '';
|
|
629
|
+
const toolCallBuffers = {}; // keyed by index
|
|
630
|
+
let stopReason = null;
|
|
631
|
+
let usage = null;
|
|
632
|
+
|
|
633
|
+
for await (const frame of parseSSEFrames(stream)) {
|
|
634
|
+
if (frame.data.trim() === '[DONE]') break;
|
|
635
|
+
|
|
636
|
+
let parsed;
|
|
637
|
+
try { parsed = JSON.parse(frame.data); } catch { continue; }
|
|
638
|
+
|
|
639
|
+
// Usage (OpenAI sends in final chunk)
|
|
640
|
+
if (parsed.usage) usage = parsed.usage;
|
|
641
|
+
|
|
642
|
+
const choice = parsed.choices?.[0];
|
|
643
|
+
if (!choice) continue;
|
|
644
|
+
|
|
645
|
+
if (choice.finish_reason) stopReason = choice.finish_reason;
|
|
646
|
+
|
|
647
|
+
const delta = choice.delta;
|
|
648
|
+
if (!delta) continue;
|
|
649
|
+
|
|
650
|
+
// Text content
|
|
651
|
+
if (delta.content) {
|
|
652
|
+
fullText += delta.content;
|
|
653
|
+
yield { type: 'text_delta', text: delta.content };
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// Tool calls — streamed incrementally by index
|
|
657
|
+
if (delta.tool_calls) {
|
|
658
|
+
for (const tc of delta.tool_calls) {
|
|
659
|
+
const idx = tc.index ?? 0;
|
|
660
|
+
if (!toolCallBuffers[idx]) {
|
|
661
|
+
toolCallBuffers[idx] = { id: tc.id || '', name: tc.function?.name || '', argFragments: [] };
|
|
662
|
+
}
|
|
663
|
+
if (tc.id) toolCallBuffers[idx].id = tc.id;
|
|
664
|
+
if (tc.function?.name) toolCallBuffers[idx].name = tc.function.name;
|
|
665
|
+
if (tc.function?.arguments) toolCallBuffers[idx].argFragments.push(tc.function.arguments);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const toolCalls = Object.values(toolCallBuffers).map(tc => {
|
|
671
|
+
let input = {};
|
|
672
|
+
if (tc.argFragments.length > 0) {
|
|
673
|
+
try { input = JSON.parse(tc.argFragments.join('')); } catch { /* malformed */ }
|
|
674
|
+
}
|
|
675
|
+
return { id: tc.id, name: tc.name, input };
|
|
676
|
+
});
|
|
677
|
+
|
|
678
|
+
yield { type: 'done', text: fullText, toolCalls, usage, stopReason };
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// ── Provider Stream Turn Functions ────────────────────────────────────────
|
|
682
|
+
|
|
683
|
+
async function* _anthropicStreamTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
684
|
+
const body = {
|
|
685
|
+
model,
|
|
686
|
+
max_tokens: maxTokens,
|
|
687
|
+
stream: true,
|
|
688
|
+
...(system ? { system } : {}),
|
|
689
|
+
messages,
|
|
690
|
+
...(tools.length ? { tools: tools.map(toAnthropicTool) } : {})
|
|
691
|
+
};
|
|
692
|
+
|
|
693
|
+
const stream = await httpsPostStream(
|
|
694
|
+
'api.anthropic.com',
|
|
695
|
+
'/v1/messages',
|
|
696
|
+
{
|
|
697
|
+
'Content-Type': 'application/json',
|
|
698
|
+
'anthropic-version': '2023-06-01',
|
|
699
|
+
'x-api-key': apiKey
|
|
700
|
+
},
|
|
701
|
+
body,
|
|
702
|
+
timeoutMs
|
|
703
|
+
);
|
|
704
|
+
|
|
705
|
+
yield* parseAnthropicStream(stream);
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
async function* _openaiCompatStreamTurn(hostname, path, headers, { model, system, messages, tools, maxTokens, timeoutMs }) {
|
|
709
|
+
const msgs = system
|
|
710
|
+
? [{ role: 'system', content: system }, ...messages]
|
|
711
|
+
: [...messages];
|
|
712
|
+
|
|
713
|
+
const body = {
|
|
714
|
+
model,
|
|
715
|
+
max_tokens: maxTokens,
|
|
716
|
+
stream: true,
|
|
717
|
+
stream_options: { include_usage: true },
|
|
718
|
+
messages: msgs,
|
|
719
|
+
...(tools.length ? { tools: tools.map(toOpenAiTool), tool_choice: 'auto' } : {})
|
|
720
|
+
};
|
|
721
|
+
|
|
722
|
+
const stream = await httpsPostStream(hostname, path, headers, body, timeoutMs);
|
|
723
|
+
yield* parseOpenAICompatStream(stream, 'openai-compat');
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
async function* _openaiStreamTurn(opts) {
|
|
727
|
+
yield* _openaiCompatStreamTurn(
|
|
728
|
+
'api.openai.com',
|
|
729
|
+
'/v1/chat/completions',
|
|
730
|
+
{ 'Content-Type': 'application/json', 'Authorization': `Bearer ${opts.apiKey}` },
|
|
731
|
+
opts
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
async function* _geminiStreamTurn(opts) {
|
|
736
|
+
yield* _openaiCompatStreamTurn(
|
|
737
|
+
'generativelanguage.googleapis.com',
|
|
738
|
+
'/v1beta/openai/chat/completions',
|
|
739
|
+
{ 'Content-Type': 'application/json', 'Authorization': `Bearer ${opts.apiKey}` },
|
|
740
|
+
opts
|
|
741
|
+
);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
async function* _deepseekStreamTurn(opts) {
|
|
745
|
+
yield* _openaiCompatStreamTurn(
|
|
746
|
+
'api.deepseek.com',
|
|
747
|
+
'/v1/chat/completions',
|
|
748
|
+
{ 'Content-Type': 'application/json', 'Authorization': `Bearer ${opts.apiKey}` },
|
|
749
|
+
opts
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// ── Unified Streaming LLM Turn ────────────────────────────────────────────
|
|
754
|
+
|
|
755
|
+
/**
|
|
756
|
+
* Perform a single streaming LLM turn. Async generator that yields
|
|
757
|
+
* { type: 'text_delta', text } during the stream, and
|
|
758
|
+
* { type: 'done', text, toolCalls, usage, stopReason } at end.
|
|
759
|
+
*
|
|
760
|
+
* Same opts as llmTurn.
|
|
761
|
+
*
|
|
762
|
+
* @param {object} opts
|
|
763
|
+
* @yields {{ type: 'text_delta', text: string } | { type: 'done', text: string, toolCalls: Array, usage: object, stopReason: string|null }}
|
|
764
|
+
*/
|
|
765
|
+
export async function* llmTurnStreaming({
|
|
766
|
+
provider,
|
|
767
|
+
apiKey,
|
|
768
|
+
model,
|
|
769
|
+
system,
|
|
770
|
+
messages,
|
|
771
|
+
tools = [],
|
|
772
|
+
maxTokens = 4096,
|
|
773
|
+
timeoutMs = 120_000
|
|
774
|
+
}) {
|
|
775
|
+
if (provider === 'anthropic') {
|
|
776
|
+
yield* _anthropicStreamTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
if (provider === 'openai') {
|
|
780
|
+
yield* _openaiStreamTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
if (provider === 'google') {
|
|
784
|
+
yield* _geminiStreamTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
if (provider === 'deepseek') {
|
|
788
|
+
yield* _deepseekStreamTurn({ apiKey, model, system, messages, tools, maxTokens, timeoutMs });
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
throw new Error(`llmTurnStreaming: unknown provider "${provider}". Expected 'anthropic', 'openai', 'google', or 'deepseek'.`);
|
|
792
|
+
}
|