@amenophis1er/foreman 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/DESIGN.md +408 -0
  2. package/LICENSE +15 -0
  3. package/README.md +133 -0
  4. package/bin/foreman.mjs +58 -0
  5. package/package.json +68 -0
  6. package/scripts/prepare.mjs +48 -0
  7. package/skills/director/SKILL.md +65 -0
  8. package/src/anthropic-models.ts +54 -0
  9. package/src/ask.test.ts +88 -0
  10. package/src/ask.ts +95 -0
  11. package/src/attachments.test.ts +33 -0
  12. package/src/attachments.ts +60 -0
  13. package/src/cli.test.ts +27 -0
  14. package/src/cli.ts +297 -0
  15. package/src/codex.test.ts +328 -0
  16. package/src/codex.ts +196 -0
  17. package/src/cost-basis.test.ts +76 -0
  18. package/src/deck.test.ts +402 -0
  19. package/src/deck.ts +892 -0
  20. package/src/fork.test.ts +31 -0
  21. package/src/gateway/ledger.cjs +326 -0
  22. package/src/gateway/ledger.test.ts +255 -0
  23. package/src/gateway/llm-gateway.cjs +1411 -0
  24. package/src/gateway/llm-gateway.test.ts +478 -0
  25. package/src/gateway.test.ts +226 -0
  26. package/src/gateway.ts +309 -0
  27. package/src/instance.ts +124 -0
  28. package/src/models.test.ts +147 -0
  29. package/src/models.ts +158 -0
  30. package/src/notify/commands.test.ts +28 -0
  31. package/src/notify/commands.ts +73 -0
  32. package/src/notify/telegram.ts +259 -0
  33. package/src/notify.test.ts +343 -0
  34. package/src/notify.ts +495 -0
  35. package/src/ollama.test.ts +49 -0
  36. package/src/ollama.ts +49 -0
  37. package/src/openai-prices.test.ts +58 -0
  38. package/src/openai-prices.ts +106 -0
  39. package/src/orchestrator.test.ts +1147 -0
  40. package/src/orchestrator.ts +2325 -0
  41. package/src/planner.test.ts +60 -0
  42. package/src/planner.ts +505 -0
  43. package/src/policy.test.ts +411 -0
  44. package/src/policy.ts +599 -0
  45. package/src/preflight.ts +348 -0
  46. package/src/prices.test.ts +69 -0
  47. package/src/prices.ts +90 -0
  48. package/src/provider.test.ts +366 -0
  49. package/src/provider.ts +502 -0
  50. package/src/secrets.test.ts +143 -0
  51. package/src/secrets.ts +66 -0
  52. package/src/server.ts +1992 -0
  53. package/src/services.test.ts +53 -0
  54. package/src/services.ts +102 -0
  55. package/src/sse-events.test.ts +83 -0
  56. package/src/store.test.ts +119 -0
  57. package/src/store.ts +346 -0
  58. package/src/tailscale.test.ts +32 -0
  59. package/src/tailscale.ts +79 -0
  60. package/src/title.ts +138 -0
  61. package/src/types.ts +442 -0
  62. package/ui/dist/assets/index-LAj0Dy9p.css +1 -0
  63. package/ui/dist/assets/index-lcBy-uRZ.js +65 -0
  64. package/ui/dist/favicon.svg +8 -0
  65. package/ui/dist/index.html +14 -0
@@ -0,0 +1,31 @@
1
+ import { test } from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { forkLabel, forkSeed } from './planner.js';
4
+
5
+ test('forkSeed: the transcript line names the run, the prompt carries brief, doc and report', () => {
6
+ const seed = forkSeed({
7
+ title: 'IRONOATH Strength House', mission: 'Build a single-page site…', status: 'done',
8
+ endedAt: Date.UTC(2026, 8, 5, 15, 14), missionDoc: '# MISSION\n- [x] index.html exists', report: 'Mission complete. Built the site.',
9
+ });
10
+ assert.equal(seed.shown, 'Plan the next step after “IRONOATH Strength House”.');
11
+ assert.match(seed.prompt, /Previous mission — IRONOATH Strength House \(done, /);
12
+ assert.match(seed.prompt, /Build a single-page site…/);
13
+ assert.match(seed.prompt, /index\.html exists/);
14
+ assert.match(seed.prompt, /Mission complete\. Built the site\./);
15
+ assert.match(seed.prompt, /Do not re-propose or redo/);
16
+ assert.match(seed.prompt, /ask_user/);
17
+ });
18
+
19
+ test('forkSeed: no title falls back to the brief\'s first line; missing doc and report leave no empty sections', () => {
20
+ const seed = forkSeed({ mission: '\n Add a contact form\nmore detail', status: 'interrupted', missionDoc: null, report: '' });
21
+ assert.equal(forkLabel({ mission: '\n Add a contact form\nmore' }), 'Add a contact form');
22
+ assert.equal(seed.shown, 'Plan the next step after “Add a contact form”.');
23
+ assert.doesNotMatch(seed.prompt, /mission doc/);
24
+ assert.doesNotMatch(seed.prompt, /final report/);
25
+ });
26
+
27
+ test('forkSeed: long inputs are clipped and say so', () => {
28
+ const seed = forkSeed({ mission: 'm', status: 'done', missionDoc: 'x'.repeat(7000), report: null });
29
+ assert.match(seed.prompt, /more characters not shown/);
30
+ assert.ok(seed.prompt.length < 7000);
31
+ });
@@ -0,0 +1,326 @@
1
+ /**
2
+ * The gateway's token ledger — a Foreman-owned wrapper around the ported
3
+ * llm-gateway.
4
+ *
5
+ * WHY THIS IS A SEPARATE FILE. `llm-gateway.cjs` is a verbatim copy from a
6
+ * sibling project, and the Step 2 decision was that it stays one so re-syncing
7
+ * is a `cp` and its upstream test suite keeps applying. Counting tokens needs
8
+ * the request (which run is this?) and the response (how many tokens?), which
9
+ * is exactly the request path that decision refused to edit. So this binds the
10
+ * socket instead, and hands each request to the ported handler untouched.
11
+ *
12
+ * WHY IT EXISTS AT ALL. An OpenAI-compatible endpoint reports usage only on
13
+ * the *final* chunk of a stream, and the SDK only surfaces it on the `result`
14
+ * message that ends a turn. A director in one long turn — every tool call,
15
+ * every file write, one turn — therefore shows `0 tok` for minutes while
16
+ * really having spent a hundred thousand. The gateway is the only place that
17
+ * sees the numbers as they flow.
18
+ *
19
+ * ATTRIBUTION. Foreman points each agent at `.../run/<key>`, and the Agent SDK
20
+ * preserves that prefix (measured, not assumed: it sends
21
+ * `POST /run/<key>/v1/messages?beta=true`). The key carries the resume attempt
22
+ * as well as the run id, so a resumed run starts a fresh bucket and its
23
+ * persisted total is never counted twice by a gateway that outlived it.
24
+ *
25
+ * REQUEST DUMPS. Set FOREMAN_GATEWAY_DUMP_DIR and every POST to /v1/messages
26
+ * is written to disk, request and response outcome, before it is served. This
27
+ * exists because a stall was only ever reproducible with the real worker
28
+ * payload — the SDK's full system prompt plus its whole tool set — and no
29
+ * hand-built request came close. The only honest way to replay it is to have
30
+ * kept it. Off by default: the files hold prompts and repository contents.
31
+ */
32
+ const http = require('node:http');
33
+ const fs = require('node:fs');
34
+ const path = require('node:path');
35
+ const { Readable } = require('node:stream');
36
+ const gateway = require('./llm-gateway.cjs');
37
+
38
+ const PORT = Number(process.env.LLM_GATEWAY_PORT || 0);
39
+ const MODE = process.env.LLM_GATEWAY_MODE || 'openai';
40
+ const DUMP_DIR = process.env.FOREMAN_GATEWAY_DUMP_DIR || '';
41
+
42
+ /** key -> running totals. In memory: a restarted gateway has counted nothing. */
43
+ const ledger = new Map();
44
+
45
+ const empty = () => ({
46
+ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, calls: 0,
47
+ });
48
+
49
+ const n = (v) => (typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : 0);
50
+
51
+ /**
52
+ * Fold one response's usage into a key's total.
53
+ *
54
+ * `costUsd` is only ever recorded when the upstream states it (OpenRouter
55
+ * does, per response). Foreman never derives one here — a gateway inventing a
56
+ * dollar figure is the precise failure the cost basis exists to prevent.
57
+ */
58
+ function record(key, usage, costUsd) {
59
+ if (!key || !usage) return;
60
+ const t = ledger.get(key) ?? empty();
61
+ t.inputTokens += n(usage.input_tokens ?? usage.prompt_tokens);
62
+ t.outputTokens += n(usage.output_tokens ?? usage.completion_tokens);
63
+ t.cacheReadTokens += n(usage.cache_read_input_tokens);
64
+ t.cacheWriteTokens += n(usage.cache_creation_input_tokens);
65
+ t.calls += 1;
66
+ if (typeof costUsd === 'number' && Number.isFinite(costUsd) && costUsd >= 0) {
67
+ t.costUsd = (t.costUsd ?? 0) + costUsd;
68
+ }
69
+ ledger.set(key, t);
70
+ }
71
+
72
+ /**
73
+ * Pull usage out of whatever the handler wrote back.
74
+ *
75
+ * Two shapes reach here and both are handled by looking for the same field
76
+ * rather than by knowing which mode produced it: a single JSON body carrying
77
+ * `usage`, or an SSE stream whose `message_delta` frame carries it at the end.
78
+ * Anything unparseable is skipped in silence — a ledger is an observation, and
79
+ * must never be able to fail a request it is only watching.
80
+ */
81
+ function extractUsage(body) {
82
+ if (!body.includes('"usage"')) return null;
83
+ let found = null;
84
+ let cost;
85
+ const consider = (obj) => {
86
+ if (!obj || typeof obj !== 'object') return;
87
+ if (obj.usage && typeof obj.usage === 'object') {
88
+ // The last usage object wins: a stream's closing `message_delta` is
89
+ // more complete than anything before it.
90
+ found = obj.usage;
91
+ if (typeof obj.usage.cost === 'number') cost = obj.usage.cost;
92
+ }
93
+ };
94
+ if (body.startsWith('data:') || body.includes('\ndata:')) {
95
+ for (const line of body.split('\n')) {
96
+ const t = line.trim();
97
+ if (!t.startsWith('data:')) continue;
98
+ const payload = t.slice(5).trim();
99
+ if (!payload || payload === '[DONE]') continue;
100
+ try { consider(JSON.parse(payload)); } catch { /* partial frame */ }
101
+ }
102
+ } else {
103
+ try { consider(JSON.parse(body)); } catch { /* not json */ }
104
+ }
105
+ return found ? { usage: found, costUsd: cost } : null;
106
+ }
107
+
108
+ /**
109
+ * Watch a response without changing it.
110
+ *
111
+ * Chunks are buffered only while they could still contain a usage object, and
112
+ * the buffer is capped: a ledger must not turn a long streamed answer into
113
+ * unbounded memory. Every write is passed straight through first, so nothing
114
+ * here can delay or alter what the agent receives.
115
+ *
116
+ * `observe`, when given, is told the outcome once at end — status, timing,
117
+ * size, whether usage ever appeared. The dumper hangs off this rather than
118
+ * wrapping `res` a second time: two observers patching write/end would each
119
+ * see the other's wrapper, and a bug in either would be twice as hard to place.
120
+ */
121
+ function tee(res, key, observe) {
122
+ const { write, end } = res;
123
+ let buf = '';
124
+ let bytes = 0;
125
+ let firstByteMs = null;
126
+ const started = Date.now();
127
+ const CAP = 256 * 1024;
128
+ const absorb = (chunk) => {
129
+ if (!chunk) return;
130
+ const len = typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length;
131
+ if (len > 0 && firstByteMs === null) firstByteMs = Date.now() - started;
132
+ bytes += len;
133
+ if (buf.length > CAP) return;
134
+ // Buffer.from, not chunk.toString(): a Uint8Array's own toString() ignores
135
+ // the encoding and yields "101,118,101,..." — comma-separated byte values
136
+ // that parse as nothing and would silently record zero tokens forever.
137
+ buf += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8');
138
+ };
139
+ res.write = function (chunk, ...rest) {
140
+ absorb(chunk);
141
+ return write.call(this, chunk, ...rest);
142
+ };
143
+ res.end = function (chunk, ...rest) {
144
+ absorb(chunk);
145
+ let sawUsage = false;
146
+ try {
147
+ const got = extractUsage(buf);
148
+ sawUsage = !!got;
149
+ if (got && key) record(key, got.usage, got.costUsd);
150
+ } catch { /* never fail a response over bookkeeping */ }
151
+ if (observe) {
152
+ try {
153
+ observe({
154
+ status: res.statusCode, durationMs: Date.now() - started, bytes, sawUsage, firstByteMs,
155
+ });
156
+ } catch { /* an observer is not allowed to fail the response either */ }
157
+ }
158
+ return end.call(this, chunk, ...rest);
159
+ };
160
+ }
161
+
162
+ /** Header copy safe to write to disk: the two places a credential travels. */
163
+ function redactHeaders(headers) {
164
+ const out = {};
165
+ for (const [k, v] of Object.entries(headers || {})) {
166
+ out[k] = /^(x-api-key|authorization)$/i.test(k) ? '[redacted]' : v;
167
+ }
168
+ return out;
169
+ }
170
+
171
+ /**
172
+ * The few numbers that tell a stalled request apart from a healthy one at a
173
+ * glance, without opening a multi-megabyte file. `system` is a string or an
174
+ * array of text blocks depending on the SDK version; both are measured.
175
+ */
176
+ function summarize(body, bodyBytes) {
177
+ const s = { model: null, stream: null, systemChars: 0, toolCount: 0, messageCount: 0, bodyBytes };
178
+ if (!body || typeof body !== 'object') return s;
179
+ s.model = typeof body.model === 'string' ? body.model : null;
180
+ s.stream = !!body.stream;
181
+ if (typeof body.system === 'string') s.systemChars = body.system.length;
182
+ else if (Array.isArray(body.system)) {
183
+ for (const b of body.system) if (b && typeof b.text === 'string') s.systemChars += b.text.length;
184
+ }
185
+ if (Array.isArray(body.tools)) s.toolCount = body.tools.length;
186
+ if (Array.isArray(body.messages)) s.messageCount = body.messages.length;
187
+ return s;
188
+ }
189
+
190
+ /**
191
+ * Build the request dumper, or nothing when the directory is unset.
192
+ *
193
+ * `begin` writes the request file and returns the function that writes its
194
+ * response file; both swallow everything. A dump is a diagnostic aid bolted
195
+ * onto the request path, and the one thing worse than a missing dump is a
196
+ * worker that failed because its gateway could not write a file.
197
+ */
198
+ function makeDumper(dir, log = (line) => process.stderr.write(`${line}\n`)) {
199
+ if (!dir) return null;
200
+ let seq = 0;
201
+ let made = false;
202
+ return {
203
+ dir,
204
+ begin(key, req, raw) {
205
+ try {
206
+ if (!made) { fs.mkdirSync(dir, { recursive: true }); made = true; }
207
+ const ts = new Date();
208
+ // The key is `<runId>.<attempt>`-ish but comes off the wire; keep the
209
+ // filename to characters every filesystem accepts.
210
+ const safeKey = String(key ?? 'unkeyed').replace(/[^A-Za-z0-9._-]/g, '_');
211
+ const name = `${ts.toISOString().replace(/[:.]/g, '-')}-${safeKey}-${++seq}`;
212
+ const text = Buffer.from(raw).toString('utf8');
213
+ let body = text;
214
+ try { body = JSON.parse(text); } catch { /* keep the raw string: a bad body is the finding */ }
215
+ const summary = summarize(body, raw.length);
216
+ fs.writeFileSync(path.join(dir, `${name}.json`), JSON.stringify({
217
+ ts: ts.toISOString(), key, url: req.url, headers: redactHeaders(req.headers), body, summary,
218
+ }));
219
+ const label = key ?? 'unkeyed';
220
+ log(`[dump] ${label} -> ${summary.model} tools=${summary.toolCount} system=${summary.systemChars} stream=${summary.stream}`);
221
+ return (info) => {
222
+ try {
223
+ fs.writeFileSync(path.join(dir, `${name}.response.json`), JSON.stringify(info));
224
+ log(`[dump] ${label} <- ${info.status} in ${info.durationMs}ms ${info.bytes}B usage=${info.sawUsage}`);
225
+ } catch { /* see above */ }
226
+ };
227
+ } catch {
228
+ return () => {};
229
+ }
230
+ },
231
+ };
232
+ }
233
+
234
+ /** Drain a request into memory. */
235
+ function readAll(req) {
236
+ return new Promise((resolve, reject) => {
237
+ const chunks = [];
238
+ req.on('data', (c) => chunks.push(c));
239
+ req.on('end', () => resolve(Buffer.concat(chunks)));
240
+ req.on('error', reject);
241
+ });
242
+ }
243
+
244
+ /**
245
+ * A stand-in request the handler cannot tell from the original.
246
+ *
247
+ * The body has been drained to dump it, so the handler gets a stream that
248
+ * plays it back. Sharing a 'data' listener with the handler instead would
249
+ * work today only because each handler subscribes before its first `await`
250
+ * — an ordering inside a verbatim upstream file that a re-sync may not
251
+ * keep, and whose failure mode is a silently truncated prompt. The handlers
252
+ * read exactly headers, method, url, on and pipe (measured, not assumed).
253
+ */
254
+ function replay(req, raw) {
255
+ const r = new Readable({ read() {} });
256
+ r.headers = req.headers;
257
+ r.method = req.method;
258
+ r.url = req.url;
259
+ r.push(raw);
260
+ r.push(null);
261
+ return r;
262
+ }
263
+
264
+ function startServer() {
265
+ const handler = MODE === 'passthrough' ? gateway.handlePassthrough
266
+ : MODE === 'codex' ? gateway.handleCodex
267
+ : gateway.handleOpenAI;
268
+ const dumper = makeDumper(DUMP_DIR);
269
+ if (dumper) process.stderr.write(`[dump] writing /v1/messages requests to ${dumper.dir}\n`);
270
+
271
+ const server = http.createServer((req, res) => {
272
+ // Foreman's own read side. Never proxied upstream.
273
+ if (req.method === 'GET' && req.url.startsWith('/_foreman/usage')) {
274
+ res.writeHead(200, { 'content-type': 'application/json' });
275
+ res.end(JSON.stringify(Object.fromEntries(ledger)));
276
+ return;
277
+ }
278
+
279
+ // Strip the run key so the ported handler sees the path it expects. An
280
+ // unkeyed request still works and is simply not attributed — a gateway
281
+ // that refused traffic it could not label would turn a bookkeeping
282
+ // feature into an outage.
283
+ const m = /^\/run\/([^/]+)(\/.*)$/.exec(req.url || '');
284
+ const key = m ? decodeURIComponent(m[1]) : null;
285
+ if (m) req.url = m[2];
286
+
287
+ const wantDump = !!dumper && req.method === 'POST' && (req.url || '').startsWith('/v1/messages');
288
+ // Assigned once the request file is written; the tee calls it at end.
289
+ let finishDump = () => {};
290
+ if (key || wantDump) tee(res, key, wantDump ? (info) => finishDump(info) : undefined);
291
+
292
+ const fail = (e) => {
293
+ if (!res.headersSent) res.writeHead(502, { 'content-type': 'application/json' });
294
+ res.end(JSON.stringify({
295
+ type: 'error',
296
+ error: { type: 'api_error', message: String((e && e.message) || e) },
297
+ }));
298
+ };
299
+ if (wantDump) {
300
+ // Drain, dump, then serve from the replay. The undumped path below is
301
+ // untouched: the handler still gets the live socket stream.
302
+ readAll(req).then((raw) => {
303
+ finishDump = dumper.begin(key, req, raw);
304
+ return handler(replay(req, raw), res);
305
+ }).catch(fail);
306
+ } else {
307
+ Promise.resolve(handler(req, res)).catch(fail);
308
+ }
309
+ });
310
+ server.on('error', (e) => {
311
+ if (e.code === 'EADDRINUSE') process.exit(0);
312
+ throw e;
313
+ });
314
+ server.listen(PORT, '127.0.0.1', () => {
315
+ const addr = server.address();
316
+ process.stdout.write(
317
+ `llm-gateway (${MODE}) listening on 127.0.0.1:${addr.port} -> ${process.env.LLM_GATEWAY_TARGET_URL}\n`,
318
+ );
319
+ });
320
+ }
321
+
322
+ module.exports = {
323
+ extractUsage, record, ledger, tee, makeDumper, summarize, redactHeaders, replay,
324
+ };
325
+
326
+ if (require.main === module) startServer();
@@ -0,0 +1,255 @@
1
+ /**
2
+ * The gateway's token ledger.
3
+ *
4
+ * What is under test is an observation that must never affect what it
5
+ * observes: the response an agent receives has to be byte-identical whether
6
+ * or not anyone is counting, and a request that cannot be attributed must
7
+ * still be served. The counting itself is second to that.
8
+ */
9
+ import test from 'node:test';
10
+ import assert from 'node:assert/strict';
11
+ import { createRequire } from 'node:module';
12
+ import { mkdtempSync, readdirSync, readFileSync, existsSync } from 'node:fs';
13
+ import { tmpdir } from 'node:os';
14
+ import { join } from 'node:path';
15
+
16
+ const require_ = createRequire(import.meta.url);
17
+ const ledgerMod = require_('./ledger.cjs') as {
18
+ extractUsage(body: string): { usage: Record<string, number>; costUsd?: number } | null;
19
+ record(key: string, usage: unknown, costUsd?: number): void;
20
+ ledger: Map<string, Record<string, number>>;
21
+ };
22
+ const gateway = require_('./llm-gateway.cjs') as Record<string, unknown>;
23
+
24
+ test('the ported handlers are exported, which is what lets the socket be wrapped', () => {
25
+ // A divergence from the verbatim upstream copy. If a re-sync drops it, the
26
+ // wrapper cannot bind the socket and every gateway run stops being counted
27
+ // — silently, which is why this is asserted rather than assumed.
28
+ for (const name of ['handleOpenAI', 'handleCodex', 'handlePassthrough']) {
29
+ assert.equal(typeof gateway[name], 'function', `${name} must stay exported`);
30
+ }
31
+ });
32
+
33
+ test('usage is read from a single JSON response', () => {
34
+ const got = ledgerMod.extractUsage(JSON.stringify({
35
+ type: 'message', usage: { input_tokens: 120, output_tokens: 34 },
36
+ }));
37
+ assert.equal(got?.usage.input_tokens, 120);
38
+ assert.equal(got?.usage.output_tokens, 34);
39
+ });
40
+
41
+ test('usage is read from the closing frame of a stream, not an earlier one', () => {
42
+ // An OpenAI-compatible endpoint reports usage only at the end, and the
43
+ // gateway's `message_start` carries zeros. Taking the first match would
44
+ // record nothing for every streamed response there is.
45
+ const body = [
46
+ 'data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}',
47
+ '',
48
+ 'data: {"type":"content_block_delta","delta":{"text":"hi"}}',
49
+ '',
50
+ 'data: {"type":"message_delta","usage":{"input_tokens":9001,"output_tokens":42}}',
51
+ '',
52
+ 'data: [DONE]',
53
+ '',
54
+ ].join('\n');
55
+ const got = ledgerMod.extractUsage(body);
56
+ assert.equal(got?.usage.input_tokens, 9001);
57
+ assert.equal(got?.usage.output_tokens, 42);
58
+ });
59
+
60
+ test('a body with no usage, or a torn one, yields nothing rather than throwing', () => {
61
+ assert.equal(ledgerMod.extractUsage('{"type":"message"}'), null);
62
+ assert.equal(ledgerMod.extractUsage('not json at all'), null);
63
+ assert.equal(ledgerMod.extractUsage(''), null);
64
+ // A stream cut mid-frame: the complete frames before it still count.
65
+ const torn = 'data: {"type":"message_delta","usage":{"input_tokens":5,"output_tokens":1}}\n\ndata: {"typ';
66
+ assert.equal(ledgerMod.extractUsage(torn)?.usage.input_tokens, 5);
67
+ });
68
+
69
+ test('an upstream-reported cost is carried, and never invented', () => {
70
+ // OpenRouter states `usage.cost` per response. Where it does, that is the
71
+ // truthful figure; where it does not, the ledger leaves the field absent
72
+ // rather than deriving one.
73
+ const withCost = ledgerMod.extractUsage(JSON.stringify({
74
+ usage: { input_tokens: 1, output_tokens: 1, cost: 0.00042 },
75
+ }));
76
+ assert.equal(withCost?.costUsd, 0.00042);
77
+ const without = ledgerMod.extractUsage(JSON.stringify({
78
+ usage: { input_tokens: 1, output_tokens: 1 },
79
+ }));
80
+ assert.equal(without?.costUsd, undefined);
81
+ });
82
+
83
+ test('totals accumulate per key, and keys do not bleed into each other', () => {
84
+ ledgerMod.ledger.clear();
85
+ ledgerMod.record('run-a.0', { input_tokens: 100, output_tokens: 10 });
86
+ ledgerMod.record('run-a.0', { input_tokens: 50, output_tokens: 5 });
87
+ ledgerMod.record('run-b.0', { input_tokens: 7, output_tokens: 1 });
88
+
89
+ assert.deepEqual(ledgerMod.ledger.get('run-a.0'), {
90
+ inputTokens: 150, outputTokens: 15, cacheReadTokens: 0, cacheWriteTokens: 0, calls: 2,
91
+ });
92
+ assert.equal(ledgerMod.ledger.get('run-b.0')?.inputTokens, 7);
93
+ });
94
+
95
+ test('a resumed run gets a fresh bucket, so persisted tokens are never counted twice', () => {
96
+ // The attempt number is part of the key precisely because a gateway can
97
+ // outlive the run that started it.
98
+ ledgerMod.ledger.clear();
99
+ ledgerMod.record('run-a.0', { input_tokens: 100, output_tokens: 10 });
100
+ ledgerMod.record('run-a.1', { input_tokens: 30, output_tokens: 3 });
101
+ assert.equal(ledgerMod.ledger.get('run-a.0')?.inputTokens, 100);
102
+ assert.equal(ledgerMod.ledger.get('run-a.1')?.inputTokens, 30);
103
+ });
104
+
105
+ test('OpenAI-shaped usage counts the same as Anthropic-shaped', () => {
106
+ // Both reach the ledger depending on mode and on where in the translation
107
+ // the response was observed.
108
+ ledgerMod.ledger.clear();
109
+ ledgerMod.record('k', { prompt_tokens: 200, completion_tokens: 20 });
110
+ assert.equal(ledgerMod.ledger.get('k')?.inputTokens, 200);
111
+ assert.equal(ledgerMod.ledger.get('k')?.outputTokens, 20);
112
+ });
113
+
114
+ test('nonsense in a usage object is ignored rather than poisoning a total', () => {
115
+ ledgerMod.ledger.clear();
116
+ ledgerMod.record('k', { input_tokens: 'lots', output_tokens: -5, cache_read_input_tokens: NaN });
117
+ assert.deepEqual(ledgerMod.ledger.get('k'), {
118
+ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, calls: 1,
119
+ });
120
+ });
121
+
122
+ // ---- request dumps --------------------------------------------------------
123
+
124
+ const dumps = ledgerMod as unknown as {
125
+ makeDumper(dir: string, log?: (line: string) => void): null | {
126
+ dir: string;
127
+ begin(key: string | null, req: { url?: string; headers?: Record<string, unknown> }, raw: Buffer):
128
+ (info: Record<string, unknown>) => void;
129
+ };
130
+ replay(req: { headers: unknown; method: string; url: string }, raw: Buffer): NodeJS.ReadableStream & {
131
+ headers: unknown; method: string; url: string;
132
+ };
133
+ tee(res: Record<string, unknown>, key: string | null, observe?: (info: Record<string, unknown>) => void): void;
134
+ };
135
+
136
+ const scratch = () => mkdtempSync(join(tmpdir(), 'foreman-dump-'));
137
+
138
+ const fakeReq = (headers: Record<string, string>, url = '/v1/messages?beta=true') => (
139
+ { headers, method: 'POST', url }
140
+ );
141
+
142
+ test('dumping is a no-op when the directory is unset', () => {
143
+ // Off by default is the whole point: these files hold prompts and repository
144
+ // contents, and no one should find them on disk without having asked.
145
+ assert.equal(dumps.makeDumper(''), null);
146
+ assert.equal(dumps.makeDumper(undefined as unknown as string), null);
147
+ });
148
+
149
+ test('a request is written with its secrets redacted and its shape summarised', () => {
150
+ const dir = join(scratch(), 'nested', 'not-yet-made');
151
+ const lines: string[] = [];
152
+ const d = dumps.makeDumper(dir, (l) => lines.push(l))!;
153
+ const body = {
154
+ model: 'kimi-k3:cloud', stream: true, max_tokens: 8,
155
+ system: [{ type: 'text', text: 'abc' }, { type: 'text', text: 'de' }],
156
+ tools: [{ name: 'Read' }, { name: 'Edit' }, { name: 'Bash' }],
157
+ messages: [{ role: 'user', content: 'hi' }, { role: 'assistant', content: 'yo' }],
158
+ };
159
+ const raw = Buffer.from(JSON.stringify(body));
160
+ const finish = d.begin('run-a.0', fakeReq({
161
+ 'x-api-key': 'sk-secret', authorization: 'Bearer also-secret', 'content-type': 'application/json',
162
+ }), raw);
163
+
164
+ assert.ok(existsSync(dir), 'the dump dir is created on first use');
165
+ const files = readdirSync(dir).sort();
166
+ assert.equal(files.length, 1);
167
+ assert.match(files[0], /-run-a\.0-1\.json$/);
168
+ const got = JSON.parse(readFileSync(join(dir, files[0]), 'utf8'));
169
+ assert.equal(got.key, 'run-a.0');
170
+ assert.equal(got.url, '/v1/messages?beta=true');
171
+ assert.equal(got.headers['x-api-key'], '[redacted]');
172
+ assert.equal(got.headers.authorization, '[redacted]');
173
+ assert.equal(got.headers['content-type'], 'application/json');
174
+ assert.deepEqual(got.body, body);
175
+ assert.deepEqual(got.summary, {
176
+ model: 'kimi-k3:cloud', stream: true, systemChars: 5, toolCount: 3, messageCount: 2,
177
+ bodyBytes: raw.length,
178
+ });
179
+ assert.deepEqual(lines, ['[dump] run-a.0 -> kimi-k3:cloud tools=3 system=5 stream=true']);
180
+
181
+ finish({ status: 200, durationMs: 1234, bytes: 99, sawUsage: true, firstByteMs: 12 });
182
+ const after = readdirSync(dir).sort();
183
+ assert.deepEqual(after, [files[0], files[0].replace(/\.json$/, '.response.json')]);
184
+ assert.deepEqual(JSON.parse(readFileSync(join(dir, after[1]), 'utf8')), {
185
+ status: 200, durationMs: 1234, bytes: 99, sawUsage: true, firstByteMs: 12,
186
+ });
187
+ assert.equal(lines[1], '[dump] run-a.0 <- 200 in 1234ms 99B usage=true');
188
+ });
189
+
190
+ test('a malformed body is still dumped, as the raw string, and still reaches the handler', async () => {
191
+ // The bad body may be the very thing being diagnosed, so it is kept
192
+ // verbatim; and the dump must not be what turns it into a failure.
193
+ const dir = scratch();
194
+ const d = dumps.makeDumper(dir, () => {})!;
195
+ const raw = Buffer.from('{"model": "x", not json');
196
+ const req = fakeReq({ 'x-api-key': 'k' });
197
+ d.begin(null, req, raw);
198
+ const [file] = readdirSync(dir);
199
+ assert.match(file, /-unkeyed-1\.json$/);
200
+ const got = JSON.parse(readFileSync(join(dir, file), 'utf8'));
201
+ assert.equal(got.body, '{"model": "x", not json');
202
+ assert.equal(got.summary.model, null);
203
+ assert.equal(got.summary.bodyBytes, raw.length);
204
+
205
+ // What the handler would then read: the same bytes, under the same identity.
206
+ const r = dumps.replay(req, raw);
207
+ assert.equal(r.method, 'POST');
208
+ assert.equal(r.url, '/v1/messages?beta=true');
209
+ assert.equal(r.headers, req.headers);
210
+ const chunks: Buffer[] = [];
211
+ for await (const c of r as AsyncIterable<Buffer>) chunks.push(c);
212
+ assert.equal(Buffer.concat(chunks).toString(), raw.toString());
213
+ });
214
+
215
+ test('an unwritable dump dir does not fail the request', () => {
216
+ // A file where a directory should be: mkdir -p cannot succeed.
217
+ const blocker = join(scratch(), 'file');
218
+ require_('node:fs').writeFileSync(blocker, '');
219
+ const d = dumps.makeDumper(join(blocker, 'sub'), () => {})!;
220
+ const finish = d.begin('k', fakeReq({}), Buffer.from('{}'));
221
+ assert.equal(typeof finish, 'function');
222
+ assert.doesNotThrow(() => finish({ status: 200 }));
223
+ });
224
+
225
+ test('the tee reports the response outcome once, and still passes every byte through', () => {
226
+ const written: unknown[] = [];
227
+ const res = {
228
+ statusCode: 200,
229
+ write(chunk: unknown) { written.push(chunk); return true; },
230
+ end(chunk?: unknown) { if (chunk !== undefined) written.push(chunk); return this; },
231
+ };
232
+ const seen: Record<string, unknown>[] = [];
233
+ ledgerMod.ledger.clear();
234
+ dumps.tee(res as unknown as Record<string, unknown>, 'run-t.0', (info) => seen.push(info));
235
+ res.write('data: {"type":"message_start"}\n\n');
236
+ res.write(Buffer.from('data: {"type":"message_delta","usage":{"input_tokens":3,"output_tokens":1}}\n\n'));
237
+ res.end('data: [DONE]\n');
238
+
239
+ assert.equal(seen.length, 1);
240
+ assert.equal(seen[0].status, 200);
241
+ assert.equal(seen[0].sawUsage, true);
242
+ assert.equal(seen[0].bytes, written.reduce<number>((a, c) => a + Buffer.byteLength(c as string), 0));
243
+ assert.equal(typeof seen[0].durationMs, 'number');
244
+ assert.equal(typeof seen[0].firstByteMs, 'number');
245
+ assert.equal(written.length, 3, 'nothing is swallowed or duplicated');
246
+ assert.equal(ledgerMod.ledger.get('run-t.0')?.inputTokens, 3, 'counting still happens alongside');
247
+ });
248
+
249
+ test('an observer that throws cannot break the response', () => {
250
+ let ended = false;
251
+ const res = { statusCode: 200, write() { return true; }, end() { ended = true; return this; } };
252
+ dumps.tee(res as unknown as Record<string, unknown>, null, () => { throw new Error('boom'); });
253
+ assert.doesNotThrow(() => res.end());
254
+ assert.ok(ended);
255
+ });