@rikcodes/teamclaude 1.1.13-rik.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/server.js ADDED
@@ -0,0 +1,1319 @@
1
+ import http from 'node:http';
2
+ import https from 'node:https';
3
+ import { timingSafeEqual } from 'node:crypto';
4
+ import { createWriteStream } from 'node:fs';
5
+ import { mkdir } from 'node:fs/promises';
6
+ import { join } from 'node:path';
7
+ import { ensureCerts, createConnectHandler } from './mitm.js';
8
+ import { patchAccountUuid } from './account-uuid-rewrite.js';
9
+ import { sanitizeToolPairs } from './tool-pair-sanitize.js';
10
+ import { parseRequestModel, parseAdvisorModel } from './account-manager.js';
11
+ import { TopLevelFieldFinder, modelGlobMatches } from './model.js';
12
+ import { BodyWriter } from './request-log.js';
13
+ import { upstreamFetch } from './upstream-fetch.js';
14
+ import { tunnelTls } from './sx.js';
15
+ import { createEgressGuard } from './egress-guard.js';
16
+
17
+
18
+ export const HOP_BY_HOP_HEADERS = new Set([
19
+ 'host', 'connection', 'keep-alive', 'transfer-encoding',
20
+ 'te', 'trailer', 'upgrade', 'proxy-authorization', 'proxy-authenticate',
21
+ ]);
22
+ // Path prefix for the deprecated URL-based account pin (superseded by TC_ACCT).
23
+ const PIN_PREFIX = '/tc-acct/';
24
+ const INLINE_RETRY_AFTER_MAX_SECONDS = 15;
25
+ // How long the proxy will absorb a rate-limit 429's retry-after inline (waiting
26
+ // on the SAME account) before surfacing a 429 + retry-after to the client. A
27
+ // rate-limit 429 never rotates accounts (that just moves the burst); it pauses
28
+ // the account so concurrent requests wait, then retries the same account.
29
+ const RATE_LIMIT_ABSORB_MAX_SECONDS =
30
+ Number(process.env.TEAMCLAUDE_RATE_LIMIT_ABSORB_MAX_SECONDS) || 60;
31
+
32
+ // Response header names that are connection-specific and thus illegal on an
33
+ // HTTP/2 response (Node's Http2ServerResponse.writeHead rejects them). Also
34
+ // hop-by-hop on h1, so stripping them is correct on both paths.
35
+ const CONNECTION_SPECIFIC_HEADERS = new Set([
36
+ 'connection', 'keep-alive', 'transfer-encoding', 'upgrade',
37
+ 'proxy-connection', 'te', 'trailer',
38
+ ]);
39
+
40
+ // Constant-time proxy-API-key comparison (both the HTTP gate and the CONNECT
41
+ // gate use it). Returns false on any type/length mismatch without leaking timing.
42
+ export function safeKeyEqual(a, b) {
43
+ if (typeof a !== 'string' || typeof b !== 'string') return false;
44
+ const ba = Buffer.from(a);
45
+ const bb = Buffer.from(b);
46
+ if (ba.length !== bb.length) return false;
47
+ return timingSafeEqual(ba, bb);
48
+ }
49
+
50
+ // True if a socket's remote address is loopback — the proxy-key gate exempts
51
+ // localhost on both the HTTP and CONNECT paths.
52
+ export function isLoopbackAddr(addr) {
53
+ return addr === '127.0.0.1' || addr === '::1' || addr === '::ffff:127.0.0.1';
54
+ }
55
+
56
+ export function createProxyServer(accountManager, config, hooks = {}, sx = null) {
57
+ const upstream = config.upstream || 'https://api.anthropic.com';
58
+ const proxyApiKey = config.proxy?.apiKey;
59
+ const logDir = config.logDir || null;
60
+ const holdMs = (config.holdSeconds || 0) * 1000;
61
+
62
+ if (logDir) {
63
+ mkdir(logDir, { recursive: true }).catch(() => {});
64
+ }
65
+
66
+ const requestHandler = async (req, res) => {
67
+ try {
68
+ // Auth check — skip for localhost connections.
69
+ const clientKey = req.headers['x-api-key'];
70
+ const isLocal = isLoopbackAddr(req.socket.remoteAddress);
71
+ if (proxyApiKey && !safeKeyEqual(clientKey, proxyApiKey) && !isLocal) {
72
+ res.writeHead(401, { 'Content-Type': 'application/json' });
73
+ res.end(JSON.stringify({
74
+ type: 'error',
75
+ error: { type: 'authentication_error', message: 'Invalid proxy API key' },
76
+ }));
77
+ return;
78
+ }
79
+
80
+ // Control-plane mutations are refused when the request was issued by a web
81
+ // page. The gate above exempts loopback from the API key, so without this
82
+ // any site the operator happens to visit can POST here cross-origin: a
83
+ // `fetch(..., {mode:'no-cors', body})` with a text/plain content type is a
84
+ // CORS "simple request", so no preflight is sent and the request lands.
85
+ // The page cannot read the reply, but the side effect is the point —
86
+ // forcing the whole fleet onto one named account is a targeted quota
87
+ // drain, and reload is reachable the same way.
88
+ //
89
+ // Origin (and Sec-Fetch-Site) are set by the browser and cannot be
90
+ // forged from page JavaScript, while curl and the CLI send neither — so
91
+ // this costs legitimate callers nothing. Deliberately not a content-type
92
+ // requirement, which would also close the hole but would break the
93
+ // documented `curl -X POST .../teamclaude/reload` that sends no body.
94
+ if (req.method === 'POST' && (req.url || '').startsWith('/teamclaude/')
95
+ && !isSameOriginControlRequest(req)) {
96
+ res.writeHead(403, { 'Content-Type': 'application/json' });
97
+ res.end(JSON.stringify({
98
+ ok: false,
99
+ error: 'cross-origin request refused: the control plane is not reachable from a web page',
100
+ }));
101
+ return;
102
+ }
103
+
104
+ // Forward-proxy request (HTTP_PROXY): an absolute-form URL is a tool
105
+ // proxying plain HTTP to some host. Account logic is only for hosts we
106
+ // manage (the Anthropic upstream, which is HTTPS-only and never arrives
107
+ // this way); forward anything else transparently instead of hijacking it.
108
+ if (/^https?:\/\//i.test(req.url || '')) { relayHttpForward(req, res); return; }
109
+
110
+ // Status endpoint
111
+ if (req.method === 'GET' && req.url === '/teamclaude/status') {
112
+ const status = accountManager.getStatus();
113
+ const extra = hooks.getStatusExtra?.() || {};
114
+ res.writeHead(200, { 'Content-Type': 'application/json' });
115
+ res.end(JSON.stringify({ ...extra, ...status }, null, 2));
116
+ return;
117
+ }
118
+
119
+ // Reload endpoint — re-sync accounts from config without a restart. This
120
+ // is the headless equivalent of pressing 'R' in the TUI. Local control
121
+ // only (no upstream calls); the auth gate above already applies.
122
+ if (req.method === 'POST' && req.url === '/teamclaude/reload') {
123
+ if (!hooks.reload) {
124
+ res.writeHead(501, { 'Content-Type': 'application/json' });
125
+ res.end(JSON.stringify({ ok: false, error: 'reload not supported' }));
126
+ return;
127
+ }
128
+ try {
129
+ const added = await hooks.reload();
130
+ res.writeHead(200, { 'Content-Type': 'application/json' });
131
+ res.end(JSON.stringify({ ok: true, added: added || 0 }));
132
+ } catch (err) {
133
+ res.writeHead(500, { 'Content-Type': 'application/json' });
134
+ res.end(JSON.stringify({ ok: false, error: err.message }));
135
+ }
136
+ return;
137
+ }
138
+
139
+ // Switch endpoint — make one account the preferred one, the headless
140
+ // equivalent of picking it with 's' in the TUI. Both do the same single
141
+ // thing: move currentIndex. That is a preference, and a weak one: _select
142
+ // abandons it as soon as the account is unavailable, and also whenever any
143
+ // available account carries a strictly lower priority value. So the answer
144
+ // reports whether the choice will actually take effect rather than only
145
+ // that it was recorded. Body:
146
+ // {"account": "<name|email|accountUuid|accountUuid/orgUuid|orgUuid>"}.
147
+ // Local control only (no upstream calls); the auth gate above applies.
148
+ if (req.method === 'POST' && req.url === '/teamclaude/switch') {
149
+ const names = () => (accountManager.accounts || []).map(a => a.name);
150
+ let target;
151
+ try {
152
+ const raw = await readControlBody(req);
153
+ target = JSON.parse(raw || '{}')?.account;
154
+ } catch (err) {
155
+ // Say which of the two it was, but never echo the parser's own message
156
+ // back to a caller — that is our internals, not their input.
157
+ const tooLarge = err.message === 'body too large';
158
+ res.writeHead(tooLarge ? 413 : 400, { 'Content-Type': 'application/json' });
159
+ res.end(JSON.stringify({ ok: false, error: tooLarge ? 'request body too large' : 'invalid request body' }));
160
+ return;
161
+ }
162
+ if (typeof target !== 'string' || !target.trim()) {
163
+ res.writeHead(400, { 'Content-Type': 'application/json' });
164
+ res.end(JSON.stringify({ ok: false, error: 'missing "account"', accounts: names() }));
165
+ return;
166
+ }
167
+ const index = resolveAccountPin(accountManager, target);
168
+ if (index == null) {
169
+ res.writeHead(404, { 'Content-Type': 'application/json' });
170
+ res.end(JSON.stringify({ ok: false, error: `no such account "${target}"`, accounts: names() }));
171
+ return;
172
+ }
173
+ accountManager.currentIndex = index;
174
+ const name = accountManager.accounts[index].name;
175
+ // Recording the choice and the choice taking effect are two different
176
+ // things: selection skips an account it cannot use on the very next
177
+ // request, so a bare "ok" would be a lie for a disabled or spent target.
178
+ // The switch still happens (that is the TUI's behaviour) and the answer
179
+ // says whether traffic will follow it.
180
+ const { eligible, reason } = accountManager.eligibility(index);
181
+ // Leave a trace where every other account change already leaves one: the
182
+ // TUI swaps console.log for its activity pane and headless mode tees it
183
+ // to the activity log, so this one line covers both. Without it a manual
184
+ // switch is the only account change that happens invisibly — on exactly
185
+ // the background-service deployment this endpoint exists for.
186
+ console.log(`[TeamClaude] Switched to account "${name}" (manual)`
187
+ + (eligible ? '' : ` — ${reason}, so rotation will not use it`));
188
+ res.writeHead(200, { 'Content-Type': 'application/json' });
189
+ res.end(JSON.stringify({ ok: true, account: name, eligible, ...(reason ? { reason } : {}) }));
190
+ return;
191
+ }
192
+
193
+ return forward(req, res);
194
+ } catch (err) {
195
+ console.error('[TeamClaude] Unhandled error:', err);
196
+ }
197
+ };
198
+
199
+ // Opt-in egress pin: null unless config.egress.pin is set, and then shared by
200
+ // the base listener and the MITM one so both honour the same hold.
201
+ const egress = createEgressGuard(config, console.error);
202
+ const forward = createProxyRequestListener({ accountManager, upstream, logDir, hooks, sx, holdMs, config, egress });
203
+ const server = http.createServer(requestHandler);
204
+
205
+ // Forward-proxy support (always on, so multiple claude instances can use
206
+ // either ANTHROPIC_BASE_URL or HTTPS_PROXY against the same server). A CONNECT
207
+ // to the upstream host is a transparent MITM relay (rewrite only auth); the
208
+ // test host is answered locally; anything else is blind-tunneled. Certs are
209
+ // minted lazily on the first intercepted CONNECT.
210
+ const mitmHost = (() => { try { return new URL(upstream).hostname; } catch { return 'api.anthropic.com'; } })();
211
+ let certsPromise = null;
212
+ const ensureLeaf = async () => {
213
+ // Reset the memo on failure so a transient cert error doesn't wedge the MITM
214
+ // path permanently (a cached rejected promise would re-throw on every CONNECT).
215
+ certsPromise ||= ensureCerts(mitmHost).catch((err) => { certsPromise = null; throw err; });
216
+ const c = await certsPromise;
217
+ return { key: c.leafKeyPem, cert: c.leafCertPem };
218
+ };
219
+ server.on('connect', createConnectHandler({ config, accountManager, ensureLeaf, logDir, hooks, log: console.error, sx, egress }));
220
+ // Remote Control's real-time channel is a WebSocket, not a request/response
221
+ // call — Node fires 'upgrade' for that handshake, never 'request', so it
222
+ // needs its own listener (base-URL routing path; the MITM path wires the
223
+ // same relayUpgrade onto its own terminating server in mitm.js).
224
+ server.on('upgrade', (req, socket, head) => relayUpgrade(req, socket, head, upstream, sx));
225
+
226
+ return server;
227
+ }
228
+
229
+ /**
230
+ * Whether a control-plane POST did NOT come from a web page.
231
+ *
232
+ * Both headers are browser-set and unforgeable from page JavaScript:
233
+ * - `Sec-Fetch-Site` is the explicit answer where it exists (Chrome, Safari,
234
+ * Firefox). Anything but `same-origin` / `none` is a page reaching across.
235
+ * - `Origin` is the fallback for browsers that send no Sec-Fetch-Site. Its
236
+ * mere presence on a POST to a local control endpoint means a page issued
237
+ * it; matching it against our own host would mean guessing which of
238
+ * localhost / 127.0.0.1 / [::1] / a LAN address the caller used, and a
239
+ * browser-issued same-origin call is not a thing worth supporting here.
240
+ *
241
+ * Non-browser callers (curl, the CLI, `teamclaude attach`) send neither and are
242
+ * unaffected.
243
+ */
244
+ export function isSameOriginControlRequest(req) {
245
+ const site = req.headers['sec-fetch-site'];
246
+ if (site) return site === 'same-origin' || site === 'none';
247
+ return !req.headers.origin;
248
+ }
249
+
250
+ // Read a control-endpoint body as text. Capped, unlike the proxied request path:
251
+ // these endpoints carry a couple of fields, so anything larger is a mistake or an
252
+ // attack and buffering it whole would be the wrong answer either way.
253
+ async function readControlBody(req, limit = 64 * 1024) {
254
+ const chunks = [];
255
+ let size = 0;
256
+ for await (const chunk of req) {
257
+ size += chunk.length;
258
+ if (size > limit) throw new Error('body too large');
259
+ chunks.push(chunk);
260
+ }
261
+ return Buffer.concat(chunks).toString('utf8');
262
+ }
263
+
264
+ /**
265
+ * Resolve an account pin to an index, or null.
266
+ *
267
+ * Accepted forms, first match wins:
268
+ * - `accountUuid/orgUuid` — fully qualified, the only form that distinguishes
269
+ * one person's accounts across several orgs
270
+ * - `accountUuid`
271
+ * - `orgUuid`
272
+ * - the display name (`email` or `email (Org)`), or the bare email
273
+ *
274
+ * UUIDs are the identity to use for anything scripted or long-lived: display
275
+ * names are rewritten in place when an email gains a second org (see
276
+ * accountsCommand), so a name is a convenience, not an identifier.
277
+ *
278
+ * The rotation index is deliberately NOT accepted. It is array position, so
279
+ * deleting an account would silently repoint every later pin at a DIFFERENT
280
+ * account — a wrong-account misroute rather than an honest failure.
281
+ */
282
+ export function resolveAccountPin(accountManager, token) {
283
+ const accounts = accountManager.accounts || [];
284
+ const norm = (s) => (s || '').trim().toLowerCase();
285
+ const t = norm(token);
286
+ if (!t) return null;
287
+
288
+ const at = (pick) => accounts.findIndex(a => norm(pick(a)) === t);
289
+ const qualified = accounts.findIndex(a => a.accountUuid && a.orgUuid
290
+ && `${norm(a.accountUuid)}/${norm(a.orgUuid)}` === t);
291
+
292
+ for (const i of [
293
+ qualified,
294
+ at(a => a.accountUuid),
295
+ at(a => a.orgUuid),
296
+ at(a => a.name),
297
+ at(a => (a.name || '').split(' (')[0]), // display name minus the org suffix
298
+ ]) if (i >= 0) return i;
299
+
300
+ return null;
301
+ }
302
+
303
+ // Paths that must reach upstream with the client's own credential (never a
304
+ // rotated account token): the Remote Control channel and attachment transfers.
305
+ // teamclaude applies its account logic (rotation, exhaustion, token injection)
306
+ // ONLY to hosts it manages — the Anthropic upstream. Anything else must be
307
+ // forwarded transparently, never hijacked into "all accounts exhausted". For
308
+ // HTTPS this is already true (the CONNECT tunnel in mitm.js blind-relays
309
+ // non-upstream hosts). This is the plain-HTTP counterpart: a tool honoring
310
+ // HTTP_PROXY sends an ABSOLUTE-form request (`GET http://host/path`), which
311
+ // otherwise gets misrouted to Anthropic. Blind-relay it to its target with the
312
+ // client's own headers — no account selection, no token injection,
313
+ // content-encoding passed through (a transparent forward proxy). Anthropic is
314
+ // HTTPS-only, so in practice this only ever sees third-party hosts.
315
+ export function relayHttpForward(req, res) {
316
+ let target;
317
+ try { target = new URL(req.url); } catch {
318
+ res.writeHead(400, { 'Content-Type': 'application/json' });
319
+ res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'Malformed forward-proxy URL' } }));
320
+ return;
321
+ }
322
+ const transport = target.protocol === 'http:' ? http : https;
323
+ const headers = {};
324
+ for (const [key, value] of Object.entries(req.headers)) {
325
+ const lk = key.toLowerCase();
326
+ // Drop hop-by-hop + proxy-control headers; `host` is reset from the target.
327
+ if (lk.startsWith(':') || HOP_BY_HOP_HEADERS.has(lk) || lk === 'proxy-connection') continue;
328
+ headers[key] = value;
329
+ }
330
+
331
+ const upstreamReq = transport.request(target, { method: req.method, headers }, (upstreamRes) => {
332
+ const responseHeaders = {};
333
+ for (const [key, value] of Object.entries(upstreamRes.headers)) {
334
+ if (CONNECTION_SPECIFIC_HEADERS.has(key)) continue;
335
+ responseHeaders[key] = value;
336
+ }
337
+ res.writeHead(upstreamRes.statusCode, responseHeaders);
338
+ upstreamRes.pipe(res);
339
+ });
340
+ upstreamReq.on('error', (err) => {
341
+ console.error(`[TeamClaude] HTTP forward to ${target.host} failed:`, err.message);
342
+ if (!res.headersSent) {
343
+ res.writeHead(502, { 'Content-Type': 'application/json' });
344
+ res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
345
+ }
346
+ });
347
+ res.on('close', () => upstreamReq.destroy());
348
+ if (['GET', 'HEAD'].includes(req.method)) upstreamReq.end();
349
+ else req.pipe(upstreamReq);
350
+ }
351
+
352
+ const CLIENT_CREDENTIAL_PATHS = ['/v1/code/', '/api/oauth/files/', '/api/oauth/file_upload'];
353
+
354
+ /**
355
+ * Build the core proxy request listener — buffer the body, then forward with
356
+ * account selection + retry (forwardRequest). Shared by the base HTTP server and
357
+ * the MITM's terminating h2/h1 server, so both get identical buffering, model-
358
+ * aware routing, and retry-on-quota behavior. Control endpoints (status/reload)
359
+ * and the proxy-API-key gate live in the base server's wrapper, not here.
360
+ */
361
+ export function createProxyRequestListener({ accountManager, upstream, logDir = null, hooks = {}, sx = null, holdMs = 0, config = {}, forcedPin = null, egress = null }) {
362
+ let counter = 0;
363
+ return async (req, res) => {
364
+ try {
365
+ // Claude Code's telemetry (`/api/event_logging/*`) is high-volume noise in
366
+ // the activity log. `config.eventLogging` (read live so the TUI toggle takes
367
+ // effect immediately): 'show' forwards + displays; 'hide' (default) forwards
368
+ // but suppresses the activity entry; 'block' answers 200 locally without
369
+ // forwarding (no upstream round-trip, no account/token spent).
370
+ const eventLogging = config?.eventLogging || 'hide';
371
+ const isEventLog = (req.url || '').startsWith('/api/event_logging');
372
+ if (isEventLog && eventLogging === 'block') {
373
+ res.writeHead(200, { 'Content-Type': 'application/json' });
374
+ res.end('{}');
375
+ return;
376
+ }
377
+ const hideActivity = isEventLog && eventLogging !== 'show';
378
+ // Egress pin (opt-in): with the exit IP off the pinned one — a VPN that
379
+ // dropped — hold rather than send. Upstream answers a request from an
380
+ // unexpected region with a 403 that Claude Code reports as a dead session,
381
+ // so sending it costs a re-login while waiting costs latency. Checked here
382
+ // rather than per-account: it is a property of the connection, and this is
383
+ // the one path every request takes, MITM included.
384
+ if (egress?.enabled()) {
385
+ const state = await egress.waitUntilPinned({ isAborted: () => res.destroyed });
386
+ if (res.destroyed) return;
387
+ if (!state.ok) {
388
+ res.writeHead(503, { 'Content-Type': 'application/json', 'retry-after': '30' });
389
+ res.end(JSON.stringify({
390
+ type: 'error',
391
+ error: {
392
+ type: 'proxy_error',
393
+ message: `Egress is ${state.ip || 'unknown'}, not the pinned ${state.expected.join(', ')} — not sending this request. Check the VPN.`,
394
+ },
395
+ }));
396
+ return;
397
+ }
398
+ }
399
+ // Client token refresh: pass through untouched (the proxy manages its own
400
+ // tokens via ensureTokenFresh; rewriting client refreshes would conflict).
401
+ if (req.method === 'POST' && req.url === '/v1/oauth/token') { await relayRaw(req, res, upstream, sx); return; }
402
+ // Remote Control (/v1/code/*) is bound to the session's paired claude.ai
403
+ // identity — forward with the client's OWN credential (streamed), never a
404
+ // rotated account token, which would 403 the worker event stream.
405
+ // Attachment transfers (/api/oauth/files/*, /api/oauth/file_upload) are
406
+ // likewise account-bound: files uploaded from claude.ai belong to the
407
+ // paired identity, so fetching them with a rotated token 403s and Claude
408
+ // Code silently drops the image from the message.
409
+ if (CLIENT_CREDENTIAL_PATHS.some((p) => (req.url || '').startsWith(p))) { await relayStream(req, res, upstream, sx); return; }
410
+
411
+ // Account pin: a request to `/tc-acct/<name-or-index>/...` (e.g. via
412
+ // ANTHROPIC_BASE_URL=http://host:port/tc-acct/deepseek) is forced onto that
413
+ // one account, bypassing rotation. Used by the keep-warm scheduler and for
414
+ // manual per-account testing. The prefix is stripped before forwarding.
415
+ let pinnedIndex = null;
416
+ // DEPRECATED: the path-prefix pin. Superseded by TC_ACCT, which works in
417
+ // MITM mode too (this form cannot — inside a CONNECT tunnel the path is
418
+ // the real upstream one). Kept for the warmer and for direct API callers.
419
+ // One segment only, so the fully-qualified `accountUuid/orgUuid` form is
420
+ // not expressible here; use TC_ACCT for that.
421
+ const url = req.url || '';
422
+ const afterPrefix = url.startsWith(PIN_PREFIX) ? url.slice(PIN_PREFIX.length) : null;
423
+ // The token runs to the next '/', which also begins the real request path.
424
+ const tokenEnd = afterPrefix == null ? -1 : afterPrefix.indexOf('/');
425
+ if (tokenEnd > 0) {
426
+ const token = decodeURIComponent(afterPrefix.slice(0, tokenEnd));
427
+ pinnedIndex = resolveAccountPin(accountManager, token);
428
+ if (pinnedIndex == null) {
429
+ const reqId = ++counter;
430
+ const sessionId = req.headers['x-claude-code-session-id'] || null;
431
+ if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: `(unknown pin: "${token}")`, status: 404, model: null, sessionId, pinned: false });
432
+ res.writeHead(404, { 'Content-Type': 'application/json' });
433
+ res.end(JSON.stringify({ type: 'error', error: { type: 'not_found_error', message: `Unknown account pin "${token}"` } }));
434
+ return;
435
+ }
436
+ req.url = afterPrefix.slice(tokenEnd);
437
+ }
438
+
439
+ // MITM-mode pin. A CONNECT carrying `Proxy-Authorization: Basic <acct>:…`
440
+ // has no URL to hang a `/tc-acct/` prefix on — the path inside the tunnel
441
+ // is the real Anthropic one — so the pin arrives as a listener bound to
442
+ // that account (see createConnectHandler). Resolved per request rather
443
+ // than at CONNECT time: a hot reload can renumber accounts while a tunnel
444
+ // is open, and a name outliving an index is the safer half of that race.
445
+ if (pinnedIndex == null && forcedPin != null) {
446
+ pinnedIndex = resolveAccountPin(accountManager, forcedPin);
447
+ if (pinnedIndex == null) {
448
+ const reqId = ++counter;
449
+ const sessionId = req.headers['x-claude-code-session-id'] || null;
450
+ if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: `(unknown pin: "${forcedPin}")`, status: 404, model: null, sessionId, pinned: false });
451
+ res.writeHead(404, { 'Content-Type': 'application/json' });
452
+ res.end(JSON.stringify({ type: 'error', error: { type: 'not_found_error', message: `Unknown account pin "${forcedPin}" (from TC_ACCT)` } }));
453
+ return;
454
+ }
455
+ }
456
+
457
+ const reqId = ++counter;
458
+ // Claude Code tags each session's requests with this header (present on
459
+ // /v1/messages and count_tokens). Read from headers up front so it drives
460
+ // session-aware routing (issue #109) and colors the TUI activity stream.
461
+ const sessionId = req.headers['x-claude-code-session-id'] || null;
462
+ if (!hideActivity) hooks.onRequestStart?.(reqId, { method: req.method, path: req.url, sessionId, pinned: pinnedIndex != null });
463
+
464
+ // Buffer request body (needed to resend on a different account after a 429).
465
+ // Peek the top-level `model` field incrementally as chunks arrive so the
466
+ // TUI can show it the instant it appears in the stream — usually the first
467
+ // frame — rather than waiting for the whole body and the request to finish.
468
+ const bodyChunks = [];
469
+ const modelFinder = new TopLevelFieldFinder('model');
470
+ for await (const chunk of req) {
471
+ bodyChunks.push(chunk);
472
+ if (!modelFinder.done) {
473
+ const found = modelFinder.push(chunk);
474
+ if (found && !hideActivity) hooks.onRequestModel?.(reqId, { model: found });
475
+ }
476
+ }
477
+ const body = Buffer.concat(bodyChunks);
478
+
479
+ const model = modelFinder.done ? modelFinder.value : parseRequestModel(body);
480
+ // An advisor request (Claude Code's advisor tool) carries a SECOND model
481
+ // nested in tools[]; the advisor sub-inference runs on the selected
482
+ // account, so selection must be eligible for it too (issue #98).
483
+ const advisorModel = parseAdvisorModel(body);
484
+
485
+ // Model blocklist (issue #116): reject a request for a blocked model right
486
+ // here instead of forwarding it. A model no account can serve (e.g. Fable
487
+ // once it left base plans) otherwise gets rate-limited upstream and hangs
488
+ // the pipeline; a fast, non-retryable 400 lets the client move on. Read
489
+ // live from the shared config so the TUI editor takes effect immediately.
490
+ const blockedBy = model ? (config?.blockedModels || []).find((p) => modelGlobMatches(p, model)) : null;
491
+ if (blockedBy) {
492
+ if (!res.headersSent) {
493
+ res.writeHead(400, { 'Content-Type': 'application/json' });
494
+ res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: `Model "${model}" is blocked by teamclaude (matched "${blockedBy}").` } }));
495
+ }
496
+ hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: '(blocked)', status: 400, model, sessionId });
497
+ return;
498
+ }
499
+
500
+ const ctx = { account: null, status: null, tried: new Set(), reauthed: new Set(), model, advisorModel, pinnedIndex, holdBudgetMs: holdMs, sessionId };
501
+ // Hold the session "in flight" across the WHOLE request (incl. retries and
502
+ // a multi-minute streaming completion) so it stays counted as active and
503
+ // never expires mid-request.
504
+ accountManager.beginSession(sessionId);
505
+ try {
506
+ await forwardRequest(req, res, body, accountManager, upstream, 0, hooks, reqId, ctx, logDir, sx);
507
+ } catch (err) {
508
+ ctx.status = ctx.status || 502;
509
+ console.error('[TeamClaude] Unhandled error:', err);
510
+ if (!res.headersSent) {
511
+ res.writeHead(502, { 'Content-Type': 'application/json' });
512
+ res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Internal proxy error' } }));
513
+ }
514
+ } finally {
515
+ accountManager.endSession(sessionId);
516
+ if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: ctx.account, status: ctx.status, model: ctx.model, sessionId, pinned: ctx.pinnedIndex != null });
517
+ }
518
+ } catch (err) {
519
+ console.error('[TeamClaude] Unhandled error:', err);
520
+ }
521
+ };
522
+ }
523
+
524
+ // Per-request https.Agent tunneled through sx.org — one-shot (no keep-alive
525
+ // reuse, matching upstream-fetch.js's proxiedFetch), so a fresh sx tunnel is
526
+ // dialed for this connection only.
527
+ function sxAgent(sx, targetHost) {
528
+ const proxy = sx.getProxy();
529
+ const agent = new https.Agent({ keepAlive: false });
530
+ agent.createConnection = (_options, cb) => {
531
+ tunnelTls({ proxy, targetHost, targetPort: 443, tlsOptions: sx.tlsOptions || {} })
532
+ .then((sock) => cb(null, sock))
533
+ .catch((err) => cb(err));
534
+ return undefined;
535
+ };
536
+ return agent;
537
+ }
538
+
539
+ /**
540
+ * Relay a request to upstream with the client's OWN headers intact (including
541
+ * its authorization) — used for Remote Control (/v1/code/*), whose event
542
+ * stream is a long-poll: the client keeps the request open indefinitely and
543
+ * the upstream may withhold response headers for minutes between events. No
544
+ * buffering, no timeout, no reconstruction — just pipe bytes both ways as they
545
+ * arrive, exactly like a transparent proxy would.
546
+ */
547
+ function relayStream(req, res, upstream, sx) {
548
+ const target = new URL(`${upstream}${req.url}`);
549
+ const headers = {};
550
+ for (const [key, value] of Object.entries(req.headers)) {
551
+ const lk = key.toLowerCase();
552
+ if (lk.startsWith(':') || HOP_BY_HOP_HEADERS.has(lk) || lk === 'accept-encoding') continue;
553
+ headers[key] = value;
554
+ }
555
+
556
+ const useProxy = !!(sx?.useByDefault() && sx.isProvisioned());
557
+ const agent = useProxy ? sxAgent(sx, target.hostname) : undefined;
558
+ const transport = target.protocol === 'http:' ? http : https;
559
+
560
+ const upstreamReq = transport.request(target, { method: req.method, headers, agent }, (upstreamRes) => {
561
+ const responseHeaders = {};
562
+ for (const [key, value] of Object.entries(upstreamRes.headers)) {
563
+ if (CONNECTION_SPECIFIC_HEADERS.has(key) || key === 'content-encoding' || key === 'content-length') continue;
564
+ responseHeaders[key] = value;
565
+ }
566
+ res.writeHead(upstreamRes.statusCode, responseHeaders);
567
+ upstreamRes.pipe(res);
568
+ });
569
+
570
+ upstreamReq.on('error', (err) => {
571
+ console.error('[TeamClaude] Remote Control relay error:', err.message);
572
+ if (!res.headersSent) {
573
+ res.writeHead(502, { 'Content-Type': 'application/json' });
574
+ res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
575
+ }
576
+ });
577
+ // Client disconnected (e.g. Claude Code closed the channel): tear down the
578
+ // upstream side too instead of leaking an open connection.
579
+ res.on('close', () => upstreamReq.destroy());
580
+
581
+ if (['GET', 'HEAD'].includes(req.method)) upstreamReq.end();
582
+ else req.pipe(upstreamReq);
583
+ }
584
+
585
+ /**
586
+ * Relay a WebSocket upgrade (e.g. Remote Control's real-time
587
+ * `/v1/session_ingress/ws/*` channel) to upstream with the client's own
588
+ * headers intact. An HTTP server never emits 'request' for an Upgrade
589
+ * handshake — only 'upgrade', with a raw socket instead of a response object —
590
+ * so this needs its own relay rather than going through relayStream/res.
591
+ * Reuses Node's http(s) client, which already knows how to speak the Upgrade
592
+ * handshake (emits its own 'upgrade' event on a 101); once that fires it's
593
+ * just two raw sockets spliced together.
594
+ */
595
+ export function relayUpgrade(req, socket, head, upstream, sx) {
596
+ const target = new URL(`${upstream}${req.url}`);
597
+ const headers = {};
598
+ for (const [key, value] of Object.entries(req.headers)) {
599
+ const lk = key.toLowerCase();
600
+ // Unlike relayStream, do NOT strip 'upgrade'/'connection' here — they ARE
601
+ // the handshake. Only 'host' (the client transport reconstructs it from
602
+ // `target`) and h2 pseudo-headers are dropped.
603
+ if (lk.startsWith(':') || lk === 'host') continue;
604
+ headers[key] = value;
605
+ }
606
+
607
+ const useProxy = !!(sx?.useByDefault() && sx.isProvisioned());
608
+ const agent = useProxy ? sxAgent(sx, target.hostname) : undefined;
609
+ const transport = target.protocol === 'http:' ? http : https;
610
+
611
+ const upstreamReq = transport.request(target, { method: req.method, headers, agent });
612
+
613
+ upstreamReq.on('upgrade', (upstreamRes, upstreamSocket, upstreamHead) => {
614
+ const headerLines = Object.entries(upstreamRes.headers)
615
+ .map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(', ') : v}`).join('\r\n');
616
+ socket.write(`HTTP/1.1 ${upstreamRes.statusCode} ${upstreamRes.statusMessage}\r\n${headerLines}\r\n\r\n`);
617
+ if (upstreamHead?.length) socket.write(upstreamHead);
618
+ if (head?.length) upstreamSocket.write(head);
619
+ socket.pipe(upstreamSocket);
620
+ upstreamSocket.pipe(socket);
621
+ // An upgraded socket defaults to half-open: the peer's FIN only ends the
622
+ // READABLE side ('end'), it does NOT destroy the socket or fire 'close' —
623
+ // so without this, one side hanging up (dropped wifi, killed CLI) leaves
624
+ // the other socket open forever. destroy() is idempotent, so reacting to
625
+ // both 'end' and 'close' on each side is a safe, redundant backstop.
626
+ socket.on('end', () => upstreamSocket.destroy());
627
+ upstreamSocket.on('end', () => socket.destroy());
628
+ socket.on('close', () => upstreamSocket.destroy());
629
+ upstreamSocket.on('close', () => socket.destroy());
630
+ // The 101 detaches this socket from upstreamReq, so the request's 'error'
631
+ // listener no longer covers it. A link that flaps mid-session then raises
632
+ // 'error' (write EPIPE / read ECONNRESET) on a socket nobody listens to,
633
+ // which Node escalates to an uncaught exception — one dropped WebSocket
634
+ // would kill the proxy for every other session. Close the pair instead.
635
+ upstreamSocket.on('error', () => socket.destroy());
636
+ });
637
+
638
+ upstreamReq.on('error', (err) => {
639
+ console.error('[TeamClaude] Remote Control WebSocket relay error:', err.message);
640
+ socket.destroy();
641
+ });
642
+ socket.on('error', () => upstreamReq.destroy());
643
+
644
+ upstreamReq.end();
645
+ }
646
+
647
+ /**
648
+ * Relay a request to upstream with no header rewriting — pure passthrough.
649
+ */
650
+ async function relayRaw(req, res, upstream, sx) {
651
+ const bodyChunks = [];
652
+ for await (const chunk of req) bodyChunks.push(chunk);
653
+ const body = Buffer.concat(bodyChunks);
654
+
655
+ try {
656
+ const upstreamRes = await upstreamFetch(`${upstream}${req.url}`, {
657
+ method: req.method,
658
+ headers: {
659
+ 'content-type': req.headers['content-type'] || 'application/json',
660
+ 'accept': req.headers['accept'] || 'application/json',
661
+ 'user-agent': req.headers['user-agent'] || 'node',
662
+ },
663
+ body: body.length > 0 ? body : undefined,
664
+ }, sx, sx?.useByDefault());
665
+
666
+ const responseBody = await upstreamRes.text();
667
+ const responseHeaders = {};
668
+ for (const [key, value] of upstreamRes.headers.entries()) {
669
+ // `.text()` already decompressed the body, so drop content-encoding and
670
+ // the now-stale content-length (both refer to the compressed bytes) — else
671
+ // a gzip'd upstream response reaches the client mis-framed / truncated.
672
+ if (key === 'transfer-encoding' || key === 'connection' ||
673
+ key === 'content-encoding' || key === 'content-length') continue;
674
+ responseHeaders[key] = value;
675
+ }
676
+ res.writeHead(upstreamRes.status, responseHeaders);
677
+ res.end(responseBody);
678
+ } catch (err) {
679
+ console.error('[TeamClaude] Raw relay error:', err.message);
680
+ if (!res.headersSent) {
681
+ res.writeHead(502, { 'Content-Type': 'application/json' });
682
+ res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
683
+ }
684
+ }
685
+ }
686
+
687
+
688
+ function logTimestamp() {
689
+ const d = new Date();
690
+ const pad = (n, w = 2) => String(n).padStart(w, '0');
691
+ return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}.${pad(d.getMilliseconds(), 3)}`;
692
+ }
693
+
694
+ // A per-request log that streams to disk as the request/response flow, instead
695
+ // of buffering the whole body in memory and writing once at the end. The file
696
+ // is opened on first write; header sections are written verbatim and bodies are
697
+ // streamed through BodyWriter (JSON pretty-printed on the fly, SSE/other raw),
698
+ // so even a ~1M-token response costs only the current chunk.
699
+ function openRequestLog(logDir, reqId) {
700
+ const filename = `${logTimestamp()}_${String(reqId).padStart(5, '0')}.log`;
701
+ const ws = createWriteStream(join(logDir, filename), { flags: 'a' });
702
+ ws.on('error', (err) => console.error(`[TeamClaude] Failed to write log: ${err.message}`));
703
+ let ended = false;
704
+ const write = (s) => { if (!ended && s) ws.write(Buffer.from(String(s), 'latin1')); };
705
+ return {
706
+ write,
707
+ // Stream a complete body buffer under a section header.
708
+ body(label, buf, contentType) {
709
+ if (!buf || !buf.length) { write(`\n\n=== ${label} ===\n(empty)`); return; }
710
+ new BodyWriter(write, label, contentType || '').chunk(buf);
711
+ },
712
+ // A BodyWriter to append chunks incrementally (e.g. an SSE response).
713
+ bodyWriter(label, contentType) { return new BodyWriter(write, label, contentType || ''); },
714
+ end() { if (!ended) { ended = true; ws.end('\n'); } },
715
+ };
716
+ }
717
+
718
+ function formatHeaders(headers) {
719
+ if (headers.entries) {
720
+ return [...headers.entries()].map(([k, v]) => ` ${k}: ${v}`).join('\n');
721
+ }
722
+ return Object.entries(headers).map(([k, v]) => ` ${k}: ${v}`).join('\n');
723
+ }
724
+
725
+ export async function forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, useSx) {
726
+ const maxRetries = accountManager.accounts.length;
727
+ // This function is exported, so a caller may hand us a ctx built elsewhere.
728
+ // The 401 path reads ctx.reauthed on every response; default it here rather
729
+ // than trusting every construction site to include it.
730
+ ctx.reauthed ??= new Set();
731
+ // Whether THIS attempt dials via sx.org. Undefined on the first call → derive
732
+ // from the default policy ('always' routes; 'off'/'429' start direct).
733
+ const route = useSx === undefined ? !!(sx?.useByDefault()) : useSx;
734
+
735
+ // Select account, skipping any already tried (and failed) this request.
736
+ // The model scopes availability so a Fable-exhausted account is skipped only
737
+ // for Fable requests (it still serves other models).
738
+ // A pinned request (via /tc-acct/<name>) forces one exact account and never
739
+ // rotates or fails over: once that account has been tried, `account` is null
740
+ // and the caller gets the exhausted response rather than leaking to another.
741
+ const account = ctx.pinnedIndex != null
742
+ ? (ctx.tried.has(ctx.pinnedIndex) ? null : accountManager.accounts[ctx.pinnedIndex])
743
+ : accountManager.getActiveAccount(ctx.tried, ctx.model, ctx.advisorModel, ctx.sessionId);
744
+ if (!account) {
745
+ // Every candidate was refused by upstream (403). Waiting will not help — the
746
+ // account needs attention, not a retry — so say so plainly rather than
747
+ // reporting a rate limit. Not a 403 either: the client's own credential is
748
+ // fine, and a 403 would make it drop its login over someone else's problem.
749
+ //
750
+ // Only when the refusals are the WHOLE story, though. If some accounts were
751
+ // refused and others are merely out of quota, a reset will still serve this
752
+ // request — so fall through to the retry-after/hold path below rather than
753
+ // failing fast on the strength of one bad credential. Reporting 502 there
754
+ // would turn a recoverable exhaustion into a hard error, and silently skip
755
+ // the holdSeconds wait an unattended run depends on.
756
+ const rejected = ctx.credentialRejected;
757
+ const allRefused = rejected?.size > 0 && (ctx.pinnedIndex != null
758
+ ? rejected.has(accountManager.accounts[ctx.pinnedIndex]?.name)
759
+ : rejected.size === accountManager.accounts.length);
760
+ if (allRefused) {
761
+ const names = [...rejected].map(n => `"${n}"`).join(', ');
762
+ ctx.status = 502;
763
+ ctx.account = `(${[...rejected].join(', ')} refused)`;
764
+ if (!res.headersSent) {
765
+ res.writeHead(502, { 'Content-Type': 'application/json' });
766
+ res.end(JSON.stringify({
767
+ type: 'error',
768
+ error: { type: 'proxy_error', message: `Upstream refused the credential for account ${names} (403). Check the account, then re-add it with: teamclaude login` },
769
+ }));
770
+ }
771
+ return;
772
+ }
773
+ // A pinned request concerns exactly one account: don't compute a fleet-wide
774
+ // retry-after or sleep on other accounts' windows — return immediately.
775
+ if (ctx.pinnedIndex != null) {
776
+ ctx.status = 429;
777
+ ctx.account = '(pinned account unavailable)';
778
+ if (!res.headersSent) {
779
+ res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': '5' });
780
+ res.end(JSON.stringify({
781
+ type: 'error',
782
+ error: { type: 'rate_limit_error', message: 'Pinned account is unavailable (rate-limited, errored, or already tried). Retry shortly.' },
783
+ }));
784
+ }
785
+ return;
786
+ }
787
+ ctx.status = 429;
788
+ ctx.account = '(none available)';
789
+ const status = accountManager.getStatus();
790
+ const retryAfter = computeRetryAfter(status.accounts);
791
+
792
+ // Long-hold mode: hold the HTTP connection and poll until an account
793
+ // recovers or the budget (holdSeconds) runs out. Claude Code waits for
794
+ // the first response byte, so this is transparent to the client as long
795
+ // as API_TIMEOUT_MS on the Claude Code side is large enough.
796
+ if (ctx.holdBudgetMs > 0) {
797
+ // Cap the per-poll sleep to 60s so a newly-available account (e.g. one
798
+ // manually enabled or whose quota reset early) is picked up within a
799
+ // minute instead of sleeping the full retryAfter (often 3600s).
800
+ const waitMs = Math.min(retryAfter * 1000, ctx.holdBudgetMs, 60_000);
801
+ ctx.holdBudgetMs -= waitMs;
802
+ console.log(`[TeamClaude] All accounts exhausted — holding connection, retry in ${Math.ceil(waitMs / 1000)}s (${Math.ceil(ctx.holdBudgetMs / 1000)}s budget left)`);
803
+ await new Promise(resolve => setTimeout(resolve, waitMs));
804
+ if (res.destroyed) return;
805
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, route);
806
+ }
807
+
808
+ const exhaustedRetries = ctx.exhaustedRetries || 0;
809
+ if (exhaustedRetries < 1 && retryAfter <= INLINE_RETRY_AFTER_MAX_SECONDS) {
810
+ ctx.exhaustedRetries = exhaustedRetries + 1;
811
+ console.log(`[TeamClaude] All accounts exhausted — waiting ${retryAfter}s before retry`);
812
+ await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
813
+ if (res.destroyed) return;
814
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, route);
815
+ }
816
+ res.writeHead(429, {
817
+ 'Content-Type': 'application/json',
818
+ 'retry-after': String(retryAfter),
819
+ });
820
+ res.end(JSON.stringify({
821
+ type: 'error',
822
+ error: {
823
+ type: 'rate_limit_error',
824
+ message: `All ${accountManager.accounts.length} accounts exhausted. Retry in ${retryAfter}s.`,
825
+ },
826
+ }));
827
+ return;
828
+ }
829
+
830
+ // Track which account handles this request
831
+ ctx.account = account.name;
832
+ // Pin this session to the serving account (for affinity) and keep it "active"
833
+ // in the running-sessions readout. Passive when distribution is off.
834
+ accountManager.recordSession(ctx.sessionId, account.index);
835
+ hooks.onRequestRouted?.(reqId, { account: account.name });
836
+
837
+ // Refresh OAuth token if needed
838
+ await accountManager.ensureTokenFresh(account.index);
839
+ if (account.status === 'error' && retryCount < maxRetries) {
840
+ ctx.tried.add(account.index);
841
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
842
+ }
843
+
844
+ // Build upstream request headers
845
+ const isOAuth = account.type === 'oauth';
846
+ const headers = {};
847
+ for (const [key, value] of Object.entries(req.headers)) {
848
+ const lk = key.toLowerCase();
849
+ // HTTP/2 pseudo-headers (:method, :path, :authority, :scheme) live in
850
+ // req.headers on the h2 server path; fetch rejects `:`-prefixed names.
851
+ if (lk.startsWith(':')) continue;
852
+ if (HOP_BY_HOP_HEADERS.has(lk)) continue;
853
+ if (lk === 'x-api-key') continue;
854
+ // Strip accept-encoding: Node fetch auto-decompresses, which would
855
+ // mismatch the Content-Encoding header we forward to the client
856
+ if (lk === 'accept-encoding') continue;
857
+ headers[key] = value;
858
+ }
859
+
860
+ if (isOAuth) {
861
+ headers['authorization'] = `Bearer ${account.credential}`;
862
+ } else {
863
+ headers['x-api-key'] = account.credential;
864
+ }
865
+
866
+ const upstreamUrl = `${account.upstream || upstream}${req.url}`;
867
+ const method = req.method;
868
+
869
+ // Strip orphaned tool_use / tool_result blocks so a client that compacted or
870
+ // interrupted a turn can't wedge the session with Anthropic's non-retryable
871
+ // 400 ("tool_use ids were found without tool_result blocks"). No-op (same
872
+ // Buffer) for a well-formed body.
873
+ let sendBody = sanitizeToolPairs(body, req.url, req.headers['content-type']);
874
+ // Align the body's account_uuid (in metadata.user_id) with the account whose
875
+ // token we're injecting (same-length patch; no-op if absent).
876
+ if (account.accountUuid) sendBody = patchAccountUuid(sendBody, account.accountUuid);
877
+ // Rewrite the model name for accounts that target a different upstream (e.g.
878
+ // GLM), which uses different model identifiers than Anthropic.
879
+ if (account.modelMap) sendBody = rewriteModel(sendBody, account.modelMap);
880
+ // If the body changed length (sanitize or model rewrite), update Content-Length
881
+ // so the upstream doesn't receive a mismatched framing and truncate or stall.
882
+ if (sendBody !== body) headers['content-length'] = String(sendBody.length);
883
+
884
+ // Streaming request log, opened lazily on the first terminal outcome (a
885
+ // pure-429-then-retry attempt writes no file, matching prior behavior). The
886
+ // request head+body are written once, just before the response is logged.
887
+ let log = null;
888
+ let reqLogged = false;
889
+ const getLog = () => (logDir ? (log ||= openRequestLog(logDir, reqId)) : null);
890
+ const logRequestHead = () => {
891
+ const l = getLog();
892
+ if (!l || reqLogged) return;
893
+ reqLogged = true;
894
+ const safeHeaders = { ...headers };
895
+ if (safeHeaders['x-api-key']) safeHeaders['x-api-key'] = safeHeaders['x-api-key'].slice(0, 15) + '...';
896
+ if (safeHeaders['authorization']) safeHeaders['authorization'] = safeHeaders['authorization'].slice(0, 20) + '...';
897
+ l.write(`=== REQUEST (account: ${account.name}, retry: ${retryCount}) ===\n${method} ${upstreamUrl}\n${formatHeaders(safeHeaders)}`);
898
+ if (body.length > 0) l.body('REQUEST BODY', body, req.headers['content-type']);
899
+ };
900
+
901
+ try {
902
+ // Storm control: pace requests onto a freshly-switched account so a failover
903
+ // burst doesn't slam it all at once and cascade (issue #84). The slot is held
904
+ // only until the response headers arrive — long enough to stagger the burst,
905
+ // then released so streaming bodies don't tie up concurrency. Fail-open: a
906
+ // client that disconnects while waiting just drops out.
907
+ if (!await accountManager.admit(account.index, () => res.destroyed)) return;
908
+ let upstreamRes;
909
+ try {
910
+ upstreamRes = await upstreamFetch(upstreamUrl, {
911
+ method,
912
+ headers,
913
+ body: ['GET', 'HEAD'].includes(method) ? undefined : sendBody,
914
+ redirect: 'manual',
915
+ }, sx, route);
916
+ } finally {
917
+ accountManager.release(account.index);
918
+ }
919
+
920
+ // Extract rate limit headers
921
+ const rateLimitHeaders = {};
922
+ for (const [key, value] of upstreamRes.headers.entries()) {
923
+ if (key.startsWith('anthropic-ratelimit-')) {
924
+ rateLimitHeaders[key] = value;
925
+ }
926
+ }
927
+ accountManager.updateQuota(account.index, rateLimitHeaders);
928
+
929
+ // Any non-429 response is live proof a rate-limit hold no longer binds —
930
+ // this is what lets a revalidation probe (a throttled account selected by
931
+ // _selectProbe) clear its own hold and return the fleet to service.
932
+ if (upstreamRes.status !== 429) accountManager.clearRateLimited(account.index);
933
+
934
+ // Two kinds of 429 are handled differently below: a quota rejection rotates
935
+ // to another account; a transient rate-limit throttle pauses + retries the
936
+ // same account (never rotates — see #84).
937
+ if (upstreamRes.status === 429) {
938
+ // Clamp Retry-After to a sane window: missing/invalid falls back to 60s,
939
+ // and out-of-range values are bounded to [1, 300]. A negative value would
940
+ // otherwise bypass the wait cap — setTimeout returns immediately and a
941
+ // pause/hold would be armed in the past.
942
+ let retryAfter = parseInt(upstreamRes.headers.get('retry-after'), 10);
943
+ if (Number.isNaN(retryAfter)) retryAfter = 60;
944
+ // Discard the 429 response body
945
+ await upstreamRes.body?.cancel();
946
+
947
+ // Durable quota exhaustion vs. a transient rate limit. A "rejected" unified
948
+ // status means a quota bucket is spent, so waiting and retrying the SAME
949
+ // account is futile — switch to another account now (updateQuota above
950
+ // already recorded the spent bucket's utilization from the headers).
951
+ const rl = rateLimitHeaders;
952
+ const generalRejected = rl['anthropic-ratelimit-unified-5h-status'] === 'rejected'
953
+ || rl['anthropic-ratelimit-unified-7d-status'] === 'rejected';
954
+ const fableRejected = rl['anthropic-ratelimit-unified-7d_oi-status'] === 'rejected' && !generalRejected;
955
+ if ((generalRejected || fableRejected) && retryCount < maxRetries) {
956
+ // A Fable-only rejection leaves the account fine for other models, so we
957
+ // do NOT throttle it globally — the recorded Fable utilization makes
958
+ // selection skip it for Fable requests only. A general rejection spends a
959
+ // shared bucket, so hold the whole account for its reset window.
960
+ if (fableRejected) {
961
+ console.log(`[TeamClaude] Fable weekly exhausted on "${account.name}" — switching account for this Fable request`);
962
+ } else {
963
+ const hold = Math.min(Math.max(retryAfter, 1), 3600);
964
+ console.log(`[TeamClaude] Quota rejection (429) on "${account.name}" — throttling ${hold}s and switching account`);
965
+ accountManager.markRateLimited(account.index, hold);
966
+ }
967
+ ctx.tried.add(account.index);
968
+ if (res.destroyed) return;
969
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
970
+ }
971
+
972
+ retryAfter = Math.min(Math.max(retryAfter, 1), 300);
973
+
974
+ // sx.org failover: 429s are IP-based, so retry via the proxy's egress IP.
975
+ // 'always' is already on sx; '429' switches direct→sx now and skips the
976
+ // wait (a fresh IP isn't throttled). Also arm the sticky window for MITM.
977
+ const nextUseSx = !!(sx?.useOn429());
978
+ const switchingToSx = nextUseSx && !route;
979
+ sx?.noteRateLimited(retryAfter);
980
+
981
+ // This is a rate-limit 429 (per-minute throttle), NOT quota exhaustion —
982
+ // quota rejection is handled above and is the only thing that rotates.
983
+ // Do NOT switch accounts here: moving the burst to the next account just
984
+ // throttles it too (thundering herd, #84) and discards this account's KV
985
+ // cache. Instead PAUSE this account so concurrent requests wait in admit()
986
+ // (capped, then released through a fresh ramp) instead of piling on, and
987
+ // retry the SAME account. The pause never marks the account throttled, so
988
+ // selection keeps choosing it.
989
+ accountManager.pauseAccount(account.index, Math.min(retryAfter, RATE_LIMIT_ABSORB_MAX_SECONDS));
990
+
991
+ // sx fresh-IP retry (still the same account) takes precedence over waiting.
992
+ // Bounded by retryCount like the inline-wait path below, so a persistently
993
+ // 429ing upstream can't loop forever through sx.
994
+ if (switchingToSx && retryCount < maxRetries) {
995
+ console.log(`[TeamClaude] 429 on "${account.name}" — retrying via sx.org (fresh egress IP)`);
996
+ if (res.destroyed) return;
997
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, nextUseSx);
998
+ }
999
+
1000
+ // Absorb short waits inline on the same account — the client never sees the
1001
+ // 429. Bounded by retryCount (maxRetries = account count) so a persistently
1002
+ // rate-limited account can't loop forever tying up the connection.
1003
+ if (retryAfter <= RATE_LIMIT_ABSORB_MAX_SECONDS && retryCount < maxRetries) {
1004
+ console.log(`[TeamClaude] Rate-limit 429 on "${account.name}" — waiting ${retryAfter}s, retrying same account (no switch)`);
1005
+ await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
1006
+ if (res.destroyed) return;
1007
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, nextUseSx);
1008
+ }
1009
+
1010
+ // Longer retry-after (or retries exhausted): don't hold the connection and
1011
+ // don't rotate — surface the 429 with retry-after so the client backs off.
1012
+ // The pause above keeps other requests off this account meanwhile.
1013
+ console.log(`[TeamClaude] Rate-limit 429 on "${account.name}" — retry-after ${retryAfter}s over inline cap; returning 429 to client (no switch)`);
1014
+ ctx.status = 429;
1015
+ if (!res.headersSent && !res.destroyed) {
1016
+ res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': String(retryAfter) });
1017
+ res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: `Rate limited; retry in ${retryAfter}s.` } }));
1018
+ }
1019
+ return;
1020
+ }
1021
+
1022
+ // A 401 means the credential we injected was rejected. For an OAuth account
1023
+ // that usually means the access token was revoked BEFORE its clock expiry —
1024
+ // something else refreshed the same token family, so upstream reports it
1025
+ // revoked while it still looks fresh locally. ensureTokenFresh's expiry
1026
+ // check cannot see that (it only compares the clock), so the account would
1027
+ // otherwise keep serving a dead token until the token aged out, and every
1028
+ // request in between would surface a 401 to the client with no recovery.
1029
+ // Force one refresh and retry. If the refresh is itself rejected the refresh
1030
+ // token is dead too: ensureTokenFresh marks the account errored, and the
1031
+ // retry's status check rotates to another account. Bounded to one re-auth
1032
+ // per account per request, so a genuinely dead credential surfaces the 401
1033
+ // instead of looping.
1034
+ // A 403 ("Request not allowed") is upstream refusing THIS account outright —
1035
+ // not a stale token a refresh could fix, and not anything the client sent.
1036
+ // The client never sees the credential we inject, so it cannot act on the
1037
+ // rejection; Claude Code reads a 403 as "your session is dead", drops its
1038
+ // own login and asks for a re-login over an account problem it has no part
1039
+ // in. Skip the account for the rest of this request and fail over. With no
1040
+ // account left, the no-account branch reports a proxy error instead.
1041
+ if (upstreamRes.status === 403 && !res.headersSent) {
1042
+ await upstreamRes.body?.cancel();
1043
+ // A set, not a name: the no-account branch needs to tell "every account was
1044
+ // refused" (fail fast, nothing to wait for) from "this one was, others are
1045
+ // just out of quota" (still worth holding for a reset).
1046
+ (ctx.credentialRejected ??= new Set()).add(account.name);
1047
+ ctx.tried.add(account.index);
1048
+ console.error(`[TeamClaude] 403 on "${account.name}" — upstream refused the account credential`);
1049
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
1050
+ }
1051
+
1052
+ if (upstreamRes.status === 401 && account.type === 'oauth' && account.refreshToken
1053
+ && retryCount < maxRetries && !ctx.reauthed.has(account.index)) {
1054
+ ctx.reauthed.add(account.index);
1055
+ await upstreamRes.body?.cancel();
1056
+ console.log(`[TeamClaude] 401 on "${account.name}" — token rejected; forcing refresh and retrying`);
1057
+ await accountManager.ensureTokenFresh(account.index, true);
1058
+ if (res.destroyed) return;
1059
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
1060
+ }
1061
+
1062
+ // Log the request head (once) followed by the response headers, streaming
1063
+ // to disk from here on.
1064
+ logRequestHead();
1065
+ getLog()?.write(`\n\n=== RESPONSE ${upstreamRes.status} ===\n${formatHeaders(upstreamRes.headers)}`);
1066
+
1067
+ ctx.status = upstreamRes.status;
1068
+
1069
+ // Build response headers (skip hop-by-hop and encoding headers). The
1070
+ // connection-specific names are also illegal on an HTTP/2 response — when
1071
+ // this runs behind the MITM's h2 server, writeHead would otherwise throw.
1072
+ const responseHeaders = {};
1073
+ for (const [key, value] of upstreamRes.headers.entries()) {
1074
+ if (CONNECTION_SPECIFIC_HEADERS.has(key)) continue;
1075
+ // Strip content-encoding/content-length since fetch may auto-decompress
1076
+ if (key === 'content-encoding' || key === 'content-length') continue;
1077
+ responseHeaders[key] = value;
1078
+ }
1079
+
1080
+ res.writeHead(upstreamRes.status, responseHeaders);
1081
+
1082
+ if (!upstreamRes.body) {
1083
+ const l = getLog();
1084
+ if (l) { l.write('\n\n=== RESPONSE BODY ===\n(empty)'); l.end(); }
1085
+ res.end();
1086
+ return;
1087
+ }
1088
+
1089
+ const contentType = upstreamRes.headers.get('content-type') || '';
1090
+ const isStreaming = contentType.includes('text/event-stream');
1091
+
1092
+ if (isStreaming) {
1093
+ // Stream each chunk straight to the log as it is relayed — never hold the
1094
+ // whole (potentially ~1M-token) SSE body in memory.
1095
+ const l = getLog();
1096
+ const bw = l ? l.bodyWriter('RESPONSE BODY (streamed)', contentType) : null;
1097
+ await streamResponse(upstreamRes.body, res, account.index, accountManager, bw);
1098
+ l?.end();
1099
+ } else {
1100
+ const buf = Buffer.from(await upstreamRes.arrayBuffer());
1101
+ extractUsageFromBody(buf, account.index, accountManager);
1102
+ const l = getLog();
1103
+ if (l) { l.body('RESPONSE BODY', buf, contentType); l.end(); }
1104
+ res.end(buf);
1105
+ }
1106
+ } catch (err) {
1107
+ console.error(`[TeamClaude] Upstream error (account "${account.name}"):`, err.message);
1108
+
1109
+ logRequestHead();
1110
+ const l = getLog();
1111
+ if (l) { l.write(`\n\n=== ERROR ===\n${err.stack || err.message}`); l.end(); }
1112
+
1113
+ const isTransient = err instanceof Error &&
1114
+ (err.code === 'TEAMCLAUDE_HEADERS_TIMEOUT' || err.code === 'TEAMCLAUDE_BODY_TIMEOUT' ||
1115
+ err.name === 'TimeoutError' || err.name === 'AbortError' ||
1116
+ err.message.includes('fetch failed') ||
1117
+ err.code === 'ECONNRESET' || err.code === 'ECONNREFUSED' ||
1118
+ err.code === 'ETIMEDOUT' || err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
1119
+ err.code === 'UND_ERR_HEADERS_TIMEOUT' || err.code === 'UND_ERR_BODY_TIMEOUT');
1120
+
1121
+ // Transient network errors (including a stale-socket headers/body timeout):
1122
+ // close the connection and let the client retry. Failing over to another
1123
+ // account would not help (the poisoned fetch pool is process-wide), but the
1124
+ // fast failure lets Node evict the dead socket so the retry reconnects
1125
+ // cleanly. If headers were already sent (a mid-stream body timeout), destroy
1126
+ // is the only option — the client sees a broken response and retries.
1127
+ if (isTransient) {
1128
+ res.destroy();
1129
+ return;
1130
+ }
1131
+
1132
+ // Any other thrown error is a transport/stream failure, NOT proof the
1133
+ // account's credentials are bad — a bad credential comes back as a 401
1134
+ // *response*, never a throw. So don't sideline the account (that would drop
1135
+ // a healthy account from rotation until a credential change). Instead skip
1136
+ // it for the rest of THIS request only and fail over to another account.
1137
+ if (retryCount < maxRetries && !res.headersSent) {
1138
+ ctx.tried.add(account.index);
1139
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
1140
+ }
1141
+ ctx.status = 502;
1142
+
1143
+ if (!res.headersSent) {
1144
+ res.writeHead(502, { 'Content-Type': 'application/json' });
1145
+ res.end(JSON.stringify({
1146
+ type: 'error',
1147
+ error: { type: 'proxy_error', message: `Upstream error: ${err.message}` },
1148
+ }));
1149
+ } else if (!res.writableEnded) {
1150
+ // Error after headers were already sent (mid-stream) and it wasn't
1151
+ // classified transient: we can't send a status or fail over, and
1152
+ // streamResponse deliberately skipped res.end(). Destroy so the client
1153
+ // sees a broken response and retries instead of hanging on an open socket.
1154
+ res.destroy();
1155
+ }
1156
+ }
1157
+ }
1158
+
1159
+ // Idle deadline for the RESPONSE BODY, complementing the headers timeout in
1160
+ // upstream-fetch.js. The headers guard only covers time-to-first-byte; once
1161
+ // headers arrive it is disarmed, so a network drop AFTER the stream starts would
1162
+ // otherwise hang the read forever (the SSE completion just goes silent mid-way).
1163
+ // This watchdog resets on every chunk, so a long but healthy stream is never
1164
+ // cut — it fires only when the socket produces nothing for the whole window,
1165
+ // converting a mid-stream hang into a fast failure that evicts the dead socket
1166
+ // (reader.cancel destroys the underlying connection on both the direct-fetch and
1167
+ // the sx-tunnel path, since both hand back a web ReadableStream). Override with
1168
+ // TEAMCLAUDE_UPSTREAM_BODY_TIMEOUT_MS.
1169
+ const DEFAULT_BODY_IDLE_TIMEOUT_MS = 120_000;
1170
+
1171
+ function resolveBodyIdleTimeout() {
1172
+ const env = Number(process.env.TEAMCLAUDE_UPSTREAM_BODY_TIMEOUT_MS);
1173
+ return env > 0 ? env : DEFAULT_BODY_IDLE_TIMEOUT_MS;
1174
+ }
1175
+
1176
+ // Race a single reader.read() against an inactivity deadline. Resolves to the
1177
+ // read result, or rejects with a transient TEAMCLAUDE_BODY_TIMEOUT if no chunk
1178
+ // arrives within `ms`. The pending read is abandoned on timeout; the caller
1179
+ // cancels the reader (evicting the socket) in its finally block.
1180
+ export function readWithIdleTimeout(reader, ms) {
1181
+ let timer;
1182
+ const timeout = new Promise((_, reject) => {
1183
+ timer = setTimeout(() => {
1184
+ const err = new Error(`upstream stream idle for ${ms}ms`);
1185
+ err.code = 'TEAMCLAUDE_BODY_TIMEOUT';
1186
+ reject(err);
1187
+ }, ms);
1188
+ timer.unref?.();
1189
+ });
1190
+ const read = reader.read();
1191
+ // If the timeout wins the race, `read` is abandoned; swallow any later
1192
+ // rejection so it can't surface as an unhandledRejection.
1193
+ read.catch(() => {});
1194
+ return Promise.race([read, timeout]).finally(() => clearTimeout(timer));
1195
+ }
1196
+
1197
+ /**
1198
+ * Stream an SSE response to the client, parsing usage data along the way.
1199
+ */
1200
+ async function streamResponse(webStream, res, accountIndex, accountManager, bodyWriter) {
1201
+ const reader = webStream.getReader();
1202
+ const idleMs = resolveBodyIdleTimeout();
1203
+ const decoder = new TextDecoder();
1204
+ let sseBuffer = '';
1205
+ let errored = false;
1206
+
1207
+ try {
1208
+ while (true) {
1209
+ const { done, value } = await readWithIdleTimeout(reader, idleMs);
1210
+ if (done) break;
1211
+
1212
+ // Client disconnected — stop reading from upstream
1213
+ if (res.destroyed) break;
1214
+
1215
+ // Forward chunk immediately
1216
+ const ok = res.write(value);
1217
+
1218
+ // Append to the log as it streams (no whole-body buffering)
1219
+ if (bodyWriter) bodyWriter.chunk(Buffer.from(value));
1220
+
1221
+ const text = decoder.decode(value, { stream: true });
1222
+
1223
+ // Parse SSE events for usage tracking
1224
+ sseBuffer += text;
1225
+ const events = sseBuffer.split('\n\n');
1226
+ sseBuffer = events.pop(); // keep incomplete event
1227
+
1228
+ for (const event of events) {
1229
+ parseSSEUsage(event, accountIndex, accountManager);
1230
+ }
1231
+
1232
+ // Handle backpressure — also bail out if client disconnects,
1233
+ // because 'drain' will never fire on a destroyed socket
1234
+ if (!ok) {
1235
+ await new Promise(resolve => {
1236
+ // Remove BOTH listeners when either fires: otherwise the un-fired one
1237
+ // (usually 'close') stays attached and accumulates one leaked listener
1238
+ // per backpressure cycle over a long SSE stream to a slow client.
1239
+ const done = () => { res.off('drain', done); res.off('close', done); resolve(); };
1240
+ res.once('drain', done);
1241
+ res.once('close', done);
1242
+ });
1243
+ if (res.destroyed) break;
1244
+ }
1245
+ }
1246
+
1247
+ // Parse any remaining buffer
1248
+ if (sseBuffer.trim()) {
1249
+ parseSSEUsage(sseBuffer, accountIndex, accountManager);
1250
+ }
1251
+ } catch (err) {
1252
+ // A mid-stream idle timeout (or any read error) means the upstream went
1253
+ // silent after headers. Rethrow to the caller's transient handler, which
1254
+ // destroys the client connection so the truncated stream is NOT ended
1255
+ // cleanly (a clean res.end() would look like a complete response and
1256
+ // suppress the client's retry). reader.cancel() in finally evicts the socket.
1257
+ errored = true;
1258
+ throw err;
1259
+ } finally {
1260
+ // Cancel upstream reader to stop consuming data nobody needs (and, on the
1261
+ // timeout path, to destroy the dead socket so the pool drops it).
1262
+ reader.cancel().catch(() => {});
1263
+ if (!errored && !res.writableEnded) res.end();
1264
+ }
1265
+ }
1266
+
1267
+ function parseSSEUsage(event, accountIndex, accountManager) {
1268
+ const dataLine = event.split('\n').find(l => l.startsWith('data: '));
1269
+ if (!dataLine) return;
1270
+
1271
+ try {
1272
+ const data = JSON.parse(dataLine.slice(6));
1273
+ if (data.type === 'message_start' && data.message?.usage) {
1274
+ accountManager.updateUsage(accountIndex, data.message.usage.input_tokens, 0);
1275
+ } else if (data.type === 'message_delta' && data.usage) {
1276
+ accountManager.updateUsage(accountIndex, 0, data.usage.output_tokens);
1277
+ }
1278
+ } catch {
1279
+ // not valid JSON, skip
1280
+ }
1281
+ }
1282
+
1283
+ function extractUsageFromBody(buffer, accountIndex, accountManager) {
1284
+ try {
1285
+ const json = JSON.parse(buffer.toString());
1286
+ if (json.usage) {
1287
+ accountManager.updateUsage(accountIndex, json.usage.input_tokens, json.usage.output_tokens);
1288
+ }
1289
+ } catch {
1290
+ // not JSON or no usage
1291
+ }
1292
+ }
1293
+
1294
+ // Rewrite the `model` field in a JSON request body using a per-account map.
1295
+ // Returns the original buffer unchanged if the model isn't in the map or the
1296
+ // body isn't valid JSON, so non-messages endpoints pass through safely.
1297
+ // Exported for tests.
1298
+ export function rewriteModel(body, modelMap) {
1299
+ try {
1300
+ const obj = JSON.parse(body.toString('utf8'));
1301
+ if (obj.model && modelMap[obj.model]) {
1302
+ obj.model = modelMap[obj.model];
1303
+ return Buffer.from(JSON.stringify(obj), 'utf8');
1304
+ }
1305
+ } catch { /* not JSON — pass through unchanged */ }
1306
+ return body;
1307
+ }
1308
+
1309
+ function computeRetryAfter(accounts) {
1310
+ let soonest = Infinity;
1311
+ for (const acct of accounts) {
1312
+ const reset = acct.rateLimitedUntil || acct.quota.resetsAt;
1313
+ if (reset) {
1314
+ const ms = new Date(reset).getTime() - Date.now();
1315
+ if (ms < soonest) soonest = ms;
1316
+ }
1317
+ }
1318
+ return soonest === Infinity ? 60 : Math.max(1, Math.ceil(soonest / 1000));
1319
+ }