@rikcodes/teamclaude 1.1.13-rik.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +122 -0
- package/package.json +43 -0
- package/src/account-manager.js +1459 -0
- package/src/account-uuid-rewrite.js +115 -0
- package/src/alias.js +125 -0
- package/src/claude-env.js +65 -0
- package/src/config.js +146 -0
- package/src/crash-log.js +27 -0
- package/src/egress-guard.js +132 -0
- package/src/identity.js +96 -0
- package/src/index.js +1873 -0
- package/src/json-format-stream.js +63 -0
- package/src/mitm.js +336 -0
- package/src/model.js +276 -0
- package/src/oauth.js +459 -0
- package/src/prober.js +158 -0
- package/src/request-log.js +32 -0
- package/src/resolve-accounts.js +43 -0
- package/src/server.js +1319 -0
- package/src/service.js +241 -0
- package/src/session-tracker.js +133 -0
- package/src/status-renderer.js +316 -0
- package/src/sx.js +218 -0
- package/src/terminal-title.js +31 -0
- package/src/tool-pair-sanitize.js +193 -0
- package/src/tui-remote.js +274 -0
- package/src/tui.js +1634 -0
- package/src/updater.js +177 -0
- package/src/upstream-fetch.js +267 -0
- package/src/upstream-proxy.js +214 -0
- package/src/warmer.js +237 -0
- package/src/x509.js +166 -0
package/src/server.js
ADDED
|
@@ -0,0 +1,1319 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import https from 'node:https';
|
|
3
|
+
import { timingSafeEqual } from 'node:crypto';
|
|
4
|
+
import { createWriteStream } from 'node:fs';
|
|
5
|
+
import { mkdir } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { ensureCerts, createConnectHandler } from './mitm.js';
|
|
8
|
+
import { patchAccountUuid } from './account-uuid-rewrite.js';
|
|
9
|
+
import { sanitizeToolPairs } from './tool-pair-sanitize.js';
|
|
10
|
+
import { parseRequestModel, parseAdvisorModel } from './account-manager.js';
|
|
11
|
+
import { TopLevelFieldFinder, modelGlobMatches } from './model.js';
|
|
12
|
+
import { BodyWriter } from './request-log.js';
|
|
13
|
+
import { upstreamFetch } from './upstream-fetch.js';
|
|
14
|
+
import { tunnelTls } from './sx.js';
|
|
15
|
+
import { createEgressGuard } from './egress-guard.js';
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
export const HOP_BY_HOP_HEADERS = new Set([
|
|
19
|
+
'host', 'connection', 'keep-alive', 'transfer-encoding',
|
|
20
|
+
'te', 'trailer', 'upgrade', 'proxy-authorization', 'proxy-authenticate',
|
|
21
|
+
]);
|
|
22
|
+
// Path prefix for the deprecated URL-based account pin (superseded by TC_ACCT).
|
|
23
|
+
const PIN_PREFIX = '/tc-acct/';
|
|
24
|
+
const INLINE_RETRY_AFTER_MAX_SECONDS = 15;
|
|
25
|
+
// How long the proxy will absorb a rate-limit 429's retry-after inline (waiting
|
|
26
|
+
// on the SAME account) before surfacing a 429 + retry-after to the client. A
|
|
27
|
+
// rate-limit 429 never rotates accounts (that just moves the burst); it pauses
|
|
28
|
+
// the account so concurrent requests wait, then retries the same account.
|
|
29
|
+
const RATE_LIMIT_ABSORB_MAX_SECONDS =
|
|
30
|
+
Number(process.env.TEAMCLAUDE_RATE_LIMIT_ABSORB_MAX_SECONDS) || 60;
|
|
31
|
+
|
|
32
|
+
// Response header names that are connection-specific and thus illegal on an
|
|
33
|
+
// HTTP/2 response (Node's Http2ServerResponse.writeHead rejects them). Also
|
|
34
|
+
// hop-by-hop on h1, so stripping them is correct on both paths.
|
|
35
|
+
const CONNECTION_SPECIFIC_HEADERS = new Set([
|
|
36
|
+
'connection', 'keep-alive', 'transfer-encoding', 'upgrade',
|
|
37
|
+
'proxy-connection', 'te', 'trailer',
|
|
38
|
+
]);
|
|
39
|
+
|
|
40
|
+
// Constant-time proxy-API-key comparison (both the HTTP gate and the CONNECT
|
|
41
|
+
// gate use it). Returns false on any type/length mismatch without leaking timing.
|
|
42
|
+
export function safeKeyEqual(a, b) {
|
|
43
|
+
if (typeof a !== 'string' || typeof b !== 'string') return false;
|
|
44
|
+
const ba = Buffer.from(a);
|
|
45
|
+
const bb = Buffer.from(b);
|
|
46
|
+
if (ba.length !== bb.length) return false;
|
|
47
|
+
return timingSafeEqual(ba, bb);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// True if a socket's remote address is loopback — the proxy-key gate exempts
|
|
51
|
+
// localhost on both the HTTP and CONNECT paths.
|
|
52
|
+
export function isLoopbackAddr(addr) {
|
|
53
|
+
return addr === '127.0.0.1' || addr === '::1' || addr === '::ffff:127.0.0.1';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function createProxyServer(accountManager, config, hooks = {}, sx = null) {
|
|
57
|
+
const upstream = config.upstream || 'https://api.anthropic.com';
|
|
58
|
+
const proxyApiKey = config.proxy?.apiKey;
|
|
59
|
+
const logDir = config.logDir || null;
|
|
60
|
+
const holdMs = (config.holdSeconds || 0) * 1000;
|
|
61
|
+
|
|
62
|
+
if (logDir) {
|
|
63
|
+
mkdir(logDir, { recursive: true }).catch(() => {});
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const requestHandler = async (req, res) => {
|
|
67
|
+
try {
|
|
68
|
+
// Auth check — skip for localhost connections.
|
|
69
|
+
const clientKey = req.headers['x-api-key'];
|
|
70
|
+
const isLocal = isLoopbackAddr(req.socket.remoteAddress);
|
|
71
|
+
if (proxyApiKey && !safeKeyEqual(clientKey, proxyApiKey) && !isLocal) {
|
|
72
|
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
|
73
|
+
res.end(JSON.stringify({
|
|
74
|
+
type: 'error',
|
|
75
|
+
error: { type: 'authentication_error', message: 'Invalid proxy API key' },
|
|
76
|
+
}));
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Control-plane mutations are refused when the request was issued by a web
|
|
81
|
+
// page. The gate above exempts loopback from the API key, so without this
|
|
82
|
+
// any site the operator happens to visit can POST here cross-origin: a
|
|
83
|
+
// `fetch(..., {mode:'no-cors', body})` with a text/plain content type is a
|
|
84
|
+
// CORS "simple request", so no preflight is sent and the request lands.
|
|
85
|
+
// The page cannot read the reply, but the side effect is the point —
|
|
86
|
+
// forcing the whole fleet onto one named account is a targeted quota
|
|
87
|
+
// drain, and reload is reachable the same way.
|
|
88
|
+
//
|
|
89
|
+
// Origin (and Sec-Fetch-Site) are set by the browser and cannot be
|
|
90
|
+
// forged from page JavaScript, while curl and the CLI send neither — so
|
|
91
|
+
// this costs legitimate callers nothing. Deliberately not a content-type
|
|
92
|
+
// requirement, which would also close the hole but would break the
|
|
93
|
+
// documented `curl -X POST .../teamclaude/reload` that sends no body.
|
|
94
|
+
if (req.method === 'POST' && (req.url || '').startsWith('/teamclaude/')
|
|
95
|
+
&& !isSameOriginControlRequest(req)) {
|
|
96
|
+
res.writeHead(403, { 'Content-Type': 'application/json' });
|
|
97
|
+
res.end(JSON.stringify({
|
|
98
|
+
ok: false,
|
|
99
|
+
error: 'cross-origin request refused: the control plane is not reachable from a web page',
|
|
100
|
+
}));
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Forward-proxy request (HTTP_PROXY): an absolute-form URL is a tool
|
|
105
|
+
// proxying plain HTTP to some host. Account logic is only for hosts we
|
|
106
|
+
// manage (the Anthropic upstream, which is HTTPS-only and never arrives
|
|
107
|
+
// this way); forward anything else transparently instead of hijacking it.
|
|
108
|
+
if (/^https?:\/\//i.test(req.url || '')) { relayHttpForward(req, res); return; }
|
|
109
|
+
|
|
110
|
+
// Status endpoint
|
|
111
|
+
if (req.method === 'GET' && req.url === '/teamclaude/status') {
|
|
112
|
+
const status = accountManager.getStatus();
|
|
113
|
+
const extra = hooks.getStatusExtra?.() || {};
|
|
114
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
115
|
+
res.end(JSON.stringify({ ...extra, ...status }, null, 2));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Reload endpoint — re-sync accounts from config without a restart. This
|
|
120
|
+
// is the headless equivalent of pressing 'R' in the TUI. Local control
|
|
121
|
+
// only (no upstream calls); the auth gate above already applies.
|
|
122
|
+
if (req.method === 'POST' && req.url === '/teamclaude/reload') {
|
|
123
|
+
if (!hooks.reload) {
|
|
124
|
+
res.writeHead(501, { 'Content-Type': 'application/json' });
|
|
125
|
+
res.end(JSON.stringify({ ok: false, error: 'reload not supported' }));
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
try {
|
|
129
|
+
const added = await hooks.reload();
|
|
130
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
131
|
+
res.end(JSON.stringify({ ok: true, added: added || 0 }));
|
|
132
|
+
} catch (err) {
|
|
133
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
134
|
+
res.end(JSON.stringify({ ok: false, error: err.message }));
|
|
135
|
+
}
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Switch endpoint — make one account the preferred one, the headless
|
|
140
|
+
// equivalent of picking it with 's' in the TUI. Both do the same single
|
|
141
|
+
// thing: move currentIndex. That is a preference, and a weak one: _select
|
|
142
|
+
// abandons it as soon as the account is unavailable, and also whenever any
|
|
143
|
+
// available account carries a strictly lower priority value. So the answer
|
|
144
|
+
// reports whether the choice will actually take effect rather than only
|
|
145
|
+
// that it was recorded. Body:
|
|
146
|
+
// {"account": "<name|email|accountUuid|accountUuid/orgUuid|orgUuid>"}.
|
|
147
|
+
// Local control only (no upstream calls); the auth gate above applies.
|
|
148
|
+
if (req.method === 'POST' && req.url === '/teamclaude/switch') {
|
|
149
|
+
const names = () => (accountManager.accounts || []).map(a => a.name);
|
|
150
|
+
let target;
|
|
151
|
+
try {
|
|
152
|
+
const raw = await readControlBody(req);
|
|
153
|
+
target = JSON.parse(raw || '{}')?.account;
|
|
154
|
+
} catch (err) {
|
|
155
|
+
// Say which of the two it was, but never echo the parser's own message
|
|
156
|
+
// back to a caller — that is our internals, not their input.
|
|
157
|
+
const tooLarge = err.message === 'body too large';
|
|
158
|
+
res.writeHead(tooLarge ? 413 : 400, { 'Content-Type': 'application/json' });
|
|
159
|
+
res.end(JSON.stringify({ ok: false, error: tooLarge ? 'request body too large' : 'invalid request body' }));
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
if (typeof target !== 'string' || !target.trim()) {
|
|
163
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
164
|
+
res.end(JSON.stringify({ ok: false, error: 'missing "account"', accounts: names() }));
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const index = resolveAccountPin(accountManager, target);
|
|
168
|
+
if (index == null) {
|
|
169
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
170
|
+
res.end(JSON.stringify({ ok: false, error: `no such account "${target}"`, accounts: names() }));
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
accountManager.currentIndex = index;
|
|
174
|
+
const name = accountManager.accounts[index].name;
|
|
175
|
+
// Recording the choice and the choice taking effect are two different
|
|
176
|
+
// things: selection skips an account it cannot use on the very next
|
|
177
|
+
// request, so a bare "ok" would be a lie for a disabled or spent target.
|
|
178
|
+
// The switch still happens (that is the TUI's behaviour) and the answer
|
|
179
|
+
// says whether traffic will follow it.
|
|
180
|
+
const { eligible, reason } = accountManager.eligibility(index);
|
|
181
|
+
// Leave a trace where every other account change already leaves one: the
|
|
182
|
+
// TUI swaps console.log for its activity pane and headless mode tees it
|
|
183
|
+
// to the activity log, so this one line covers both. Without it a manual
|
|
184
|
+
// switch is the only account change that happens invisibly — on exactly
|
|
185
|
+
// the background-service deployment this endpoint exists for.
|
|
186
|
+
console.log(`[TeamClaude] Switched to account "${name}" (manual)`
|
|
187
|
+
+ (eligible ? '' : ` — ${reason}, so rotation will not use it`));
|
|
188
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
189
|
+
res.end(JSON.stringify({ ok: true, account: name, eligible, ...(reason ? { reason } : {}) }));
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return forward(req, res);
|
|
194
|
+
} catch (err) {
|
|
195
|
+
console.error('[TeamClaude] Unhandled error:', err);
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
// Opt-in egress pin: null unless config.egress.pin is set, and then shared by
|
|
200
|
+
// the base listener and the MITM one so both honour the same hold.
|
|
201
|
+
const egress = createEgressGuard(config, console.error);
|
|
202
|
+
const forward = createProxyRequestListener({ accountManager, upstream, logDir, hooks, sx, holdMs, config, egress });
|
|
203
|
+
const server = http.createServer(requestHandler);
|
|
204
|
+
|
|
205
|
+
// Forward-proxy support (always on, so multiple claude instances can use
|
|
206
|
+
// either ANTHROPIC_BASE_URL or HTTPS_PROXY against the same server). A CONNECT
|
|
207
|
+
// to the upstream host is a transparent MITM relay (rewrite only auth); the
|
|
208
|
+
// test host is answered locally; anything else is blind-tunneled. Certs are
|
|
209
|
+
// minted lazily on the first intercepted CONNECT.
|
|
210
|
+
const mitmHost = (() => { try { return new URL(upstream).hostname; } catch { return 'api.anthropic.com'; } })();
|
|
211
|
+
let certsPromise = null;
|
|
212
|
+
const ensureLeaf = async () => {
|
|
213
|
+
// Reset the memo on failure so a transient cert error doesn't wedge the MITM
|
|
214
|
+
// path permanently (a cached rejected promise would re-throw on every CONNECT).
|
|
215
|
+
certsPromise ||= ensureCerts(mitmHost).catch((err) => { certsPromise = null; throw err; });
|
|
216
|
+
const c = await certsPromise;
|
|
217
|
+
return { key: c.leafKeyPem, cert: c.leafCertPem };
|
|
218
|
+
};
|
|
219
|
+
server.on('connect', createConnectHandler({ config, accountManager, ensureLeaf, logDir, hooks, log: console.error, sx, egress }));
|
|
220
|
+
// Remote Control's real-time channel is a WebSocket, not a request/response
|
|
221
|
+
// call — Node fires 'upgrade' for that handshake, never 'request', so it
|
|
222
|
+
// needs its own listener (base-URL routing path; the MITM path wires the
|
|
223
|
+
// same relayUpgrade onto its own terminating server in mitm.js).
|
|
224
|
+
server.on('upgrade', (req, socket, head) => relayUpgrade(req, socket, head, upstream, sx));
|
|
225
|
+
|
|
226
|
+
return server;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Whether a control-plane POST did NOT come from a web page.
|
|
231
|
+
*
|
|
232
|
+
* Both headers are browser-set and unforgeable from page JavaScript:
|
|
233
|
+
* - `Sec-Fetch-Site` is the explicit answer where it exists (Chrome, Safari,
|
|
234
|
+
* Firefox). Anything but `same-origin` / `none` is a page reaching across.
|
|
235
|
+
* - `Origin` is the fallback for browsers that send no Sec-Fetch-Site. Its
|
|
236
|
+
* mere presence on a POST to a local control endpoint means a page issued
|
|
237
|
+
* it; matching it against our own host would mean guessing which of
|
|
238
|
+
* localhost / 127.0.0.1 / [::1] / a LAN address the caller used, and a
|
|
239
|
+
* browser-issued same-origin call is not a thing worth supporting here.
|
|
240
|
+
*
|
|
241
|
+
* Non-browser callers (curl, the CLI, `teamclaude attach`) send neither and are
|
|
242
|
+
* unaffected.
|
|
243
|
+
*/
|
|
244
|
+
export function isSameOriginControlRequest(req) {
|
|
245
|
+
const site = req.headers['sec-fetch-site'];
|
|
246
|
+
if (site) return site === 'same-origin' || site === 'none';
|
|
247
|
+
return !req.headers.origin;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Read a control-endpoint body as text. Capped, unlike the proxied request path:
|
|
251
|
+
// these endpoints carry a couple of fields, so anything larger is a mistake or an
|
|
252
|
+
// attack and buffering it whole would be the wrong answer either way.
|
|
253
|
+
async function readControlBody(req, limit = 64 * 1024) {
|
|
254
|
+
const chunks = [];
|
|
255
|
+
let size = 0;
|
|
256
|
+
for await (const chunk of req) {
|
|
257
|
+
size += chunk.length;
|
|
258
|
+
if (size > limit) throw new Error('body too large');
|
|
259
|
+
chunks.push(chunk);
|
|
260
|
+
}
|
|
261
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Resolve an account pin to an index, or null.
|
|
266
|
+
*
|
|
267
|
+
* Accepted forms, first match wins:
|
|
268
|
+
* - `accountUuid/orgUuid` — fully qualified, the only form that distinguishes
|
|
269
|
+
* one person's accounts across several orgs
|
|
270
|
+
* - `accountUuid`
|
|
271
|
+
* - `orgUuid`
|
|
272
|
+
* - the display name (`email` or `email (Org)`), or the bare email
|
|
273
|
+
*
|
|
274
|
+
* UUIDs are the identity to use for anything scripted or long-lived: display
|
|
275
|
+
* names are rewritten in place when an email gains a second org (see
|
|
276
|
+
* accountsCommand), so a name is a convenience, not an identifier.
|
|
277
|
+
*
|
|
278
|
+
* The rotation index is deliberately NOT accepted. It is array position, so
|
|
279
|
+
* deleting an account would silently repoint every later pin at a DIFFERENT
|
|
280
|
+
* account — a wrong-account misroute rather than an honest failure.
|
|
281
|
+
*/
|
|
282
|
+
export function resolveAccountPin(accountManager, token) {
|
|
283
|
+
const accounts = accountManager.accounts || [];
|
|
284
|
+
const norm = (s) => (s || '').trim().toLowerCase();
|
|
285
|
+
const t = norm(token);
|
|
286
|
+
if (!t) return null;
|
|
287
|
+
|
|
288
|
+
const at = (pick) => accounts.findIndex(a => norm(pick(a)) === t);
|
|
289
|
+
const qualified = accounts.findIndex(a => a.accountUuid && a.orgUuid
|
|
290
|
+
&& `${norm(a.accountUuid)}/${norm(a.orgUuid)}` === t);
|
|
291
|
+
|
|
292
|
+
for (const i of [
|
|
293
|
+
qualified,
|
|
294
|
+
at(a => a.accountUuid),
|
|
295
|
+
at(a => a.orgUuid),
|
|
296
|
+
at(a => a.name),
|
|
297
|
+
at(a => (a.name || '').split(' (')[0]), // display name minus the org suffix
|
|
298
|
+
]) if (i >= 0) return i;
|
|
299
|
+
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Paths that must reach upstream with the client's own credential (never a
|
|
304
|
+
// rotated account token): the Remote Control channel and attachment transfers.
|
|
305
|
+
// teamclaude applies its account logic (rotation, exhaustion, token injection)
|
|
306
|
+
// ONLY to hosts it manages — the Anthropic upstream. Anything else must be
|
|
307
|
+
// forwarded transparently, never hijacked into "all accounts exhausted". For
|
|
308
|
+
// HTTPS this is already true (the CONNECT tunnel in mitm.js blind-relays
|
|
309
|
+
// non-upstream hosts). This is the plain-HTTP counterpart: a tool honoring
|
|
310
|
+
// HTTP_PROXY sends an ABSOLUTE-form request (`GET http://host/path`), which
|
|
311
|
+
// otherwise gets misrouted to Anthropic. Blind-relay it to its target with the
|
|
312
|
+
// client's own headers — no account selection, no token injection,
|
|
313
|
+
// content-encoding passed through (a transparent forward proxy). Anthropic is
|
|
314
|
+
// HTTPS-only, so in practice this only ever sees third-party hosts.
|
|
315
|
+
export function relayHttpForward(req, res) {
|
|
316
|
+
let target;
|
|
317
|
+
try { target = new URL(req.url); } catch {
|
|
318
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
319
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'Malformed forward-proxy URL' } }));
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
const transport = target.protocol === 'http:' ? http : https;
|
|
323
|
+
const headers = {};
|
|
324
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
325
|
+
const lk = key.toLowerCase();
|
|
326
|
+
// Drop hop-by-hop + proxy-control headers; `host` is reset from the target.
|
|
327
|
+
if (lk.startsWith(':') || HOP_BY_HOP_HEADERS.has(lk) || lk === 'proxy-connection') continue;
|
|
328
|
+
headers[key] = value;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const upstreamReq = transport.request(target, { method: req.method, headers }, (upstreamRes) => {
|
|
332
|
+
const responseHeaders = {};
|
|
333
|
+
for (const [key, value] of Object.entries(upstreamRes.headers)) {
|
|
334
|
+
if (CONNECTION_SPECIFIC_HEADERS.has(key)) continue;
|
|
335
|
+
responseHeaders[key] = value;
|
|
336
|
+
}
|
|
337
|
+
res.writeHead(upstreamRes.statusCode, responseHeaders);
|
|
338
|
+
upstreamRes.pipe(res);
|
|
339
|
+
});
|
|
340
|
+
upstreamReq.on('error', (err) => {
|
|
341
|
+
console.error(`[TeamClaude] HTTP forward to ${target.host} failed:`, err.message);
|
|
342
|
+
if (!res.headersSent) {
|
|
343
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
344
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
res.on('close', () => upstreamReq.destroy());
|
|
348
|
+
if (['GET', 'HEAD'].includes(req.method)) upstreamReq.end();
|
|
349
|
+
else req.pipe(upstreamReq);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const CLIENT_CREDENTIAL_PATHS = ['/v1/code/', '/api/oauth/files/', '/api/oauth/file_upload'];
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Build the core proxy request listener — buffer the body, then forward with
|
|
356
|
+
* account selection + retry (forwardRequest). Shared by the base HTTP server and
|
|
357
|
+
* the MITM's terminating h2/h1 server, so both get identical buffering, model-
|
|
358
|
+
* aware routing, and retry-on-quota behavior. Control endpoints (status/reload)
|
|
359
|
+
* and the proxy-API-key gate live in the base server's wrapper, not here.
|
|
360
|
+
*/
|
|
361
|
+
export function createProxyRequestListener({ accountManager, upstream, logDir = null, hooks = {}, sx = null, holdMs = 0, config = {}, forcedPin = null, egress = null }) {
|
|
362
|
+
let counter = 0;
|
|
363
|
+
return async (req, res) => {
|
|
364
|
+
try {
|
|
365
|
+
// Claude Code's telemetry (`/api/event_logging/*`) is high-volume noise in
|
|
366
|
+
// the activity log. `config.eventLogging` (read live so the TUI toggle takes
|
|
367
|
+
// effect immediately): 'show' forwards + displays; 'hide' (default) forwards
|
|
368
|
+
// but suppresses the activity entry; 'block' answers 200 locally without
|
|
369
|
+
// forwarding (no upstream round-trip, no account/token spent).
|
|
370
|
+
const eventLogging = config?.eventLogging || 'hide';
|
|
371
|
+
const isEventLog = (req.url || '').startsWith('/api/event_logging');
|
|
372
|
+
if (isEventLog && eventLogging === 'block') {
|
|
373
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
374
|
+
res.end('{}');
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
const hideActivity = isEventLog && eventLogging !== 'show';
|
|
378
|
+
// Egress pin (opt-in): with the exit IP off the pinned one — a VPN that
|
|
379
|
+
// dropped — hold rather than send. Upstream answers a request from an
|
|
380
|
+
// unexpected region with a 403 that Claude Code reports as a dead session,
|
|
381
|
+
// so sending it costs a re-login while waiting costs latency. Checked here
|
|
382
|
+
// rather than per-account: it is a property of the connection, and this is
|
|
383
|
+
// the one path every request takes, MITM included.
|
|
384
|
+
if (egress?.enabled()) {
|
|
385
|
+
const state = await egress.waitUntilPinned({ isAborted: () => res.destroyed });
|
|
386
|
+
if (res.destroyed) return;
|
|
387
|
+
if (!state.ok) {
|
|
388
|
+
res.writeHead(503, { 'Content-Type': 'application/json', 'retry-after': '30' });
|
|
389
|
+
res.end(JSON.stringify({
|
|
390
|
+
type: 'error',
|
|
391
|
+
error: {
|
|
392
|
+
type: 'proxy_error',
|
|
393
|
+
message: `Egress is ${state.ip || 'unknown'}, not the pinned ${state.expected.join(', ')} — not sending this request. Check the VPN.`,
|
|
394
|
+
},
|
|
395
|
+
}));
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
// Client token refresh: pass through untouched (the proxy manages its own
|
|
400
|
+
// tokens via ensureTokenFresh; rewriting client refreshes would conflict).
|
|
401
|
+
if (req.method === 'POST' && req.url === '/v1/oauth/token') { await relayRaw(req, res, upstream, sx); return; }
|
|
402
|
+
// Remote Control (/v1/code/*) is bound to the session's paired claude.ai
|
|
403
|
+
// identity — forward with the client's OWN credential (streamed), never a
|
|
404
|
+
// rotated account token, which would 403 the worker event stream.
|
|
405
|
+
// Attachment transfers (/api/oauth/files/*, /api/oauth/file_upload) are
|
|
406
|
+
// likewise account-bound: files uploaded from claude.ai belong to the
|
|
407
|
+
// paired identity, so fetching them with a rotated token 403s and Claude
|
|
408
|
+
// Code silently drops the image from the message.
|
|
409
|
+
if (CLIENT_CREDENTIAL_PATHS.some((p) => (req.url || '').startsWith(p))) { await relayStream(req, res, upstream, sx); return; }
|
|
410
|
+
|
|
411
|
+
// Account pin: a request to `/tc-acct/<name-or-index>/...` (e.g. via
|
|
412
|
+
// ANTHROPIC_BASE_URL=http://host:port/tc-acct/deepseek) is forced onto that
|
|
413
|
+
// one account, bypassing rotation. Used by the keep-warm scheduler and for
|
|
414
|
+
// manual per-account testing. The prefix is stripped before forwarding.
|
|
415
|
+
let pinnedIndex = null;
|
|
416
|
+
// DEPRECATED: the path-prefix pin. Superseded by TC_ACCT, which works in
|
|
417
|
+
// MITM mode too (this form cannot — inside a CONNECT tunnel the path is
|
|
418
|
+
// the real upstream one). Kept for the warmer and for direct API callers.
|
|
419
|
+
// One segment only, so the fully-qualified `accountUuid/orgUuid` form is
|
|
420
|
+
// not expressible here; use TC_ACCT for that.
|
|
421
|
+
const url = req.url || '';
|
|
422
|
+
const afterPrefix = url.startsWith(PIN_PREFIX) ? url.slice(PIN_PREFIX.length) : null;
|
|
423
|
+
// The token runs to the next '/', which also begins the real request path.
|
|
424
|
+
const tokenEnd = afterPrefix == null ? -1 : afterPrefix.indexOf('/');
|
|
425
|
+
if (tokenEnd > 0) {
|
|
426
|
+
const token = decodeURIComponent(afterPrefix.slice(0, tokenEnd));
|
|
427
|
+
pinnedIndex = resolveAccountPin(accountManager, token);
|
|
428
|
+
if (pinnedIndex == null) {
|
|
429
|
+
const reqId = ++counter;
|
|
430
|
+
const sessionId = req.headers['x-claude-code-session-id'] || null;
|
|
431
|
+
if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: `(unknown pin: "${token}")`, status: 404, model: null, sessionId, pinned: false });
|
|
432
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
433
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'not_found_error', message: `Unknown account pin "${token}"` } }));
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
req.url = afterPrefix.slice(tokenEnd);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// MITM-mode pin. A CONNECT carrying `Proxy-Authorization: Basic <acct>:…`
|
|
440
|
+
// has no URL to hang a `/tc-acct/` prefix on — the path inside the tunnel
|
|
441
|
+
// is the real Anthropic one — so the pin arrives as a listener bound to
|
|
442
|
+
// that account (see createConnectHandler). Resolved per request rather
|
|
443
|
+
// than at CONNECT time: a hot reload can renumber accounts while a tunnel
|
|
444
|
+
// is open, and a name outliving an index is the safer half of that race.
|
|
445
|
+
if (pinnedIndex == null && forcedPin != null) {
|
|
446
|
+
pinnedIndex = resolveAccountPin(accountManager, forcedPin);
|
|
447
|
+
if (pinnedIndex == null) {
|
|
448
|
+
const reqId = ++counter;
|
|
449
|
+
const sessionId = req.headers['x-claude-code-session-id'] || null;
|
|
450
|
+
if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: `(unknown pin: "${forcedPin}")`, status: 404, model: null, sessionId, pinned: false });
|
|
451
|
+
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
452
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'not_found_error', message: `Unknown account pin "${forcedPin}" (from TC_ACCT)` } }));
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
const reqId = ++counter;
|
|
458
|
+
// Claude Code tags each session's requests with this header (present on
|
|
459
|
+
// /v1/messages and count_tokens). Read from headers up front so it drives
|
|
460
|
+
// session-aware routing (issue #109) and colors the TUI activity stream.
|
|
461
|
+
const sessionId = req.headers['x-claude-code-session-id'] || null;
|
|
462
|
+
if (!hideActivity) hooks.onRequestStart?.(reqId, { method: req.method, path: req.url, sessionId, pinned: pinnedIndex != null });
|
|
463
|
+
|
|
464
|
+
// Buffer request body (needed to resend on a different account after a 429).
|
|
465
|
+
// Peek the top-level `model` field incrementally as chunks arrive so the
|
|
466
|
+
// TUI can show it the instant it appears in the stream — usually the first
|
|
467
|
+
// frame — rather than waiting for the whole body and the request to finish.
|
|
468
|
+
const bodyChunks = [];
|
|
469
|
+
const modelFinder = new TopLevelFieldFinder('model');
|
|
470
|
+
for await (const chunk of req) {
|
|
471
|
+
bodyChunks.push(chunk);
|
|
472
|
+
if (!modelFinder.done) {
|
|
473
|
+
const found = modelFinder.push(chunk);
|
|
474
|
+
if (found && !hideActivity) hooks.onRequestModel?.(reqId, { model: found });
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
const body = Buffer.concat(bodyChunks);
|
|
478
|
+
|
|
479
|
+
const model = modelFinder.done ? modelFinder.value : parseRequestModel(body);
|
|
480
|
+
// An advisor request (Claude Code's advisor tool) carries a SECOND model
|
|
481
|
+
// nested in tools[]; the advisor sub-inference runs on the selected
|
|
482
|
+
// account, so selection must be eligible for it too (issue #98).
|
|
483
|
+
const advisorModel = parseAdvisorModel(body);
|
|
484
|
+
|
|
485
|
+
// Model blocklist (issue #116): reject a request for a blocked model right
|
|
486
|
+
// here instead of forwarding it. A model no account can serve (e.g. Fable
|
|
487
|
+
// once it left base plans) otherwise gets rate-limited upstream and hangs
|
|
488
|
+
// the pipeline; a fast, non-retryable 400 lets the client move on. Read
|
|
489
|
+
// live from the shared config so the TUI editor takes effect immediately.
|
|
490
|
+
const blockedBy = model ? (config?.blockedModels || []).find((p) => modelGlobMatches(p, model)) : null;
|
|
491
|
+
if (blockedBy) {
|
|
492
|
+
if (!res.headersSent) {
|
|
493
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
494
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: `Model "${model}" is blocked by teamclaude (matched "${blockedBy}").` } }));
|
|
495
|
+
}
|
|
496
|
+
hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: '(blocked)', status: 400, model, sessionId });
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const ctx = { account: null, status: null, tried: new Set(), reauthed: new Set(), model, advisorModel, pinnedIndex, holdBudgetMs: holdMs, sessionId };
|
|
501
|
+
// Hold the session "in flight" across the WHOLE request (incl. retries and
|
|
502
|
+
// a multi-minute streaming completion) so it stays counted as active and
|
|
503
|
+
// never expires mid-request.
|
|
504
|
+
accountManager.beginSession(sessionId);
|
|
505
|
+
try {
|
|
506
|
+
await forwardRequest(req, res, body, accountManager, upstream, 0, hooks, reqId, ctx, logDir, sx);
|
|
507
|
+
} catch (err) {
|
|
508
|
+
ctx.status = ctx.status || 502;
|
|
509
|
+
console.error('[TeamClaude] Unhandled error:', err);
|
|
510
|
+
if (!res.headersSent) {
|
|
511
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
512
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Internal proxy error' } }));
|
|
513
|
+
}
|
|
514
|
+
} finally {
|
|
515
|
+
accountManager.endSession(sessionId);
|
|
516
|
+
if (!hideActivity) hooks.onRequestEnd?.(reqId, { method: req.method, path: req.url, account: ctx.account, status: ctx.status, model: ctx.model, sessionId, pinned: ctx.pinnedIndex != null });
|
|
517
|
+
}
|
|
518
|
+
} catch (err) {
|
|
519
|
+
console.error('[TeamClaude] Unhandled error:', err);
|
|
520
|
+
}
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Per-request https.Agent tunneled through sx.org — one-shot (no keep-alive
|
|
525
|
+
// reuse, matching upstream-fetch.js's proxiedFetch), so a fresh sx tunnel is
|
|
526
|
+
// dialed for this connection only.
|
|
527
|
+
function sxAgent(sx, targetHost) {
|
|
528
|
+
const proxy = sx.getProxy();
|
|
529
|
+
const agent = new https.Agent({ keepAlive: false });
|
|
530
|
+
agent.createConnection = (_options, cb) => {
|
|
531
|
+
tunnelTls({ proxy, targetHost, targetPort: 443, tlsOptions: sx.tlsOptions || {} })
|
|
532
|
+
.then((sock) => cb(null, sock))
|
|
533
|
+
.catch((err) => cb(err));
|
|
534
|
+
return undefined;
|
|
535
|
+
};
|
|
536
|
+
return agent;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Relay a request to upstream with the client's OWN headers intact (including
|
|
541
|
+
* its authorization) — used for Remote Control (/v1/code/*), whose event
|
|
542
|
+
* stream is a long-poll: the client keeps the request open indefinitely and
|
|
543
|
+
* the upstream may withhold response headers for minutes between events. No
|
|
544
|
+
* buffering, no timeout, no reconstruction — just pipe bytes both ways as they
|
|
545
|
+
* arrive, exactly like a transparent proxy would.
|
|
546
|
+
*/
|
|
547
|
+
function relayStream(req, res, upstream, sx) {
|
|
548
|
+
const target = new URL(`${upstream}${req.url}`);
|
|
549
|
+
const headers = {};
|
|
550
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
551
|
+
const lk = key.toLowerCase();
|
|
552
|
+
if (lk.startsWith(':') || HOP_BY_HOP_HEADERS.has(lk) || lk === 'accept-encoding') continue;
|
|
553
|
+
headers[key] = value;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const useProxy = !!(sx?.useByDefault() && sx.isProvisioned());
|
|
557
|
+
const agent = useProxy ? sxAgent(sx, target.hostname) : undefined;
|
|
558
|
+
const transport = target.protocol === 'http:' ? http : https;
|
|
559
|
+
|
|
560
|
+
const upstreamReq = transport.request(target, { method: req.method, headers, agent }, (upstreamRes) => {
|
|
561
|
+
const responseHeaders = {};
|
|
562
|
+
for (const [key, value] of Object.entries(upstreamRes.headers)) {
|
|
563
|
+
if (CONNECTION_SPECIFIC_HEADERS.has(key) || key === 'content-encoding' || key === 'content-length') continue;
|
|
564
|
+
responseHeaders[key] = value;
|
|
565
|
+
}
|
|
566
|
+
res.writeHead(upstreamRes.statusCode, responseHeaders);
|
|
567
|
+
upstreamRes.pipe(res);
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
upstreamReq.on('error', (err) => {
|
|
571
|
+
console.error('[TeamClaude] Remote Control relay error:', err.message);
|
|
572
|
+
if (!res.headersSent) {
|
|
573
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
574
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
|
|
575
|
+
}
|
|
576
|
+
});
|
|
577
|
+
// Client disconnected (e.g. Claude Code closed the channel): tear down the
|
|
578
|
+
// upstream side too instead of leaking an open connection.
|
|
579
|
+
res.on('close', () => upstreamReq.destroy());
|
|
580
|
+
|
|
581
|
+
if (['GET', 'HEAD'].includes(req.method)) upstreamReq.end();
|
|
582
|
+
else req.pipe(upstreamReq);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Relay a WebSocket upgrade (e.g. Remote Control's real-time
|
|
587
|
+
* `/v1/session_ingress/ws/*` channel) to upstream with the client's own
|
|
588
|
+
* headers intact. An HTTP server never emits 'request' for an Upgrade
|
|
589
|
+
* handshake — only 'upgrade', with a raw socket instead of a response object —
|
|
590
|
+
* so this needs its own relay rather than going through relayStream/res.
|
|
591
|
+
* Reuses Node's http(s) client, which already knows how to speak the Upgrade
|
|
592
|
+
* handshake (emits its own 'upgrade' event on a 101); once that fires it's
|
|
593
|
+
* just two raw sockets spliced together.
|
|
594
|
+
*/
|
|
595
|
+
export function relayUpgrade(req, socket, head, upstream, sx) {
|
|
596
|
+
const target = new URL(`${upstream}${req.url}`);
|
|
597
|
+
const headers = {};
|
|
598
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
599
|
+
const lk = key.toLowerCase();
|
|
600
|
+
// Unlike relayStream, do NOT strip 'upgrade'/'connection' here — they ARE
|
|
601
|
+
// the handshake. Only 'host' (the client transport reconstructs it from
|
|
602
|
+
// `target`) and h2 pseudo-headers are dropped.
|
|
603
|
+
if (lk.startsWith(':') || lk === 'host') continue;
|
|
604
|
+
headers[key] = value;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
const useProxy = !!(sx?.useByDefault() && sx.isProvisioned());
|
|
608
|
+
const agent = useProxy ? sxAgent(sx, target.hostname) : undefined;
|
|
609
|
+
const transport = target.protocol === 'http:' ? http : https;
|
|
610
|
+
|
|
611
|
+
const upstreamReq = transport.request(target, { method: req.method, headers, agent });
|
|
612
|
+
|
|
613
|
+
upstreamReq.on('upgrade', (upstreamRes, upstreamSocket, upstreamHead) => {
|
|
614
|
+
const headerLines = Object.entries(upstreamRes.headers)
|
|
615
|
+
.map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(', ') : v}`).join('\r\n');
|
|
616
|
+
socket.write(`HTTP/1.1 ${upstreamRes.statusCode} ${upstreamRes.statusMessage}\r\n${headerLines}\r\n\r\n`);
|
|
617
|
+
if (upstreamHead?.length) socket.write(upstreamHead);
|
|
618
|
+
if (head?.length) upstreamSocket.write(head);
|
|
619
|
+
socket.pipe(upstreamSocket);
|
|
620
|
+
upstreamSocket.pipe(socket);
|
|
621
|
+
// An upgraded socket defaults to half-open: the peer's FIN only ends the
|
|
622
|
+
// READABLE side ('end'), it does NOT destroy the socket or fire 'close' —
|
|
623
|
+
// so without this, one side hanging up (dropped wifi, killed CLI) leaves
|
|
624
|
+
// the other socket open forever. destroy() is idempotent, so reacting to
|
|
625
|
+
// both 'end' and 'close' on each side is a safe, redundant backstop.
|
|
626
|
+
socket.on('end', () => upstreamSocket.destroy());
|
|
627
|
+
upstreamSocket.on('end', () => socket.destroy());
|
|
628
|
+
socket.on('close', () => upstreamSocket.destroy());
|
|
629
|
+
upstreamSocket.on('close', () => socket.destroy());
|
|
630
|
+
// The 101 detaches this socket from upstreamReq, so the request's 'error'
|
|
631
|
+
// listener no longer covers it. A link that flaps mid-session then raises
|
|
632
|
+
// 'error' (write EPIPE / read ECONNRESET) on a socket nobody listens to,
|
|
633
|
+
// which Node escalates to an uncaught exception — one dropped WebSocket
|
|
634
|
+
// would kill the proxy for every other session. Close the pair instead.
|
|
635
|
+
upstreamSocket.on('error', () => socket.destroy());
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
upstreamReq.on('error', (err) => {
|
|
639
|
+
console.error('[TeamClaude] Remote Control WebSocket relay error:', err.message);
|
|
640
|
+
socket.destroy();
|
|
641
|
+
});
|
|
642
|
+
socket.on('error', () => upstreamReq.destroy());
|
|
643
|
+
|
|
644
|
+
upstreamReq.end();
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Relay a request to upstream with no header rewriting — pure passthrough.
|
|
649
|
+
*/
|
|
650
|
+
async function relayRaw(req, res, upstream, sx) {
|
|
651
|
+
const bodyChunks = [];
|
|
652
|
+
for await (const chunk of req) bodyChunks.push(chunk);
|
|
653
|
+
const body = Buffer.concat(bodyChunks);
|
|
654
|
+
|
|
655
|
+
try {
|
|
656
|
+
const upstreamRes = await upstreamFetch(`${upstream}${req.url}`, {
|
|
657
|
+
method: req.method,
|
|
658
|
+
headers: {
|
|
659
|
+
'content-type': req.headers['content-type'] || 'application/json',
|
|
660
|
+
'accept': req.headers['accept'] || 'application/json',
|
|
661
|
+
'user-agent': req.headers['user-agent'] || 'node',
|
|
662
|
+
},
|
|
663
|
+
body: body.length > 0 ? body : undefined,
|
|
664
|
+
}, sx, sx?.useByDefault());
|
|
665
|
+
|
|
666
|
+
const responseBody = await upstreamRes.text();
|
|
667
|
+
const responseHeaders = {};
|
|
668
|
+
for (const [key, value] of upstreamRes.headers.entries()) {
|
|
669
|
+
// `.text()` already decompressed the body, so drop content-encoding and
|
|
670
|
+
// the now-stale content-length (both refer to the compressed bytes) — else
|
|
671
|
+
// a gzip'd upstream response reaches the client mis-framed / truncated.
|
|
672
|
+
if (key === 'transfer-encoding' || key === 'connection' ||
|
|
673
|
+
key === 'content-encoding' || key === 'content-length') continue;
|
|
674
|
+
responseHeaders[key] = value;
|
|
675
|
+
}
|
|
676
|
+
res.writeHead(upstreamRes.status, responseHeaders);
|
|
677
|
+
res.end(responseBody);
|
|
678
|
+
} catch (err) {
|
|
679
|
+
console.error('[TeamClaude] Raw relay error:', err.message);
|
|
680
|
+
if (!res.headersSent) {
|
|
681
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
682
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'proxy_error', message: 'Upstream unreachable' } }));
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
function logTimestamp() {
|
|
689
|
+
const d = new Date();
|
|
690
|
+
const pad = (n, w = 2) => String(n).padStart(w, '0');
|
|
691
|
+
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}.${pad(d.getMilliseconds(), 3)}`;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// A per-request log that streams to disk as the request/response flow, instead
|
|
695
|
+
// of buffering the whole body in memory and writing once at the end. The file
|
|
696
|
+
// is opened on first write; header sections are written verbatim and bodies are
|
|
697
|
+
// streamed through BodyWriter (JSON pretty-printed on the fly, SSE/other raw),
|
|
698
|
+
// so even a ~1M-token response costs only the current chunk.
|
|
699
|
+
function openRequestLog(logDir, reqId) {
|
|
700
|
+
const filename = `${logTimestamp()}_${String(reqId).padStart(5, '0')}.log`;
|
|
701
|
+
const ws = createWriteStream(join(logDir, filename), { flags: 'a' });
|
|
702
|
+
ws.on('error', (err) => console.error(`[TeamClaude] Failed to write log: ${err.message}`));
|
|
703
|
+
let ended = false;
|
|
704
|
+
const write = (s) => { if (!ended && s) ws.write(Buffer.from(String(s), 'latin1')); };
|
|
705
|
+
return {
|
|
706
|
+
write,
|
|
707
|
+
// Stream a complete body buffer under a section header.
|
|
708
|
+
body(label, buf, contentType) {
|
|
709
|
+
if (!buf || !buf.length) { write(`\n\n=== ${label} ===\n(empty)`); return; }
|
|
710
|
+
new BodyWriter(write, label, contentType || '').chunk(buf);
|
|
711
|
+
},
|
|
712
|
+
// A BodyWriter to append chunks incrementally (e.g. an SSE response).
|
|
713
|
+
bodyWriter(label, contentType) { return new BodyWriter(write, label, contentType || ''); },
|
|
714
|
+
end() { if (!ended) { ended = true; ws.end('\n'); } },
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
function formatHeaders(headers) {
|
|
719
|
+
if (headers.entries) {
|
|
720
|
+
return [...headers.entries()].map(([k, v]) => ` ${k}: ${v}`).join('\n');
|
|
721
|
+
}
|
|
722
|
+
return Object.entries(headers).map(([k, v]) => ` ${k}: ${v}`).join('\n');
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
export async function forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, useSx) {
|
|
726
|
+
const maxRetries = accountManager.accounts.length;
|
|
727
|
+
// This function is exported, so a caller may hand us a ctx built elsewhere.
|
|
728
|
+
// The 401 path reads ctx.reauthed on every response; default it here rather
|
|
729
|
+
// than trusting every construction site to include it.
|
|
730
|
+
ctx.reauthed ??= new Set();
|
|
731
|
+
// Whether THIS attempt dials via sx.org. Undefined on the first call → derive
|
|
732
|
+
// from the default policy ('always' routes; 'off'/'429' start direct).
|
|
733
|
+
const route = useSx === undefined ? !!(sx?.useByDefault()) : useSx;
|
|
734
|
+
|
|
735
|
+
// Select account, skipping any already tried (and failed) this request.
|
|
736
|
+
// The model scopes availability so a Fable-exhausted account is skipped only
|
|
737
|
+
// for Fable requests (it still serves other models).
|
|
738
|
+
// A pinned request (via /tc-acct/<name>) forces one exact account and never
|
|
739
|
+
// rotates or fails over: once that account has been tried, `account` is null
|
|
740
|
+
// and the caller gets the exhausted response rather than leaking to another.
|
|
741
|
+
const account = ctx.pinnedIndex != null
|
|
742
|
+
? (ctx.tried.has(ctx.pinnedIndex) ? null : accountManager.accounts[ctx.pinnedIndex])
|
|
743
|
+
: accountManager.getActiveAccount(ctx.tried, ctx.model, ctx.advisorModel, ctx.sessionId);
|
|
744
|
+
if (!account) {
|
|
745
|
+
// Every candidate was refused by upstream (403). Waiting will not help — the
|
|
746
|
+
// account needs attention, not a retry — so say so plainly rather than
|
|
747
|
+
// reporting a rate limit. Not a 403 either: the client's own credential is
|
|
748
|
+
// fine, and a 403 would make it drop its login over someone else's problem.
|
|
749
|
+
//
|
|
750
|
+
// Only when the refusals are the WHOLE story, though. If some accounts were
|
|
751
|
+
// refused and others are merely out of quota, a reset will still serve this
|
|
752
|
+
// request — so fall through to the retry-after/hold path below rather than
|
|
753
|
+
// failing fast on the strength of one bad credential. Reporting 502 there
|
|
754
|
+
// would turn a recoverable exhaustion into a hard error, and silently skip
|
|
755
|
+
// the holdSeconds wait an unattended run depends on.
|
|
756
|
+
const rejected = ctx.credentialRejected;
|
|
757
|
+
const allRefused = rejected?.size > 0 && (ctx.pinnedIndex != null
|
|
758
|
+
? rejected.has(accountManager.accounts[ctx.pinnedIndex]?.name)
|
|
759
|
+
: rejected.size === accountManager.accounts.length);
|
|
760
|
+
if (allRefused) {
|
|
761
|
+
const names = [...rejected].map(n => `"${n}"`).join(', ');
|
|
762
|
+
ctx.status = 502;
|
|
763
|
+
ctx.account = `(${[...rejected].join(', ')} refused)`;
|
|
764
|
+
if (!res.headersSent) {
|
|
765
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
766
|
+
res.end(JSON.stringify({
|
|
767
|
+
type: 'error',
|
|
768
|
+
error: { type: 'proxy_error', message: `Upstream refused the credential for account ${names} (403). Check the account, then re-add it with: teamclaude login` },
|
|
769
|
+
}));
|
|
770
|
+
}
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
// A pinned request concerns exactly one account: don't compute a fleet-wide
|
|
774
|
+
// retry-after or sleep on other accounts' windows — return immediately.
|
|
775
|
+
if (ctx.pinnedIndex != null) {
|
|
776
|
+
ctx.status = 429;
|
|
777
|
+
ctx.account = '(pinned account unavailable)';
|
|
778
|
+
if (!res.headersSent) {
|
|
779
|
+
res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': '5' });
|
|
780
|
+
res.end(JSON.stringify({
|
|
781
|
+
type: 'error',
|
|
782
|
+
error: { type: 'rate_limit_error', message: 'Pinned account is unavailable (rate-limited, errored, or already tried). Retry shortly.' },
|
|
783
|
+
}));
|
|
784
|
+
}
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
ctx.status = 429;
|
|
788
|
+
ctx.account = '(none available)';
|
|
789
|
+
const status = accountManager.getStatus();
|
|
790
|
+
const retryAfter = computeRetryAfter(status.accounts);
|
|
791
|
+
|
|
792
|
+
// Long-hold mode: hold the HTTP connection and poll until an account
|
|
793
|
+
// recovers or the budget (holdSeconds) runs out. Claude Code waits for
|
|
794
|
+
// the first response byte, so this is transparent to the client as long
|
|
795
|
+
// as API_TIMEOUT_MS on the Claude Code side is large enough.
|
|
796
|
+
if (ctx.holdBudgetMs > 0) {
|
|
797
|
+
// Cap the per-poll sleep to 60s so a newly-available account (e.g. one
|
|
798
|
+
// manually enabled or whose quota reset early) is picked up within a
|
|
799
|
+
// minute instead of sleeping the full retryAfter (often 3600s).
|
|
800
|
+
const waitMs = Math.min(retryAfter * 1000, ctx.holdBudgetMs, 60_000);
|
|
801
|
+
ctx.holdBudgetMs -= waitMs;
|
|
802
|
+
console.log(`[TeamClaude] All accounts exhausted — holding connection, retry in ${Math.ceil(waitMs / 1000)}s (${Math.ceil(ctx.holdBudgetMs / 1000)}s budget left)`);
|
|
803
|
+
await new Promise(resolve => setTimeout(resolve, waitMs));
|
|
804
|
+
if (res.destroyed) return;
|
|
805
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, route);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
const exhaustedRetries = ctx.exhaustedRetries || 0;
|
|
809
|
+
if (exhaustedRetries < 1 && retryAfter <= INLINE_RETRY_AFTER_MAX_SECONDS) {
|
|
810
|
+
ctx.exhaustedRetries = exhaustedRetries + 1;
|
|
811
|
+
console.log(`[TeamClaude] All accounts exhausted — waiting ${retryAfter}s before retry`);
|
|
812
|
+
await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
|
|
813
|
+
if (res.destroyed) return;
|
|
814
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, sx, route);
|
|
815
|
+
}
|
|
816
|
+
res.writeHead(429, {
|
|
817
|
+
'Content-Type': 'application/json',
|
|
818
|
+
'retry-after': String(retryAfter),
|
|
819
|
+
});
|
|
820
|
+
res.end(JSON.stringify({
|
|
821
|
+
type: 'error',
|
|
822
|
+
error: {
|
|
823
|
+
type: 'rate_limit_error',
|
|
824
|
+
message: `All ${accountManager.accounts.length} accounts exhausted. Retry in ${retryAfter}s.`,
|
|
825
|
+
},
|
|
826
|
+
}));
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
// Track which account handles this request
|
|
831
|
+
ctx.account = account.name;
|
|
832
|
+
// Pin this session to the serving account (for affinity) and keep it "active"
|
|
833
|
+
// in the running-sessions readout. Passive when distribution is off.
|
|
834
|
+
accountManager.recordSession(ctx.sessionId, account.index);
|
|
835
|
+
hooks.onRequestRouted?.(reqId, { account: account.name });
|
|
836
|
+
|
|
837
|
+
// Refresh OAuth token if needed
|
|
838
|
+
await accountManager.ensureTokenFresh(account.index);
|
|
839
|
+
if (account.status === 'error' && retryCount < maxRetries) {
|
|
840
|
+
ctx.tried.add(account.index);
|
|
841
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
// Build upstream request headers
|
|
845
|
+
const isOAuth = account.type === 'oauth';
|
|
846
|
+
const headers = {};
|
|
847
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
848
|
+
const lk = key.toLowerCase();
|
|
849
|
+
// HTTP/2 pseudo-headers (:method, :path, :authority, :scheme) live in
|
|
850
|
+
// req.headers on the h2 server path; fetch rejects `:`-prefixed names.
|
|
851
|
+
if (lk.startsWith(':')) continue;
|
|
852
|
+
if (HOP_BY_HOP_HEADERS.has(lk)) continue;
|
|
853
|
+
if (lk === 'x-api-key') continue;
|
|
854
|
+
// Strip accept-encoding: Node fetch auto-decompresses, which would
|
|
855
|
+
// mismatch the Content-Encoding header we forward to the client
|
|
856
|
+
if (lk === 'accept-encoding') continue;
|
|
857
|
+
headers[key] = value;
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
if (isOAuth) {
|
|
861
|
+
headers['authorization'] = `Bearer ${account.credential}`;
|
|
862
|
+
} else {
|
|
863
|
+
headers['x-api-key'] = account.credential;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
const upstreamUrl = `${account.upstream || upstream}${req.url}`;
|
|
867
|
+
const method = req.method;
|
|
868
|
+
|
|
869
|
+
// Strip orphaned tool_use / tool_result blocks so a client that compacted or
|
|
870
|
+
// interrupted a turn can't wedge the session with Anthropic's non-retryable
|
|
871
|
+
// 400 ("tool_use ids were found without tool_result blocks"). No-op (same
|
|
872
|
+
// Buffer) for a well-formed body.
|
|
873
|
+
let sendBody = sanitizeToolPairs(body, req.url, req.headers['content-type']);
|
|
874
|
+
// Align the body's account_uuid (in metadata.user_id) with the account whose
|
|
875
|
+
// token we're injecting (same-length patch; no-op if absent).
|
|
876
|
+
if (account.accountUuid) sendBody = patchAccountUuid(sendBody, account.accountUuid);
|
|
877
|
+
// Rewrite the model name for accounts that target a different upstream (e.g.
|
|
878
|
+
// GLM), which uses different model identifiers than Anthropic.
|
|
879
|
+
if (account.modelMap) sendBody = rewriteModel(sendBody, account.modelMap);
|
|
880
|
+
// If the body changed length (sanitize or model rewrite), update Content-Length
|
|
881
|
+
// so the upstream doesn't receive a mismatched framing and truncate or stall.
|
|
882
|
+
if (sendBody !== body) headers['content-length'] = String(sendBody.length);
|
|
883
|
+
|
|
884
|
+
// Streaming request log, opened lazily on the first terminal outcome (a
|
|
885
|
+
// pure-429-then-retry attempt writes no file, matching prior behavior). The
|
|
886
|
+
// request head+body are written once, just before the response is logged.
|
|
887
|
+
let log = null;
|
|
888
|
+
let reqLogged = false;
|
|
889
|
+
const getLog = () => (logDir ? (log ||= openRequestLog(logDir, reqId)) : null);
|
|
890
|
+
const logRequestHead = () => {
|
|
891
|
+
const l = getLog();
|
|
892
|
+
if (!l || reqLogged) return;
|
|
893
|
+
reqLogged = true;
|
|
894
|
+
const safeHeaders = { ...headers };
|
|
895
|
+
if (safeHeaders['x-api-key']) safeHeaders['x-api-key'] = safeHeaders['x-api-key'].slice(0, 15) + '...';
|
|
896
|
+
if (safeHeaders['authorization']) safeHeaders['authorization'] = safeHeaders['authorization'].slice(0, 20) + '...';
|
|
897
|
+
l.write(`=== REQUEST (account: ${account.name}, retry: ${retryCount}) ===\n${method} ${upstreamUrl}\n${formatHeaders(safeHeaders)}`);
|
|
898
|
+
if (body.length > 0) l.body('REQUEST BODY', body, req.headers['content-type']);
|
|
899
|
+
};
|
|
900
|
+
|
|
901
|
+
try {
|
|
902
|
+
// Storm control: pace requests onto a freshly-switched account so a failover
|
|
903
|
+
// burst doesn't slam it all at once and cascade (issue #84). The slot is held
|
|
904
|
+
// only until the response headers arrive — long enough to stagger the burst,
|
|
905
|
+
// then released so streaming bodies don't tie up concurrency. Fail-open: a
|
|
906
|
+
// client that disconnects while waiting just drops out.
|
|
907
|
+
if (!await accountManager.admit(account.index, () => res.destroyed)) return;
|
|
908
|
+
let upstreamRes;
|
|
909
|
+
try {
|
|
910
|
+
upstreamRes = await upstreamFetch(upstreamUrl, {
|
|
911
|
+
method,
|
|
912
|
+
headers,
|
|
913
|
+
body: ['GET', 'HEAD'].includes(method) ? undefined : sendBody,
|
|
914
|
+
redirect: 'manual',
|
|
915
|
+
}, sx, route);
|
|
916
|
+
} finally {
|
|
917
|
+
accountManager.release(account.index);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
// Extract rate limit headers
|
|
921
|
+
const rateLimitHeaders = {};
|
|
922
|
+
for (const [key, value] of upstreamRes.headers.entries()) {
|
|
923
|
+
if (key.startsWith('anthropic-ratelimit-')) {
|
|
924
|
+
rateLimitHeaders[key] = value;
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
accountManager.updateQuota(account.index, rateLimitHeaders);
|
|
928
|
+
|
|
929
|
+
// Any non-429 response is live proof a rate-limit hold no longer binds —
|
|
930
|
+
// this is what lets a revalidation probe (a throttled account selected by
|
|
931
|
+
// _selectProbe) clear its own hold and return the fleet to service.
|
|
932
|
+
if (upstreamRes.status !== 429) accountManager.clearRateLimited(account.index);
|
|
933
|
+
|
|
934
|
+
// Two kinds of 429 are handled differently below: a quota rejection rotates
|
|
935
|
+
// to another account; a transient rate-limit throttle pauses + retries the
|
|
936
|
+
// same account (never rotates — see #84).
|
|
937
|
+
if (upstreamRes.status === 429) {
|
|
938
|
+
// Clamp Retry-After to a sane window: missing/invalid falls back to 60s,
|
|
939
|
+
// and out-of-range values are bounded to [1, 300]. A negative value would
|
|
940
|
+
// otherwise bypass the wait cap — setTimeout returns immediately and a
|
|
941
|
+
// pause/hold would be armed in the past.
|
|
942
|
+
let retryAfter = parseInt(upstreamRes.headers.get('retry-after'), 10);
|
|
943
|
+
if (Number.isNaN(retryAfter)) retryAfter = 60;
|
|
944
|
+
// Discard the 429 response body
|
|
945
|
+
await upstreamRes.body?.cancel();
|
|
946
|
+
|
|
947
|
+
// Durable quota exhaustion vs. a transient rate limit. A "rejected" unified
|
|
948
|
+
// status means a quota bucket is spent, so waiting and retrying the SAME
|
|
949
|
+
// account is futile — switch to another account now (updateQuota above
|
|
950
|
+
// already recorded the spent bucket's utilization from the headers).
|
|
951
|
+
const rl = rateLimitHeaders;
|
|
952
|
+
const generalRejected = rl['anthropic-ratelimit-unified-5h-status'] === 'rejected'
|
|
953
|
+
|| rl['anthropic-ratelimit-unified-7d-status'] === 'rejected';
|
|
954
|
+
const fableRejected = rl['anthropic-ratelimit-unified-7d_oi-status'] === 'rejected' && !generalRejected;
|
|
955
|
+
if ((generalRejected || fableRejected) && retryCount < maxRetries) {
|
|
956
|
+
// A Fable-only rejection leaves the account fine for other models, so we
|
|
957
|
+
// do NOT throttle it globally — the recorded Fable utilization makes
|
|
958
|
+
// selection skip it for Fable requests only. A general rejection spends a
|
|
959
|
+
// shared bucket, so hold the whole account for its reset window.
|
|
960
|
+
if (fableRejected) {
|
|
961
|
+
console.log(`[TeamClaude] Fable weekly exhausted on "${account.name}" — switching account for this Fable request`);
|
|
962
|
+
} else {
|
|
963
|
+
const hold = Math.min(Math.max(retryAfter, 1), 3600);
|
|
964
|
+
console.log(`[TeamClaude] Quota rejection (429) on "${account.name}" — throttling ${hold}s and switching account`);
|
|
965
|
+
accountManager.markRateLimited(account.index, hold);
|
|
966
|
+
}
|
|
967
|
+
ctx.tried.add(account.index);
|
|
968
|
+
if (res.destroyed) return;
|
|
969
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
retryAfter = Math.min(Math.max(retryAfter, 1), 300);
|
|
973
|
+
|
|
974
|
+
// sx.org failover: 429s are IP-based, so retry via the proxy's egress IP.
|
|
975
|
+
// 'always' is already on sx; '429' switches direct→sx now and skips the
|
|
976
|
+
// wait (a fresh IP isn't throttled). Also arm the sticky window for MITM.
|
|
977
|
+
const nextUseSx = !!(sx?.useOn429());
|
|
978
|
+
const switchingToSx = nextUseSx && !route;
|
|
979
|
+
sx?.noteRateLimited(retryAfter);
|
|
980
|
+
|
|
981
|
+
// This is a rate-limit 429 (per-minute throttle), NOT quota exhaustion —
|
|
982
|
+
// quota rejection is handled above and is the only thing that rotates.
|
|
983
|
+
// Do NOT switch accounts here: moving the burst to the next account just
|
|
984
|
+
// throttles it too (thundering herd, #84) and discards this account's KV
|
|
985
|
+
// cache. Instead PAUSE this account so concurrent requests wait in admit()
|
|
986
|
+
// (capped, then released through a fresh ramp) instead of piling on, and
|
|
987
|
+
// retry the SAME account. The pause never marks the account throttled, so
|
|
988
|
+
// selection keeps choosing it.
|
|
989
|
+
accountManager.pauseAccount(account.index, Math.min(retryAfter, RATE_LIMIT_ABSORB_MAX_SECONDS));
|
|
990
|
+
|
|
991
|
+
// sx fresh-IP retry (still the same account) takes precedence over waiting.
|
|
992
|
+
// Bounded by retryCount like the inline-wait path below, so a persistently
|
|
993
|
+
// 429ing upstream can't loop forever through sx.
|
|
994
|
+
if (switchingToSx && retryCount < maxRetries) {
|
|
995
|
+
console.log(`[TeamClaude] 429 on "${account.name}" — retrying via sx.org (fresh egress IP)`);
|
|
996
|
+
if (res.destroyed) return;
|
|
997
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, nextUseSx);
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
// Absorb short waits inline on the same account — the client never sees the
|
|
1001
|
+
// 429. Bounded by retryCount (maxRetries = account count) so a persistently
|
|
1002
|
+
// rate-limited account can't loop forever tying up the connection.
|
|
1003
|
+
if (retryAfter <= RATE_LIMIT_ABSORB_MAX_SECONDS && retryCount < maxRetries) {
|
|
1004
|
+
console.log(`[TeamClaude] Rate-limit 429 on "${account.name}" — waiting ${retryAfter}s, retrying same account (no switch)`);
|
|
1005
|
+
await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
|
|
1006
|
+
if (res.destroyed) return;
|
|
1007
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, nextUseSx);
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
// Longer retry-after (or retries exhausted): don't hold the connection and
|
|
1011
|
+
// don't rotate — surface the 429 with retry-after so the client backs off.
|
|
1012
|
+
// The pause above keeps other requests off this account meanwhile.
|
|
1013
|
+
console.log(`[TeamClaude] Rate-limit 429 on "${account.name}" — retry-after ${retryAfter}s over inline cap; returning 429 to client (no switch)`);
|
|
1014
|
+
ctx.status = 429;
|
|
1015
|
+
if (!res.headersSent && !res.destroyed) {
|
|
1016
|
+
res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': String(retryAfter) });
|
|
1017
|
+
res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: `Rate limited; retry in ${retryAfter}s.` } }));
|
|
1018
|
+
}
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
// A 401 means the credential we injected was rejected. For an OAuth account
|
|
1023
|
+
// that usually means the access token was revoked BEFORE its clock expiry —
|
|
1024
|
+
// something else refreshed the same token family, so upstream reports it
|
|
1025
|
+
// revoked while it still looks fresh locally. ensureTokenFresh's expiry
|
|
1026
|
+
// check cannot see that (it only compares the clock), so the account would
|
|
1027
|
+
// otherwise keep serving a dead token until the token aged out, and every
|
|
1028
|
+
// request in between would surface a 401 to the client with no recovery.
|
|
1029
|
+
// Force one refresh and retry. If the refresh is itself rejected the refresh
|
|
1030
|
+
// token is dead too: ensureTokenFresh marks the account errored, and the
|
|
1031
|
+
// retry's status check rotates to another account. Bounded to one re-auth
|
|
1032
|
+
// per account per request, so a genuinely dead credential surfaces the 401
|
|
1033
|
+
// instead of looping.
|
|
1034
|
+
// A 403 ("Request not allowed") is upstream refusing THIS account outright —
|
|
1035
|
+
// not a stale token a refresh could fix, and not anything the client sent.
|
|
1036
|
+
// The client never sees the credential we inject, so it cannot act on the
|
|
1037
|
+
// rejection; Claude Code reads a 403 as "your session is dead", drops its
|
|
1038
|
+
// own login and asks for a re-login over an account problem it has no part
|
|
1039
|
+
// in. Skip the account for the rest of this request and fail over. With no
|
|
1040
|
+
// account left, the no-account branch reports a proxy error instead.
|
|
1041
|
+
if (upstreamRes.status === 403 && !res.headersSent) {
|
|
1042
|
+
await upstreamRes.body?.cancel();
|
|
1043
|
+
// A set, not a name: the no-account branch needs to tell "every account was
|
|
1044
|
+
// refused" (fail fast, nothing to wait for) from "this one was, others are
|
|
1045
|
+
// just out of quota" (still worth holding for a reset).
|
|
1046
|
+
(ctx.credentialRejected ??= new Set()).add(account.name);
|
|
1047
|
+
ctx.tried.add(account.index);
|
|
1048
|
+
console.error(`[TeamClaude] 403 on "${account.name}" — upstream refused the account credential`);
|
|
1049
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
if (upstreamRes.status === 401 && account.type === 'oauth' && account.refreshToken
|
|
1053
|
+
&& retryCount < maxRetries && !ctx.reauthed.has(account.index)) {
|
|
1054
|
+
ctx.reauthed.add(account.index);
|
|
1055
|
+
await upstreamRes.body?.cancel();
|
|
1056
|
+
console.log(`[TeamClaude] 401 on "${account.name}" — token rejected; forcing refresh and retrying`);
|
|
1057
|
+
await accountManager.ensureTokenFresh(account.index, true);
|
|
1058
|
+
if (res.destroyed) return;
|
|
1059
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
// Log the request head (once) followed by the response headers, streaming
|
|
1063
|
+
// to disk from here on.
|
|
1064
|
+
logRequestHead();
|
|
1065
|
+
getLog()?.write(`\n\n=== RESPONSE ${upstreamRes.status} ===\n${formatHeaders(upstreamRes.headers)}`);
|
|
1066
|
+
|
|
1067
|
+
ctx.status = upstreamRes.status;
|
|
1068
|
+
|
|
1069
|
+
// Build response headers (skip hop-by-hop and encoding headers). The
|
|
1070
|
+
// connection-specific names are also illegal on an HTTP/2 response — when
|
|
1071
|
+
// this runs behind the MITM's h2 server, writeHead would otherwise throw.
|
|
1072
|
+
const responseHeaders = {};
|
|
1073
|
+
for (const [key, value] of upstreamRes.headers.entries()) {
|
|
1074
|
+
if (CONNECTION_SPECIFIC_HEADERS.has(key)) continue;
|
|
1075
|
+
// Strip content-encoding/content-length since fetch may auto-decompress
|
|
1076
|
+
if (key === 'content-encoding' || key === 'content-length') continue;
|
|
1077
|
+
responseHeaders[key] = value;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
res.writeHead(upstreamRes.status, responseHeaders);
|
|
1081
|
+
|
|
1082
|
+
if (!upstreamRes.body) {
|
|
1083
|
+
const l = getLog();
|
|
1084
|
+
if (l) { l.write('\n\n=== RESPONSE BODY ===\n(empty)'); l.end(); }
|
|
1085
|
+
res.end();
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
const contentType = upstreamRes.headers.get('content-type') || '';
|
|
1090
|
+
const isStreaming = contentType.includes('text/event-stream');
|
|
1091
|
+
|
|
1092
|
+
if (isStreaming) {
|
|
1093
|
+
// Stream each chunk straight to the log as it is relayed — never hold the
|
|
1094
|
+
// whole (potentially ~1M-token) SSE body in memory.
|
|
1095
|
+
const l = getLog();
|
|
1096
|
+
const bw = l ? l.bodyWriter('RESPONSE BODY (streamed)', contentType) : null;
|
|
1097
|
+
await streamResponse(upstreamRes.body, res, account.index, accountManager, bw);
|
|
1098
|
+
l?.end();
|
|
1099
|
+
} else {
|
|
1100
|
+
const buf = Buffer.from(await upstreamRes.arrayBuffer());
|
|
1101
|
+
extractUsageFromBody(buf, account.index, accountManager);
|
|
1102
|
+
const l = getLog();
|
|
1103
|
+
if (l) { l.body('RESPONSE BODY', buf, contentType); l.end(); }
|
|
1104
|
+
res.end(buf);
|
|
1105
|
+
}
|
|
1106
|
+
} catch (err) {
|
|
1107
|
+
console.error(`[TeamClaude] Upstream error (account "${account.name}"):`, err.message);
|
|
1108
|
+
|
|
1109
|
+
logRequestHead();
|
|
1110
|
+
const l = getLog();
|
|
1111
|
+
if (l) { l.write(`\n\n=== ERROR ===\n${err.stack || err.message}`); l.end(); }
|
|
1112
|
+
|
|
1113
|
+
const isTransient = err instanceof Error &&
|
|
1114
|
+
(err.code === 'TEAMCLAUDE_HEADERS_TIMEOUT' || err.code === 'TEAMCLAUDE_BODY_TIMEOUT' ||
|
|
1115
|
+
err.name === 'TimeoutError' || err.name === 'AbortError' ||
|
|
1116
|
+
err.message.includes('fetch failed') ||
|
|
1117
|
+
err.code === 'ECONNRESET' || err.code === 'ECONNREFUSED' ||
|
|
1118
|
+
err.code === 'ETIMEDOUT' || err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
|
|
1119
|
+
err.code === 'UND_ERR_HEADERS_TIMEOUT' || err.code === 'UND_ERR_BODY_TIMEOUT');
|
|
1120
|
+
|
|
1121
|
+
// Transient network errors (including a stale-socket headers/body timeout):
|
|
1122
|
+
// close the connection and let the client retry. Failing over to another
|
|
1123
|
+
// account would not help (the poisoned fetch pool is process-wide), but the
|
|
1124
|
+
// fast failure lets Node evict the dead socket so the retry reconnects
|
|
1125
|
+
// cleanly. If headers were already sent (a mid-stream body timeout), destroy
|
|
1126
|
+
// is the only option — the client sees a broken response and retries.
|
|
1127
|
+
if (isTransient) {
|
|
1128
|
+
res.destroy();
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// Any other thrown error is a transport/stream failure, NOT proof the
|
|
1133
|
+
// account's credentials are bad — a bad credential comes back as a 401
|
|
1134
|
+
// *response*, never a throw. So don't sideline the account (that would drop
|
|
1135
|
+
// a healthy account from rotation until a credential change). Instead skip
|
|
1136
|
+
// it for the rest of THIS request only and fail over to another account.
|
|
1137
|
+
if (retryCount < maxRetries && !res.headersSent) {
|
|
1138
|
+
ctx.tried.add(account.index);
|
|
1139
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
1140
|
+
}
|
|
1141
|
+
ctx.status = 502;
|
|
1142
|
+
|
|
1143
|
+
if (!res.headersSent) {
|
|
1144
|
+
res.writeHead(502, { 'Content-Type': 'application/json' });
|
|
1145
|
+
res.end(JSON.stringify({
|
|
1146
|
+
type: 'error',
|
|
1147
|
+
error: { type: 'proxy_error', message: `Upstream error: ${err.message}` },
|
|
1148
|
+
}));
|
|
1149
|
+
} else if (!res.writableEnded) {
|
|
1150
|
+
// Error after headers were already sent (mid-stream) and it wasn't
|
|
1151
|
+
// classified transient: we can't send a status or fail over, and
|
|
1152
|
+
// streamResponse deliberately skipped res.end(). Destroy so the client
|
|
1153
|
+
// sees a broken response and retries instead of hanging on an open socket.
|
|
1154
|
+
res.destroy();
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
// Idle deadline for the RESPONSE BODY, complementing the headers timeout in
|
|
1160
|
+
// upstream-fetch.js. The headers guard only covers time-to-first-byte; once
|
|
1161
|
+
// headers arrive it is disarmed, so a network drop AFTER the stream starts would
|
|
1162
|
+
// otherwise hang the read forever (the SSE completion just goes silent mid-way).
|
|
1163
|
+
// This watchdog resets on every chunk, so a long but healthy stream is never
|
|
1164
|
+
// cut — it fires only when the socket produces nothing for the whole window,
|
|
1165
|
+
// converting a mid-stream hang into a fast failure that evicts the dead socket
|
|
1166
|
+
// (reader.cancel destroys the underlying connection on both the direct-fetch and
|
|
1167
|
+
// the sx-tunnel path, since both hand back a web ReadableStream). Override with
|
|
1168
|
+
// TEAMCLAUDE_UPSTREAM_BODY_TIMEOUT_MS.
|
|
1169
|
+
const DEFAULT_BODY_IDLE_TIMEOUT_MS = 120_000;
|
|
1170
|
+
|
|
1171
|
+
function resolveBodyIdleTimeout() {
|
|
1172
|
+
const env = Number(process.env.TEAMCLAUDE_UPSTREAM_BODY_TIMEOUT_MS);
|
|
1173
|
+
return env > 0 ? env : DEFAULT_BODY_IDLE_TIMEOUT_MS;
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
// Race a single reader.read() against an inactivity deadline. Resolves to the
|
|
1177
|
+
// read result, or rejects with a transient TEAMCLAUDE_BODY_TIMEOUT if no chunk
|
|
1178
|
+
// arrives within `ms`. The pending read is abandoned on timeout; the caller
|
|
1179
|
+
// cancels the reader (evicting the socket) in its finally block.
|
|
1180
|
+
export function readWithIdleTimeout(reader, ms) {
|
|
1181
|
+
let timer;
|
|
1182
|
+
const timeout = new Promise((_, reject) => {
|
|
1183
|
+
timer = setTimeout(() => {
|
|
1184
|
+
const err = new Error(`upstream stream idle for ${ms}ms`);
|
|
1185
|
+
err.code = 'TEAMCLAUDE_BODY_TIMEOUT';
|
|
1186
|
+
reject(err);
|
|
1187
|
+
}, ms);
|
|
1188
|
+
timer.unref?.();
|
|
1189
|
+
});
|
|
1190
|
+
const read = reader.read();
|
|
1191
|
+
// If the timeout wins the race, `read` is abandoned; swallow any later
|
|
1192
|
+
// rejection so it can't surface as an unhandledRejection.
|
|
1193
|
+
read.catch(() => {});
|
|
1194
|
+
return Promise.race([read, timeout]).finally(() => clearTimeout(timer));
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
/**
|
|
1198
|
+
* Stream an SSE response to the client, parsing usage data along the way.
|
|
1199
|
+
*/
|
|
1200
|
+
async function streamResponse(webStream, res, accountIndex, accountManager, bodyWriter) {
|
|
1201
|
+
const reader = webStream.getReader();
|
|
1202
|
+
const idleMs = resolveBodyIdleTimeout();
|
|
1203
|
+
const decoder = new TextDecoder();
|
|
1204
|
+
let sseBuffer = '';
|
|
1205
|
+
let errored = false;
|
|
1206
|
+
|
|
1207
|
+
try {
|
|
1208
|
+
while (true) {
|
|
1209
|
+
const { done, value } = await readWithIdleTimeout(reader, idleMs);
|
|
1210
|
+
if (done) break;
|
|
1211
|
+
|
|
1212
|
+
// Client disconnected — stop reading from upstream
|
|
1213
|
+
if (res.destroyed) break;
|
|
1214
|
+
|
|
1215
|
+
// Forward chunk immediately
|
|
1216
|
+
const ok = res.write(value);
|
|
1217
|
+
|
|
1218
|
+
// Append to the log as it streams (no whole-body buffering)
|
|
1219
|
+
if (bodyWriter) bodyWriter.chunk(Buffer.from(value));
|
|
1220
|
+
|
|
1221
|
+
const text = decoder.decode(value, { stream: true });
|
|
1222
|
+
|
|
1223
|
+
// Parse SSE events for usage tracking
|
|
1224
|
+
sseBuffer += text;
|
|
1225
|
+
const events = sseBuffer.split('\n\n');
|
|
1226
|
+
sseBuffer = events.pop(); // keep incomplete event
|
|
1227
|
+
|
|
1228
|
+
for (const event of events) {
|
|
1229
|
+
parseSSEUsage(event, accountIndex, accountManager);
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
// Handle backpressure — also bail out if client disconnects,
|
|
1233
|
+
// because 'drain' will never fire on a destroyed socket
|
|
1234
|
+
if (!ok) {
|
|
1235
|
+
await new Promise(resolve => {
|
|
1236
|
+
// Remove BOTH listeners when either fires: otherwise the un-fired one
|
|
1237
|
+
// (usually 'close') stays attached and accumulates one leaked listener
|
|
1238
|
+
// per backpressure cycle over a long SSE stream to a slow client.
|
|
1239
|
+
const done = () => { res.off('drain', done); res.off('close', done); resolve(); };
|
|
1240
|
+
res.once('drain', done);
|
|
1241
|
+
res.once('close', done);
|
|
1242
|
+
});
|
|
1243
|
+
if (res.destroyed) break;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// Parse any remaining buffer
|
|
1248
|
+
if (sseBuffer.trim()) {
|
|
1249
|
+
parseSSEUsage(sseBuffer, accountIndex, accountManager);
|
|
1250
|
+
}
|
|
1251
|
+
} catch (err) {
|
|
1252
|
+
// A mid-stream idle timeout (or any read error) means the upstream went
|
|
1253
|
+
// silent after headers. Rethrow to the caller's transient handler, which
|
|
1254
|
+
// destroys the client connection so the truncated stream is NOT ended
|
|
1255
|
+
// cleanly (a clean res.end() would look like a complete response and
|
|
1256
|
+
// suppress the client's retry). reader.cancel() in finally evicts the socket.
|
|
1257
|
+
errored = true;
|
|
1258
|
+
throw err;
|
|
1259
|
+
} finally {
|
|
1260
|
+
// Cancel upstream reader to stop consuming data nobody needs (and, on the
|
|
1261
|
+
// timeout path, to destroy the dead socket so the pool drops it).
|
|
1262
|
+
reader.cancel().catch(() => {});
|
|
1263
|
+
if (!errored && !res.writableEnded) res.end();
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
function parseSSEUsage(event, accountIndex, accountManager) {
|
|
1268
|
+
const dataLine = event.split('\n').find(l => l.startsWith('data: '));
|
|
1269
|
+
if (!dataLine) return;
|
|
1270
|
+
|
|
1271
|
+
try {
|
|
1272
|
+
const data = JSON.parse(dataLine.slice(6));
|
|
1273
|
+
if (data.type === 'message_start' && data.message?.usage) {
|
|
1274
|
+
accountManager.updateUsage(accountIndex, data.message.usage.input_tokens, 0);
|
|
1275
|
+
} else if (data.type === 'message_delta' && data.usage) {
|
|
1276
|
+
accountManager.updateUsage(accountIndex, 0, data.usage.output_tokens);
|
|
1277
|
+
}
|
|
1278
|
+
} catch {
|
|
1279
|
+
// not valid JSON, skip
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
function extractUsageFromBody(buffer, accountIndex, accountManager) {
|
|
1284
|
+
try {
|
|
1285
|
+
const json = JSON.parse(buffer.toString());
|
|
1286
|
+
if (json.usage) {
|
|
1287
|
+
accountManager.updateUsage(accountIndex, json.usage.input_tokens, json.usage.output_tokens);
|
|
1288
|
+
}
|
|
1289
|
+
} catch {
|
|
1290
|
+
// not JSON or no usage
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// Rewrite the `model` field in a JSON request body using a per-account map.
|
|
1295
|
+
// Returns the original buffer unchanged if the model isn't in the map or the
|
|
1296
|
+
// body isn't valid JSON, so non-messages endpoints pass through safely.
|
|
1297
|
+
// Exported for tests.
|
|
1298
|
+
export function rewriteModel(body, modelMap) {
|
|
1299
|
+
try {
|
|
1300
|
+
const obj = JSON.parse(body.toString('utf8'));
|
|
1301
|
+
if (obj.model && modelMap[obj.model]) {
|
|
1302
|
+
obj.model = modelMap[obj.model];
|
|
1303
|
+
return Buffer.from(JSON.stringify(obj), 'utf8');
|
|
1304
|
+
}
|
|
1305
|
+
} catch { /* not JSON — pass through unchanged */ }
|
|
1306
|
+
return body;
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
function computeRetryAfter(accounts) {
|
|
1310
|
+
let soonest = Infinity;
|
|
1311
|
+
for (const acct of accounts) {
|
|
1312
|
+
const reset = acct.rateLimitedUntil || acct.quota.resetsAt;
|
|
1313
|
+
if (reset) {
|
|
1314
|
+
const ms = new Date(reset).getTime() - Date.now();
|
|
1315
|
+
if (ms < soonest) soonest = ms;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
return soonest === Infinity ? 60 : Math.max(1, Math.ceil(soonest / 1000));
|
|
1319
|
+
}
|