@rikcodes/teamclaude 1.1.20-rik.7 → 1.1.20-rik.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rikcodes/teamclaude",
-  "version": "1.1.20-rik.7",
+  "version": "1.1.20-rik.9",
   "description": "Multi-account proxy for Claude Code and Codex: pools Claude Max, ChatGPT/Codex, API-key and third-party backend accounts, and rotates on quota",
   "type": "module",
   "main": "src/index.js",

package/src/crash-log.js

@@ -1,5 +1,14 @@
 import { appendFileSync } from 'node:fs';
 
+/** Did a write fail because whatever was reading has gone?
+ *
+ *  Node reports this asynchronously when the stream is non-blocking, so it
+ *  arrives as an event rather than at the call site and no `try` around the
+ *  write can catch it. */
+function isBrokenPipe(err) {
+  return err?.code === 'EPIPE' || err?.code === 'ERR_STREAM_DESTROYED';
+}
+
 /**
  * Write a fatal error to `path` before the process dies.
  *
@@ -11,16 +20,39 @@ import { appendFileSync } from 'node:fs';
  * Handling these events replaces Node's own behaviour, so this must do what
  * Node would: report and exit non-zero. Continuing after an uncaught exception
  * would leave the proxy running on unknown state.
+ *
+ * A broken pipe is the one exception, and it is deliberate. The terminal going
+ * away says nothing about the proxy's state — every account, route and inflight
+ * request is exactly as it was — so ending the process punishes every routed
+ * session for a closed pane. It happened three times in two days here: the TUI
+ * writes to a non-blocking stdout, so the failure could not be caught where it
+ * was written, and each death also orphaned a sidecar on its port. Recorded
+ * once, because a stream dying is worth knowing, then survived. Once is enough:
+ * the stream stays dead, and a storm of them would say nothing new.
  * @param {string} path
  * @param {{ exit?: (code: number) => void, log?: { write: (s: string) => void } }} [opts]
  */
 export function installCrashHandlers(path, { exit = process.exit, log = process.stderr } = {}) {
+  let brokenPipeRecorded = false;
+  // 0600: a stack can carry request context. A write failure (read-only home,
+  // full disk) must not mask the crash itself — stderr still gets the entry.
+  const record = (/** @type {string} */ entry) => {
+    try { appendFileSync(path, entry, { mode: 0o600 }); } catch { /* report to stderr regardless */ }
+  };
   const report = (/** @type {string} */ kind) => (/** @type {any} */ err) => {
     const stack = err?.stack || String(err);
-    const entry = `\n=== ${new Date().toISOString()} ${kind} ===\n${stack}\n`;
-    // 0600: a stack can carry request context. A write failure (read-only home,
-    // full disk) must not mask the crash itself — stderr still gets the entry.
-    try { appendFileSync(path, entry, { mode: 0o600 }); } catch { /* report to stderr regardless */ }
+    // A bare "Error: write EPIPE" names neither the stream nor the call. The
+    // code and syscall are what turn the next one of these into a diagnosis
+    // rather than an afternoon of inference.
+    const detail = [err?.code, err?.syscall].filter(Boolean).join(' ');
+    const entry = `\n=== ${new Date().toISOString()} ${kind}${detail ? ` (${detail})` : ''} ===\n${stack}\n`;
+    if (isBrokenPipe(err)) {
+      if (!brokenPipeRecorded) { brokenPipeRecorded = true; record(entry); }
+      // Deliberately not written to `log`: that is very likely the stream that
+      // just failed, and would fail again.
+      return;
+    }
+    record(entry);
     log.write(entry);
     exit(1);
   };

package/src/index.js

@@ -323,7 +323,7 @@ async function serverCommand() {
 
   // Periodically persist quota (and once more on shutdown) to the state file.
   const persistQuotaState = () =>
-    saveState({ quota: accountManager.exportQuotaState(), clients: clientUsage.export(), usageDimensions: dimensionUsage.export() })
+    saveState({ quota: accountManager.exportQuotaState(), clients: clientUsage.export(), usageDimensions: dimensionUsage.export(), sidecars: sidecar?.exportPids() || savedState?.sidecars || {} })
       .catch(err => console.error(`[TeamClaude] Failed to save quota state: ${err.message}`));
   let quotaSaveInterval = null;
 
@@ -709,7 +709,13 @@ async function serverCommand() {
   warmer.start();
 
   // Launch supervised sidecars (no-op when config.sidecars is empty).
-  sidecar = new Sidecar(config.sidecars);
+  sidecar = new Sidecar(config.sidecars, {
+    // A sidecar outlives a server that was killed rather than asked to stop,
+    // and goes on holding its port. The pid recorded here is the only thing
+    // that lets the next start tell its own leftover from a stranger's process.
+    savedPids: savedState?.sidecars || null,
+    onPids: () => { persistQuotaState(); },
+  });
   sidecar.start();
 
   // Background self-update for a backgrounded (headless) server. Skipped under

package/src/server.js

@@ -63,6 +63,17 @@ const RATE_LIMIT_ABSORB_MAX_SECONDS =
   Number(process.env.TEAMCLAUDE_RATE_LIMIT_ABSORB_MAX_SECONDS) || 60;
 const OAUTH_ENTITLEMENT_ERROR_CODE = 'oauth_not_allowed_for_organization';
 const ERROR_BODY_INSPECTION_LIMIT = 64 * 1024;
+// How long an idle keep-alive connection is held open.
+//
+// Node's default is 5s, but a client's connection pool may hold the same socket
+// far longer, and whoever closes first wins: when the server does, the client
+// finds out only by writing to a socket that is already gone, which surfaces as
+// a request that fails in ~130ms with no upstream involvement. The Codex
+// sidecar is such a client — reqwest's pool_idle_timeout defaults to 90s and it
+// never overrides it — so outlive the longest pool and let the client always be
+// the one to close. headersTimeout bounds an in-progress request's headers, not
+// the idle gap between them (measured), so it is deliberately left alone.
+const KEEP_ALIVE_TIMEOUT_MS = 120_000;
 
 /** Classify only the structured organization-policy denial observed upstream.
  * Message text and generic permission errors are deliberately not enough. */
@@ -476,6 +487,7 @@ export function createProxyServer(accountManager, config, hooks = {}, sx = null,
   const egress = createEgressGuard(config, console.error);
   const forward = createProxyRequestListener({ accountManager, upstream, logDir, hooks, sx, holdMs, config, egress, clientUsage, dimensionUsage });
   const server = http.createServer(requestHandler);
+  server.keepAliveTimeout = KEEP_ALIVE_TIMEOUT_MS;
 
   // What bounds a directory of one-shot dumps is deleting the expired ones, not
   // rotating a growing file. Swept once at startup, because a backlog is usually

package/src/sidecar.js

@@ -11,8 +11,14 @@
 // stdout is ignored (sidecars keep their own log files); stderr's last few
 // lines are kept in a ring buffer so `getStatus()` can say WHY a sidecar is
 // crash-looping without anyone hunting for its logs.
+//
+// One failure mode earned its own handling. A sidecar binds a fixed port, so a
+// copy that outlives its server keeps that port and every later server fails to
+// bind — reported, before this, as a bare "code 1" retried forever, while the
+// old process quietly went on serving. Two halves: name the conflict instead of
+// guessing at it, and reap a leftover this server can prove is its own.
 
-import { spawn } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 
 /** Delay before restart attempt N (0-based): base, doubled per consecutive
  *  crash, capped. Pure so the schedule is testable without timers. */
@@ -20,6 +26,13 @@ export function restartDelayMs(restarts, { baseRestartMs, maxRestartMs }) {
   return Math.min(baseRestartMs * 2 ** restarts, maxRestartMs);
 }
 
+/** Does this stderr line say the port is already taken? The wording is the
+ *  runtime's, not ours — Rust prints the OS string, Node prints EADDRINUSE —
+ *  so match the phrasings rather than one library's spelling. */
+export function isBindConflict(line) {
+  return /address already in use|address in use|EADDRINUSE/i.test(String(line));
+}
+
 export class Sidecar {
   constructor(entries, {
     spawnFn = defaultSpawn,
@@ -28,6 +41,10 @@ export class Sidecar {
     stableMs = 30_000,
     stderrTailLines = 20,
     log = console.log,
+    savedPids = null,
+    onPids = null,
+    readProcess = defaultReadProcess,
+    killFn = (pid, signal) => process.kill(pid, signal),
   } = {}) {
     this.entries = Array.isArray(entries) ? entries : [];
     this.spawnFn = spawnFn;
@@ -36,6 +53,12 @@ export class Sidecar {
     this.stableMs = stableMs;
     this.stderrTailLines = stderrTailLines;
     this.log = log;
+    // Pids this server recorded on a previous run, by entry name. Only ever
+    // used to recognise our own leftovers; see _reapOrphan.
+    this.savedPids = (savedPids && typeof savedPids === 'object') ? { ...savedPids } : {};
+    this.onPids = onPids;
+    this.readProcess = readProcess;
+    this.killFn = killFn;
     this.stopping = false;
     // Per-entry runtime state, keyed by entry (parallel array to this.entries).
     this.states = this.entries.map(entry => ({
@@ -46,11 +69,15 @@ export class Sidecar {
       lastExit: null,
       timer: null,
       stderrTail: [],
+      blocked: null,   // stderr line proving the port is held, else null
     }));
   }
 
   start() {
-    for (const state of this.states) this._spawn(state);
+    for (const state of this.states) {
+      this._reapOrphan(state);
+      this._spawn(state);
+    }
   }
 
   stop() {
@@ -68,10 +95,51 @@ export class Sidecar {
       pid: state.child?.pid ?? null,
       restarts: state.restarts,
       lastExit: state.lastExit,
+      // A held port is not a crash: it says nothing is wrong with the binary
+      // and everything is wrong with the port, which is a different fix.
+      blocked: !!state.blocked,
+      blockedReason: state.blocked,
       stderrTail: [...state.stderrTail],
     }));
   }
 
+  /** Last known pid per entry name, for the owner to persist.
+   *
+   *  Deliberately the last pid rather than only a live one. A sidecar that is
+   *  down — crashed, or blocked because something else holds its port — is
+   *  exactly when the pid is worth keeping, and a record that emptied itself
+   *  the moment the child exited would be absent whenever it was needed.
+   */
+  exportPids() {
+    return { ...this.savedPids };
+  }
+
+  /** Kill a sidecar this server started that outlived it.
+   *
+   *  A server killed with SIGKILL never reaches stop(), and the child does not
+   *  die with it (see defaultSpawn), so the sidecar keeps running and holding
+   *  its port. Three things must all hold before anything is signalled: the pid
+   *  was recorded by us, it is now reparented to init (nothing else supervises
+   *  it), and it is still running the same program. A recycled pid fails the
+   *  last test, and a sidecar belonging to another live server fails the second,
+   *  so neither is touched. The port may take a moment to free after this; the
+   *  ordinary restart backoff covers that.
+   */
+  _reapOrphan(state) {
+    const pid = Number(this.savedPids[state.entry.name]?.pid);
+    if (!Number.isInteger(pid) || pid <= 1) return;
+    let info = null;
+    try { info = this.readProcess(pid); } catch { return; }
+    if (!info) return;                 // not running: already gone
+    if (info.ppid !== 1) return;       // still has a parent, so not ours to reap
+    const program = state.entry.command?.[0];
+    if (!program || !String(info.command || '').includes(program)) return; // pid recycled
+    try {
+      this.killFn(pid, 'SIGTERM');
+      this.log(`[TeamClaude] Sidecar "${state.entry.name}": reaped orphan pid ${pid} left by a previous run`);
+    } catch { /* exited between the check and the signal, which is the goal anyway */ }
+  }
+
   _spawn(state) {
     const { entry } = state;
     const [command, ...args] = entry.command;
@@ -89,6 +157,12 @@ export class Sidecar {
     }
     state.child = child;
     state.startedAt = Date.now();
+    state.blocked = null;   // a fresh attempt: whatever the last one hit is history
+    // Recorded now, while the pid is known. The record has to outlive the child
+    // itself: the case it exists for is this server being killed outright, and
+    // by then there is nobody left to write anything down.
+    this.savedPids[entry.name] = { pid: child.pid, command: entry.command?.[0] ?? null };
+    this._publishPids();
     child.stderr?.on('data', (chunk) => this._recordStderr(state, chunk));
     child.once('error', (err) => {
       if (state.child !== child) return;
@@ -108,7 +182,12 @@ export class Sidecar {
     state.lastExit = lastExit;
     if (this.stopping) return;
     const delay = restartDelayMs(state.restarts, this);
-    this.log(`[TeamClaude] Sidecar "${state.entry.name}" down (${lastExit}); restarting in ${Math.round(delay / 1000)}s`);
+    // Retrying still makes sense while blocked — a port held by a process we
+    // did not start frees when that process ends, and this is the only thing
+    // watching for it — but saying "down (code 1)" about it does not.
+    this.log(state.blocked
+      ? `[TeamClaude] Sidecar "${state.entry.name}" cannot bind: ${state.blocked}; retrying in ${Math.round(delay / 1000)}s`
+      : `[TeamClaude] Sidecar "${state.entry.name}" down (${lastExit}); restarting in ${Math.round(delay / 1000)}s`);
     state.restarts += 1;
     state.timer = setTimeout(() => {
       state.timer = null;
@@ -117,8 +196,16 @@ export class Sidecar {
     state.timer.unref?.();
   }
 
+  _publishPids() {
+    if (!this.onPids) return;
+    // Persistence is the owner's business and best-effort: failing to record a
+    // pid costs a manual reap later, never this start.
+    try { this.onPids(this.exportPids()); } catch { /* not worth failing a spawn over */ }
+  }
+
   _recordStderr(state, chunk) {
     const lines = String(chunk).split('\n').map(s => s.trim()).filter(Boolean);
+    for (const line of lines) if (isBindConflict(line)) state.blocked = line;
     state.stderrTail.push(...lines);
     if (state.stderrTail.length > this.stderrTailLines) {
       state.stderrTail.splice(0, state.stderrTail.length - this.stderrTailLines);
@@ -127,8 +214,28 @@ export class Sidecar {
 }
 
 // Real spawner: stdout ignored (sidecars log to their own files), stderr piped
-// for the ring buffer. detached:false so the child dies with us as a backstop.
+// for the ring buffer.
+//
+// The child stays in our process group (`detached` defaults to false), so an
+// interactive ctrl-c reaches it too. That is NOT the same as dying with us:
+// there is no PDEATHSIG on macOS, and a server killed with SIGKILL leaves the
+// sidecar running and holding its port. stop() is the ordinary path out;
+// Sidecar._reapOrphan covers the rest.
 /** @param {{name?: string, command: string, args: string[], env: Record<string, string|undefined>}} spec */
 function defaultSpawn({ command, args, env }) {
   return spawn(command, args, { env, stdio: ['ignore', 'ignore', 'pipe'] });
 }
+
+/** Parent pid and command line for a pid, or null when it is not running.
+ *
+ *  `ps` rather than /proc, which macOS does not have. This runs once per
+ *  sidecar at startup, never on a request path.
+ *  @param {number} pid
+ *  @returns {{ppid: number, command: string}|null}
+ */
+function defaultReadProcess(pid) {
+  const r = spawnSync('ps', ['-o', 'ppid=,command=', '-p', String(pid)], { encoding: 'utf8' });
+  const out = r.status === 0 ? String(r.stdout || '').trim() : '';
+  const m = out.match(/^(\d+)\s+(.*)$/s);
+  return m ? { ppid: Number(m[1]), command: m[2].trim() } : null;
+}

package/src/tui.js

@@ -586,13 +586,16 @@ export class TUI {
     this._setStdoutBlocking(true);
     // Blocking again means a failed write throws here instead of arriving as
     // an event, and a terminal that has already gone will fail. Restoring the
-    // screen is best-effort: there is nobody left to restore it for. The error
-    // listener outlives this write, so a late async failure is absorbed too.
+    // screen is best-effort: there is nobody left to restore it for.
     try { process.stdout.write(`${ESC}?25h${ESC}?1049l`); } catch { /* terminal already gone */ }
-    if (this._stdoutErrorHandler) {
-      process.stdout.removeListener('error', this._stdoutErrorHandler);
-      this._stdoutErrorHandler = null;
-    }
+    // The error listener stays. Flipping back to blocking does not make writes
+    // ALREADY QUEUED synchronous, so a paint still in flight can fail after
+    // this point, and shutdown() runs well past it — stopping the prober, the
+    // warmer and the sidecar, then awaiting a state save. An earlier version
+    // removed the listener here while claiming it outlived the write; it did
+    // not, and the proxy died of an unhandled EPIPE in exactly that window. It
+    // only sets a flag, so leaving it attached for the rest of the process
+    // costs nothing.
     try { process.stdin.setRawMode(false); } catch {}
     process.stdin.pause();
   }
@@ -1707,10 +1710,13 @@ export class TUI {
       // Matched by name: a sidecars[] entry and the account that routes to it
       // are named by the same operator, and nothing else pairs them.
       const proc = sidecars.find(sc => sc.name === a.name) || null;
+      // A held port reads as a crash loop but is not one: the binary is fine and
+      // something else owns the address, which is a different thing to go and fix.
       const state = a.disabled ? red('disabled')
         : a.rateLimitedUntil > Date.now() ? yellow('throttled')
-          : proc && !proc.running ? red(`down (${proc.lastExit || 'restarting'})`)
-            : proc ? green('up') : green('ok');
+          : proc?.blocked ? red('port in use')
+            : proc && !proc.running ? red(`down (${proc.lastExit || 'restarting'})`)
+              : proc ? green('up') : green('ok');
       const pid = proc?.running ? dim(` pid ${proc.pid}`) : '';
       const restarts = proc?.restarts ? yellow(` ${proc.restarts} restarts`) : '';
       return ` ${dim('⚙')} ${a.name} ${dim('→')} ${dim(host)}  ${state}${pid}${restarts}`;