agent-dag 1.42.0 → 1.44.1

package/src/server/claude-accounts.mjs

@@ -48,6 +48,85 @@ let _cacheAt = 0;
 // tracks what claude-swap is doing. No network cost to amortise.
 const CACHE_MS = 5_000;
 
+// ── what a forced read may cost ──────────────────────────────────────────────
+//
+// Until #604 the cache above was the whole of the admission control here, and
+// `force` walked straight past it. `handleClaudeAccounts` reads `refresh=1` off
+// the query string and passes it through, and reads on this server are
+// deliberately open — `isTrustedRead` does not apply the `Sec-Fetch-Site` test
+// that `isTrustedMutation` does, because a cross-site read of
+// `http://127.0.0.1:4317` is an ordinary top-level navigation — so any page the
+// user had open could run a `?refresh=1` loop and get one roster read per
+// request, concurrently.
+//
+// What that buys is the cheapest of the six forcible routes and it is worth
+// saying so plainly rather than dressing it up: two small local JSON reads,
+// sequence.json and cache/usage.json, plus a call into nudgeCollector that is
+// throttled on its own terms and spawns nothing when nothing is due. There is no
+// network here by design — see the note at the top of this file — and no
+// subprocess per request. This is the SHAPE the four routes before it were fixed
+// for, not a cost anyone would have noticed.
+//
+// The reason to fix it anyway is the second half. This is the only one of the
+// six whose cache is invalidated from elsewhere — nine call sites in four
+// modules: six mutations in cswap-admin.mjs, an auto-switch in cswap-auto.mjs, a
+// manual one in index.mjs, and the first `cswap add` below — and #582 has
+// already shown what a read that started before an invalidation does when it
+// lands after one. So the in-flight slot this route was missing arrives with the
+// generation guard that makes it safe, rather than after the next bug report.
+const FORCE_POLL_MS = 60_000;
+
+// A read in progress, offered to callers that arrive while it is running.
+let _inflight = null;
+// Stamped when a read STARTS rather than when it lands: what the floor rations
+// is the trip to disk, and one that is still running has already been paid for.
+let _lastReadAt = 0;
+// Which roster the reading below is about — as a counter, because the answer is
+// about whichever account claude-swap's store says is active, and that can move
+// under a read that is already running. invalidateClaudeAccountsCache bumps it;
+// every write is stamped with the value that was current when the read STARTED.
+// quota.mjs's `_generation`, for quota.mjs's reason (#582).
+let _generation = 0;
+
+/**
+ * Whether we may go to disk for the roster again.
+ *
+ * The same shape and the same minute as quota.mjs's `maySelfPoll`, and exported
+ * for the same reason it is: this is the rule, it is pure, and it belongs
+ * somewhere a test can point at it.
+ *
+ * Two intervals, like `maySelfPoll`'s. A forced read may beat the cache, but not
+ * turn into a poll loop when the button is held down, so it takes the minute the
+ * other four forcible routes use. An unforced read takes the cache's own
+ * interval — it is the panel's ordinary poll, the cache above has already
+ * answered it, and measuring from the START of the last read rather than from
+ * its end is the only difference between the two rules.
+ */
+export function mayReadAccounts({ now, force, lastReadAt }) {
+  return now - lastReadAt >= (force ? FORCE_POLL_MS : CACHE_MS);
+}
+
+/**
+ * The answer to a read the floor refused.
+ *
+ * A reading, not an error. AccountsPanel renders `data.accounts`, and an
+ * `{ ok: false }` refusal would empty the roster for a minute — the deck
+ * teaching itself a new failure mode in order to defend against a loop nobody
+ * ran. `stale` is the flag quota.mjs, codex-quota.mjs and codex-usage.mjs all
+ * use for exactly this, and every account row already carries its own
+ * `fetchedAt` from claude-swap's store, which is the age the panel draws and
+ * which nothing here touches.
+ */
+function heldReading(now) {
+  if (_cache) return { ..._cache, stale: true };
+  // Unreachable in practice, and spelled the way codex-quota.mjs and
+  // codex-usage.mjs spell the same state. Every outcome below is cached, a read
+  // still running is served by `_inflight`, and the one moment `_cache` is empty
+  // with a recent stamp — just after an invalidation — is exactly the moment
+  // invalidateClaudeAccountsCache clears the stamp as well.
+  return { ok: false, reason: "waiting", fetchedAt: now };
+}
+
 // Past this, claude-swap's own numbers are old enough that showing them
 // without a marker would misrepresent them (its own trust ceiling is 3600s).
 const STALE_AFTER_MS = 15 * 60_000;
@@ -255,8 +334,51 @@ function lane(id, label, win) {
 export async function fetchClaudeAccounts({ force = false } = {}) {
   const now = Date.now();
   if (!force && _cache && now - _cacheAt < CACHE_MS) return _cache;
+  // Offered before the floor: a read that has not finished yet is a reading
+  // newer than the cache, which is what refresh asked for, and joining it costs
+  // nothing.
+  if (_inflight) return _inflight;
+  if (!mayReadAccounts({ now, force, lastReadAt: _lastReadAt })) return heldReading(now);
+  _lastReadAt = now;
+
+  // `_inflight === mine` rather than a bare clear, which is quota.mjs's guard
+  // and is here for quota.mjs's reason: invalidateClaudeAccountsCache drops
+  // `_inflight` so the next caller starts a read that knows the roster moved,
+  // and that read installs its own promise here. A read from before the switch
+  // finishing afterwards would otherwise clear the NEW one on its way out.
+  const mine = readRoster(now, _generation)
+    .finally(() => { if (_inflight === mine) _inflight = null; });
+  _inflight = mine;
+  return mine;
+}
 
-  const finish = (r) => { _cache = r; _cacheAt = Date.now(); return r; };
+/**
+ * The read itself, split out from the admission control above it so the guard is
+ * readable as the four lines it is.
+ *
+ * `gen` is the generation that was current when this read STARTED, and finish()
+ * refuses to write anything under it once that has moved.
+ *
+ * Every write here happens after at least one await, and
+ * invalidateClaudeAccountsCache clears variables — which does nothing to a
+ * function that is already running and still holds the old roster in a local. So
+ * a switch landing mid-read was followed, milliseconds later, by the pre-switch
+ * roster being written straight back over the cleared cache: the invalidation
+ * looked like it worked and was undone before the next poll could observe it,
+ * and the panel went on showing the account the user had just switched away
+ * from. That is #582's defect, in the module #582 did not touch.
+ *
+ * The read is deliberately not cancelled. Whoever asked for it is still owed an
+ * answer, and the answer is not wrong — it is about a roster that has since
+ * moved, which makes it a fine return value and a bad cached one.
+ */
+async function readRoster(now, gen) {
+  const finish = (r) => {
+    if (gen !== _generation) return r;
+    _cache = r;
+    _cacheAt = Date.now();
+    return r;
+  };
 
   const root = backupRoot();
   const seq  = await readJson(join(root, "sequence.json"));
@@ -395,9 +517,31 @@ export async function requestCollection() {
   return _lastNudge !== before;
 }
 
+/**
+ * Forget the roster, because something just made it wrong.
+ *
+ * Called from nine places in four modules — every `cswap` mutation the deck
+ * performs — and every one of them is behind a POST that `isTrustedMutation`
+ * guards, so nothing a page can send in a loop reaches this.
+ *
+ * Three things go besides the reading itself:
+ *
+ *   `_generation`  so a read that STARTED before this call cannot write its
+ *                  answer into the cache afterwards. See finish() in readRoster.
+ *   `_inflight`    so a caller arriving after the switch is not handed the read
+ *                  that began before it — joining a run is only free when the
+ *                  run is still about the right thing.
+ *   `_lastReadAt`  so the very next read is real work rather than a refusal.
+ *                  Every one of these call sites is followed by the panel
+ *                  reloading with ?refresh=1, and a floor that answered THAT
+ *                  with the pre-switch roster would make the guard the bug.
+ */
 export function invalidateClaudeAccountsCache() {
   _cache = null;
   _cacheAt = 0;
+  _generation++;
+  _inflight = null;
+  _lastReadAt = 0;
 }
 
 /**

package/src/server/codex-quota.mjs

@@ -243,9 +243,84 @@ async function requestUsage(base, auth) {
 // to race over the single-use refresh token.
 let _inflight = null;
 
+// ── what a forced read may cost ────────────────────────────────────────────
+// The floor between two reads WE pay for, and it is the same number and the
+// same rule quota.mjs gives the Claude half — see FORCE_POLL_MS and maySelfPoll
+// there. The two routes are four lines apart in the router and had no business
+// disagreeing about what `?refresh=1` costs.
+//
+// `force` used to mean "skip the cache", and the cache was the ONLY thing
+// between a caller and chatgpt.com. `_inflight` deduplicates callers that
+// overlap and nothing else, so a caller that waits for one fetch to settle and
+// then asks again got a fresh round trip every time — two authenticated HTTPS
+// GETs carrying the user's live ChatGPT session, as fast as the round trip
+// allows, from any page the user happens to have open (#580). Reads on this
+// server are deliberately open (isTrustedRead), so "any page" is the real
+// threat model rather than a hypothetical one.
+//
+// The sharper half is the credential rather than the traffic. On a 401
+// doFetchCodexQuota spends the SINGLE-USE refresh token via forceCodexRefresh,
+// and `staleAccessToken` only stops that happening twice for the same rejected
+// token — every turn re-reads auth.json and sees the token the previous turn
+// rotated to, so a backend that keeps answering 401 rotated a fresh credential
+// once per request, racing the Codex CLI for each one. codex-auth.mjs's own
+// EXPIRY_SKEW_MS comment says what losing that race costs the user: a
+// `refresh_token_reused` that reads as "your login is broken", recoverable only
+// with `codex login`.
+const FORCE_POLL_MS = 60_000;
+
+// Set from a 429 or a rejected refresh: a backend that is refusing us must not
+// be asked once per request, whoever is asking. Same shape as quota.mjs's
+// _rateLimitedUntil, which is likewise never beaten by force.
+let _rateLimitedUntil = 0;
+const COOLDOWN_MS = 5 * 60_000;
+
+// Stamped when a fetch STARTS rather than when it lands: what the floor is
+// rationing is the round trip, and one that is still in flight has already been
+// paid for.
+let _lastFetchAt = 0;
+
+/**
+ * Whether we may spend a request of the user's ChatGPT session right now.
+ *
+ * Exported for tests, for the same reason quota.mjs exports maySelfPoll: this
+ * is the rule, it is pure, and it is worth pinning down away from the fetch it
+ * guards.
+ */
+export function mayFetchQuota({ now, lastFetchAt, rateLimitedUntil }) {
+  if (now < rateLimitedUntil) return false;
+  return now - lastFetchAt >= FORCE_POLL_MS;
+}
+
+/**
+ * The answer to a read the floor refused.
+ *
+ * A reading, not an error — a user who clicks ↻ twice in a second must get the
+ * numbers they already have rather than a red hint, which is exactly what
+ * quota.mjs does with `{ ...held, stale: true }`. The timestamp stays the one
+ * the data was fetched at, so the panel's age label keeps telling the truth
+ * instead of vouching for a reading it did not take.
+ */
+function heldReading(now) {
+  if (_cache) return { ..._cache, stale: true };
+  // Only reachable before the first fetch has ever landed — `finish` caches
+  // every outcome, failures included — and spelled the way the Claude side
+  // spells the same two states.
+  return { ok: false, reason: now < _rateLimitedUntil ? "rate_limited" : "waiting", fetchedAt: now };
+}
+
 export function fetchCodexQuota({ force = false } = {}) {
-  if (!force && _cache && Date.now() - _cacheAt < CACHE_MS) return Promise.resolve(_cache);
-  _inflight ??= doFetchCodexQuota().finally(() => { _inflight = null; });
+  const now = Date.now();
+  if (!force && _cache && now - _cacheAt < CACHE_MS) return Promise.resolve(_cache);
+  // Joining a run already in flight costs nothing, so it is offered before the
+  // floor: what refresh asks for is a reading newer than the cache, and a fetch
+  // that has not landed yet is one.
+  if (_inflight) return _inflight;
+  if (!mayFetchQuota({ now, lastFetchAt: _lastFetchAt, rateLimitedUntil: _rateLimitedUntil })) {
+    return Promise.resolve(heldReading(now));
+  }
+  _lastFetchAt = now;
+  _inflight = doFetchCodexQuota().finally(() => { _inflight = null; });
   return _inflight;
 }
 
@@ -256,6 +331,13 @@ async function doFetchCodexQuota() {
   // shortens the effective TTL for no reason.
   const finish = (r) => { _cache = r; _cacheAt = Date.now(); return r; };
   const fail   = (reason) => finish({ ok: false, reason, fetchedAt: started });
+  // A refusal we were told about, rather than one we inferred: back off further
+  // than the ordinary floor before asking again. `retry-after` is honoured when
+  // the backend sends one, because it knows better than the constant does.
+  const cooldown = (res) => {
+    const after = parseInt(res?.headers?.get?.("retry-after") ?? "", 10);
+    _rateLimitedUntil = Date.now() + (Number.isFinite(after) ? after * 1000 : COOLDOWN_MS);
+  };
 
   let auth, base, res;
   try {
@@ -294,12 +376,22 @@ async function doFetchCodexQuota() {
     // destroyed.
     if (res.status === 401) {
       const refreshed = await forceCodexRefresh(auth.accessToken);
-      if (!refreshed.ok) return fail(refreshed.reason);
+      if (!refreshed.ok) {
+        // The credential is gone and only `codex login` brings it back, so
+        // rotating another single-use token at the next request would burn the
+        // one the CLI is still holding. Wait.
+        if (refreshed.reason === "refresh_rejected") cooldown(null);
+        return fail(refreshed.reason);
+      }
       auth = refreshed;
       res  = await requestUsage(base, auth);
     }
 
     if (!res.ok) {
+      // A second 401 means the token we just rotated to was rejected as well —
+      // the case that turned into one rotation per request. 429 is the backend
+      // saying the same thing in the ordinary way.
+      if (res.status === 401 || res.status === 429) cooldown(res);
       return fail(res.status === 401 ? "refresh_rejected" : `http_${res.status}`);
     }
   } catch (err) {

package/src/server/codex-usage.mjs

@@ -12,6 +12,88 @@ let _cache = null;
 let _cacheAt = 0;
 const CACHE_MS = 60_000;
 
+// ── what a forced read may cost ─────────────────────────────────────────────
+// Until #600 the cache above was the whole of the admission control here, and
+// `force` walked straight past it. That made one GET worth a full week of the
+// rollout tree: listRolloutFiles(WINDOW_7D_MS) and then a read of every file it
+// returns. Measured on this repo's machine against 280 rollouts of ~90KB — a
+// week of ordinary use — one forced call is 685ms, 280 file opens and a peak of
+// four descriptors. Nothing bounded how many of those calls ran at once, and
+// the cost scaled exactly linearly: 16 concurrent forced reads were 4,480 opens
+// and 64 descriptors, 128 were 35,840 opens, 512 descriptors and 54.7s.
+//
+// Reads on this server are deliberately open — `isTrustedRead` does not apply
+// the `Sec-Fetch-Site` test that `isTrustedMutation` does, because a cross-site
+// read of `http://127.0.0.1:4317` is an ordinary top-level navigation — so any
+// page the user has open could run
+//
+//     for (;;) fetch("http://127.0.0.1:4317/api/codex-usage?refresh=1",
+//                    { mode: "no-cors" });
+//
+// and get one of those scans per request. MAX_PARALLEL_READS below bounds the
+// fan-out WITHIN one call, and it exists because opening a week of rollouts at
+// once risked EMFILE — which readTokenSeries swallows into `return null`, a
+// silent undercount rather than an error. Unbounded calls let that EMFILE back
+// in through the door the pool does not cover.
+//
+// The two things between a caller and a scan are the ones quota.mjs established
+// and codex-quota.mjs adopted in #597, spelled the same way in all three:
+//
+//   _inflight     — callers that overlap wait on the one scan already running,
+//                   `force` included. What ?refresh=1 asks for is a reading
+//                   newer than the cache, and a scan in progress is one, so
+//                   joining it costs nothing and is offered before the floor.
+//   FORCE_POLL_MS — the minimum interval between two scans WE pay for.
+//                   `_inflight` deduplicates callers that overlap and nothing
+//                   else, so a caller that waits for one scan to settle and then
+//                   asks again was a fresh week of the disk every time.
+//
+// This module has neither of the extra parts its two siblings carry, and
+// deliberately: there is no upstream backend to rate-limit us, so no cooldown,
+// and nothing outside this file invalidates the cache, so no generation guard.
+let _inflight = null;
+
+// Stamped when a scan STARTS rather than when it lands: what the floor rations
+// is the walk of the disk, and one that is still running has already been paid
+// for.
+let _lastScanAt = 0;
+
+// quota.mjs's number, and codex-quota.mjs's, for the reason those two give it:
+// "The refresh button may beat that floor, but not turn into a poll loop when
+// held down." Three routes within a few lines of each other in the router have
+// no business disagreeing about what `?refresh=1` costs.
+const FORCE_POLL_MS = 60_000;
+
+/**
+ * Whether we may walk a week of the rollout tree right now.
+ *
+ * Exported for tests, for the same reason quota.mjs exports `maySelfPoll` and
+ * codex-quota.mjs exports `mayFetchQuota`: this is the rule, it is pure, and it
+ * is worth pinning down away from the scan it guards.
+ */
+export function mayScanUsage({ now, lastScanAt }) {
+  return now - lastScanAt >= FORCE_POLL_MS;
+}
+
+/**
+ * The answer to a read the floor refused.
+ *
+ * A reading, not an error. The panel draws this number from `codexUsage?.ok &&
+ * window7d.sessionCount > 0`, so an `{ ok: false }` refusal would make the token
+ * line vanish for a minute — the deck teaching itself a new failure mode in
+ * order to defend against a loop nobody ran. `stale` is the flag quota.mjs and
+ * codex-quota.mjs both use for exactly this, and `fetchedAt` keeps the moment
+ * the DATA was read rather than the moment of the read that was refused, so an
+ * age label drawn from it never vouches for a scan that did not happen.
+ */
+function heldReading(now) {
+  if (_cache) return { ..._cache, stale: true };
+  // Only reachable before the first scan has ever landed — every outcome below
+  // is cached, failures included, and a scan that is still running is served by
+  // `_inflight` — and spelled the way codex-quota.mjs spells the same state.
+  return { ok: false, reason: "waiting", fetchedAt: now };
+}
+
 const WINDOW_5H_MS  = 5 * 60 * 60 * 1000;
 const WINDOW_7D_MS  = 7 * 24 * 60 * 60 * 1000;
 
@@ -106,7 +188,16 @@ async function readTokenSeries(filePath) {
     if (pending) foldTokenLine(series, pending);
     return series.length ? series : null;
   } catch { return null; }
-  finally { fd?.close().catch(() => {}); }
+  // AWAITED. A close scheduled and abandoned means this function resolves while
+  // the descriptor is still open, and on Windows a file with any handle on it
+  // cannot be deleted: the unlink marks it delete-pending, the name stays in
+  // the directory, and the next rmdir of the parent fails with ENOTEMPTY. The
+  // deck sweeps its own rollout copies, and the suite tears down a sandbox full
+  // of them — this is how "the read is finished" stops being a lie about the
+  // handle. Awaiting in a finally costs one microtask and cannot change what is
+  // returned. Errors stay swallowed: a close that fails is nothing a reader can
+  // act on.
+  finally { await fd?.close().catch(() => {}); }
 }
 
 // Tokens spent within [windowStartMs, now]: the last cumulative snapshot minus
@@ -197,10 +288,24 @@ function emptyWindow() {
   return { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreateTokens: 0, totalTokens: 0, sessionCount: 0 };
 }
 
-export async function fetchCodexUsage({ force = false } = {}) {
+export function fetchCodexUsage({ force = false } = {}) {
   const now = Date.now();
-  if (!force && _cache && now - _cacheAt < CACHE_MS) return _cache;
+  if (!force && _cache && now - _cacheAt < CACHE_MS) return Promise.resolve(_cache);
+  // Offered before the floor: a scan that has not finished yet is a reading
+  // newer than the cache, which is what refresh asked for, and joining it costs
+  // nothing.
+  if (_inflight) return _inflight;
+  if (!mayScanUsage({ now, lastScanAt: _lastScanAt })) return Promise.resolve(heldReading(now));
+  _lastScanAt = now;
+  // A bare clear rather than quota.mjs's `_inflight === mine` check: that guard
+  // is there because invalidateQuotaCache drops the slot mid-flight, and this
+  // module has no invalidator to race with. If one is ever added, it needs the
+  // same check adding with it.
+  _inflight = scanCodexUsage(now).finally(() => { _inflight = null; });
+  return _inflight;
+}
 
+async function scanCodexUsage(now) {
   const w5h  = emptyWindow();
   const w7d  = emptyWindow();
   const start5h = now - WINDOW_5H_MS;