claude-multiacc 1.0.13 → 1.0.15

package/bin/codex

@@ -13,7 +13,10 @@ set -u
 
 # ${HOME:-} guards: with HOME stripped (env -i, some cron/systemd units) the shim
 # must still fail OPEN into plain passthrough, never abort on an unbound variable.
-ACC_ROOT="${CODEX_ACCOUNTS_DIR:-${HOME:-/nonexistent}/.codex-accounts}"
+# CODEX_ACCOUNTS_ROOT scopes the pool to one app-robot instance; CODEX_ACCOUNTS_DIR
+# is the older spelling and still works. Same precedence as lib/common.sh, so the shim
+# and codex-accounts always look at the same pool.
+ACC_ROOT="${CODEX_ACCOUNTS_ROOT:-${CODEX_ACCOUNTS_DIR:-${HOME:-/nonexistent}/.codex-accounts}}"
 MANIFEST="$ACC_ROOT/accounts.json"
 
 canon_path() {
@@ -83,8 +86,11 @@ now="$(date +%s)"
 # sed with a safe default, never a JSON parse.
 if [ -z "${CODEX_MULTIACC_THRESHOLD:-}" ]; then
   CODEX_MULTIACC_THRESHOLD="$(sed -n 's/.*"threshold"[^0-9]*\([0-9][0-9]*\).*/\1/p' "$MANIFEST" 2>/dev/null | head -1)"
-  case "$CODEX_MULTIACC_THRESHOLD" in ''|*[!0-9]*) CODEX_MULTIACC_THRESHOLD=90 ;; esac
+  CODEX_MULTIACC_THRESHOLD="$CODEX_MULTIACC_THRESHOLD"
 fi
+# Scraped or handed in by the caller, it has to be a number bash can compare without
+# complaining to stderr.
+case "$CODEX_MULTIACC_THRESHOLD" in ''|*[!0-9]*|??????*) CODEX_MULTIACC_THRESHOLD=90 ;; esac
 
 if [ "$(uname -s)" = "Darwin" ]; then
   file_mtime() { stat -f %m "$1" 2>/dev/null || echo 0; }
@@ -92,21 +98,28 @@ else
   file_mtime() { stat -c %Y "$1" 2>/dev/null || echo 0; }
 fi
 
+
+# A number this shim will do ARITHMETIC on: digits only, and short enough that bash
+# cannot go out of range. An over-range value makes `[ x -lt y ]` print
+# "integer expression expected" on stderr — which a service-spawned run must never see —
+# and makes $((x + 1)) wrap negative. Pool state is a file anyone can corrupt, so every
+# scraped number goes through here.
+num_ok() { case "$1" in ''|*[!0-9]*) return 1 ;; esac; [ "${#1}" -le 18 ]; }
+
 sel_log() {
-  printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$ACC_ROOT/selection.log" 2>/dev/null || true
+  printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" 2>/dev/null >> "$ACC_ROOT/selection.log" || true
 }
 
 marker_active() { # true if $1/.limited is still in force; clears cleanly-expired markers
   local m="$1/.limited" reset=""
   [ -f "$m" ] || return 1
   IFS= read -r reset < "$m" 2>/dev/null || reset=""
-  case "$reset" in
-    ''|*[!0-9]*)
-      # Empty/partial/garbled marker — e.g. read during a concurrent rewrite.
-      # Treat as ACTIVE and never delete: deleting here could destroy a marker
-      # another process is mid-write. The next limits refresh rewrites or clears it.
-      return 0 ;;
-  esac
+  if ! num_ok "$reset"; then
+    # Empty/partial/garbled/absurd marker — e.g. read during a concurrent rewrite.
+    # Treat as ACTIVE and never delete: deleting here could destroy a marker
+    # another process is mid-write. The next limits refresh rewrites or clears it.
+    return 0
+  fi
   if [ "$now" -ge "$reset" ]; then
     rm -f "$m" 2>/dev/null
     return 1
@@ -120,10 +133,10 @@ fresh_field() { # fresh_field <acct dir> <json key> -> integer if telemetry fres
   local f="$1/limits.json" fetched v
   [ -f "$f" ] || return 1
   fetched="$(sed -n 's/.*"fetched_at"[^0-9]*\([0-9][0-9]*\).*/\1/p' "$f" 2>/dev/null | head -1)"
-  case "$fetched" in ''|*[!0-9]*) return 1 ;; esac
+  num_ok "$fetched" || return 1
   [ $((now - fetched)) -le "$STALE_AFTER" ] || return 1
   v="$(sed -n "s/.*\"$2\"[^0-9]*\([0-9][0-9]*\).*/\1/p" "$f" 2>/dev/null | head -1)"
-  case "$v" in ''|*[!0-9]*) return 1 ;; esac
+  num_ok "$v" || return 1
   printf '%s\n' "$v"
 }
 
@@ -148,6 +161,151 @@ over_threshold() { # $1 = acct dir
   [ "$v" -ge "${CODEX_MULTIACC_THRESHOLD:-90}" ]
 }
 
+# ---- client-reported rate limits ---------------------------------------------
+# Same idea as the claude shim: do not depend on the usage endpoint to notice that an
+# account ran dry. The codex CLI writes every `token_count` event into the run's rollout
+# with the windows the server just reported:
+#   "rate_limits":{"primary":{"used_percent":97.4,"window_minutes":10080,"resets_at":<epoch>},
+#                  "secondary":{...}}
+# Rollouts live under $CODEX_HOME/sessions/<Y>/<M>/<D>/. When that tree really belongs to
+# the account they need no session->account index — but the installed layout SYMLINKS it
+# to a shared ~/.codex/sessions so `codex resume` finds every session, and there it proves
+# nothing about who spent the quota. sessions_owned() below is what keeps one account's
+# spend from marking the whole pool; on the shared layout this scan simply stays off and
+# the usage endpoint remains the only limit signal for codex.
+# Usage only grows inside a window, so a report from earlier in the same window is still a
+# valid lower bound — which is why an in-force `resets_at` is the only freshness test.
+# ...and only when that tree is private to the account: a shared one says nothing
+# about who spent the quota.
+sessions_owned() { # $1 acct dir
+  # Structural and deliberately FORK-FREE: this runs for every account on every single
+  # invocation, and a pair of canon_path calls here cost more than the whole scan.
+  # A session tree is this account's own evidence only when neither the account dir nor
+  # its sessions dir is a symlink — which is exactly how the installed layout shares them
+  # (lib/common.sh seeds codex accounts with sessions -> ~/.codex/sessions, and an account
+  # dir may itself be a symlink to ~/.codex). Anything shared fails OPEN: no ownership,
+  # no exclusion, and the usage endpoint stays the only limit signal for that account.
+  [ -d "$1/sessions" ] || return 1
+  [ -L "$1/sessions" ] && return 1
+  [ -L "$1" ] && return 1
+  return 0
+}
+
+QUOTA_SCAN_BYTES=262144    # rollout tail read per session (the newest report is last)
+QUOTA_SCAN_MAX_AGE=21600   # 6h of rollout mtime — older runs are re-reported by newer ones
+QUOTA_SCAN_MAX_FILES=3     # hard cap per account: a limit still in force rejects the
+                           # newest sessions too, so older ones can only repeat the news
+
+rl_field() { # rl_field <json fragment> <key> -> leading integer of that key's value
+  printf '%s' "$1" | LC_ALL=C sed -n "s/.*\"$2\"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p" | head -1
+}
+
+# Newest still-in-force over-threshold window this account's own runs recorded.
+# Prints "<reset-epoch> <window>"; fails when there is none.
+client_limit_scan() { # $1 acct dir
+  local day f line frag pct reset best=0 bestwin="" scanned=0 thr="${CODEX_MULTIACC_THRESHOLD:-90}" which
+  local memo="$1/.client-scan" last="" ttl
+  [ "${CODEX_MULTIACC_CLIENT_LIMITS:-1}" = "0" ] && return 1
+  sessions_owned "$1" || return 1
+  # A CLEAN result is remembered for a few seconds so a tight loop of `codex exec` runs
+  # does not re-read the same rollout tails every time. Only the clean answer is
+  # memoized — a spent window becomes a .limited marker, and marker_active short-circuits
+  # this scan from then on.
+  if [ -f "$memo" ]; then
+    IFS= read -r last < "$memo" 2>/dev/null || last=""
+    num_ok "$last" || last=0
+    # The TTL is caller-supplied, so it goes through num_ok too: `[ x -lt bogus ]` would
+    # print "integer expression expected" on the caller's stderr before exec.
+    ttl="${CODEX_MULTIACC_CLIENT_SCAN_TTL:-20}"
+    num_ok "$ttl" || ttl=20
+    [ $((now - last)) -lt "$ttl" ] && return 1
+  fi
+  # NEWEST FIRST, always. Both day dirs and rollout filenames start with an ISO timestamp,
+  # so lexicographic order IS chronological order — walking it forwards would spend the
+  # whole file budget on the oldest runs and never reach the one that actually hit the
+  # wall (codex-review finding: 13 rollouts, only the newest over threshold). The first
+  # file that reports a spent window wins: usage only grows inside a window, so nothing
+  # older can be more current.
+  #
+  # find(1), not a glob, for two reasons a second review pass turned up:
+  #   * with -P (the default) find never descends a SYMLINKED component, so a nested
+  #     sessions/<year> -> /somewhere/shared cannot smuggle another pool's rollouts in
+  #     under an account whose own sessions/ dir is real;
+  #   * `tail -n` bounds the list inside the pipe, so a tree with a hundred thousand stale
+  #     rollouts never becomes a hundred-thousand-element shell array before the cap.
+  local days=() files=() di fx
+  while IFS= read -r day; do
+    # find(1) output is newline-delimited, so a pool file whose NAME contains a newline
+    # arrives as two lines and its tail would resolve relative to $PWD — outside the pool
+    # entirely. Every path is therefore re-checked against the tree it must have come from.
+    case "$day" in "$1"/sessions/?*) ;; *) continue ;; esac
+    days+=("$day")
+  done <<EOF
+$(find "$1/sessions" -mindepth 3 -maxdepth 3 -type d 2>/dev/null | LC_ALL=C sort | tail -3)
+EOF
+  di=$(( ${#days[@]} - 1 ))
+  while [ "$di" -ge 0 ] && [ "$scanned" -lt "$QUOTA_SCAN_MAX_FILES" ]; do
+    day="${days[$di]}"
+    di=$((di - 1))
+    files=()
+    while IFS= read -r f; do
+      case "$f" in "$day"/rollout-?*) ;; *) continue ;; esac
+      files+=("$f")
+    done <<EOF
+$(find "$day" -maxdepth 1 -type f -name 'rollout-*.jsonl' 2>/dev/null | LC_ALL=C sort | tail -n "$QUOTA_SCAN_MAX_FILES")
+EOF
+    fx=$(( ${#files[@]} - 1 ))
+    # A file that fails the staleness test still costs a stat, so it spends budget too.
+    while [ "$fx" -ge 0 ] && [ "$scanned" -lt "$QUOTA_SCAN_MAX_FILES" ]; do
+      f="${files[$fx]}"
+      fx=$((fx - 1))
+      scanned=$((scanned + 1))
+      [ -L "$f" ] && continue
+      [ $((now - $(file_mtime "$f"))) -le "$QUOTA_SCAN_MAX_AGE" ] || continue
+      line="$(tail -c "$QUOTA_SCAN_BYTES" "$f" 2>/dev/null \
+        | LC_ALL=C grep -a '^{".*"rate_limits"' \
+        | tail -1)"
+      [ -n "$line" ] || continue
+      for which in primary secondary; do
+        frag="$(printf '%s' "$line" | LC_ALL=C sed -n "s/.*\"$which\"[[:space:]]*:[[:space:]]*{\([^}]*\)}.*/\1/p")"
+        [ -n "$frag" ] || continue
+        pct="$(rl_field "$frag" used_percent)"
+        reset="$(rl_field "$frag" resets_at)"
+        num_ok "$pct" || continue
+        num_ok "$reset" || continue
+        [ "$pct" -ge "$thr" ] || continue
+        if [ "$reset" -gt "$best" ]; then best="$reset"; bestwin="$which:$pct"; fi
+      done
+      [ "$best" -gt "$now" ] && break 2
+    done
+  done
+  if [ "$best" -le "$now" ]; then
+    printf '%s\n' "$now" 2>/dev/null > "$memo.$$" \
+      && mv -f "$memo.$$" "$memo" 2>/dev/null || rm -f "$memo.$$" 2>/dev/null
+    return 1
+  fi
+  printf '%s %s\n' "$best" "$bestwin"
+}
+
+mark_client_limit() { # $1 acct dir, $2 reset epoch, $3 window:pct
+  local m="$1/.limited" cur=""
+  # Never shorten a marker that already reaches further out, and never rewrite the
+  # same one on every invocation.
+  if [ -f "$m" ]; then
+    IFS= read -r cur < "$m" 2>/dev/null || cur=""
+    num_ok "$cur" || cur=0
+    [ "$cur" -ge "$2" ] && return 0
+  fi
+  {
+    echo "$2"
+    echo "bucket=client:${3%%:*} percent=${3##*:} marked_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) reason=client-rate-limit"
+  } 2>/dev/null > "$1/.limited.$$" \
+    && mv -f "$1/.limited.$$" "$m" 2>/dev/null \
+    || rm -f "$1/.limited.$$" 2>/dev/null || true
+  sel_log "$(basename "$1") LIMITED by its own run ($3, resets $2) — client-reported"
+  return 0
+}
+
 # Auth is a ChatGPT login: auth.json carrying a non-empty access token. An API-key-only
 # auth.json is NOT auth here (subscription-only by design), and an empty file is NOT
 # auth (an interrupted write must not make a dead account selectable).
@@ -225,6 +383,16 @@ for d in "$ACC_ROOT"/acct-*; do
   fi
   valid+=("$d")
   marker_active "$d" && continue
+  # The account's own records are consulted BEFORE telemetry: what the server told a real
+  # call is first-hand and carries the real reset, while limits.json can be days stale —
+  # the usage endpoint rate-limits its own callers. Marking here (rather than lazily, on
+  # whichever account happens to be picked) is what makes the marker visible to
+  # `codex-accounts status`, to a concurrent run in another terminal, and to sync.
+  # The cost is bounded by the scan's own file budget and its clean-result memo.
+  if lim="$(client_limit_scan "$d")"; then
+    mark_client_limit "$d" "${lim%% *}" "${lim##* }"
+    continue
+  fi
   over_threshold "$d" && continue
   eligible+=("$d")
 done
@@ -240,7 +408,7 @@ if [ "${#expired[@]}" -gt 0 ]; then
     last=0
     [ -f "$n" ] && last="$(file_mtime "$n")"
     if [ $((now - last)) -gt 3600 ]; then
-      : > "$n" 2>/dev/null || true
+      : 2>/dev/null > "$n" || true
       printf 'codex-multiacc: %s account(s) unusable (%s) — see: codex-accounts expired\n' \
         "${#expired[@]}" "${ids# }" >&2
     fi
@@ -260,6 +428,30 @@ if [ "${#valid[@]}" -eq 0 ]; then
   exec "$REAL" "$@"
 fi
 
+# ---- rotation ----------------------------------------------------------------
+# Deliberately NOT a full least-recently-used order: just "do not hand back the account
+# you were on a moment ago". That is the whole of the bug (quit a session that ran into
+# its limit, start another, land straight back on it), and it is the only part that can
+# be done without serialising selection. Among equally-ranked candidates the most recent
+# pick is dropped and the REST ARE SAMPLED RANDOMLY — so a burst of parallel `codex exec` runs
+# still spreads across the pool instead of every one of them computing the same "oldest"
+# account and piling onto it.
+# The state is one id in one file. A pool root that cannot be written just leaves a stale
+# id there, which costs one avoided account and nothing else — it can never starve one.
+last_pick_id() { # -> id this pool last handed out, or empty
+  local v=""
+  [ -f "$ACC_ROOT/.last-pick" ] && { IFS= read -r v < "$ACC_ROOT/.last-pick" 2>/dev/null || v=""; }
+  case "$v" in acct-[0-9][0-9]) printf '%s\n' "$v" ;; esac
+}
+
+remember_pick() { # $1 acct dir — best effort. stderr is silenced BEFORE the redirect, or
+                  # a read-only pool root prints "Permission denied" on every single run.
+  local f="$ACC_ROOT/.last-pick" id="${1##*/}"
+  printf '%s\n' "$id" 2>/dev/null > "$f.$$" \
+    && mv -f "$f.$$" "$f" 2>/dev/null || rm -f "$f.$$" 2>/dev/null
+  return 0
+}
+
 # Pick the account with the MOST headroom (lowest ranking score = most weekly headroom,
 # session as tiebreaker). Ties break randomly so equally-idle accounts still spread load.
 # Sets PICK_DIR/PICK_SCORE as globals — it must never touch "$@", which holds the
@@ -267,16 +459,41 @@ fi
 PICK_DIR=""
 PICK_SCORE=""
 pick_best() { # args: candidate dirs
-  local d v best="" bestv=1000000 ties=1
+  local d avoid best="" bestv=1000000 ties=0 i n
+  local cand=() score=()
+  avoid="$(last_pick_id)"
   for d in "$@"; do
-    v="$(sel_score_of "$d")"
-    if [ "$v" -lt "$bestv" ]; then
-      bestv="$v"; best="$d"; ties=1
-    elif [ "$v" -eq "$bestv" ]; then
+    cand+=("$d")
+    score+=("$(sel_score_of "$d")")
+  done
+  n=${#cand[@]}
+  i=0
+  while [ "$i" -lt "$n" ]; do
+    [ "${score[$i]}" -lt "$bestv" ] && bestv="${score[$i]}"
+    i=$((i + 1))
+  done
+  # Reservoir-sample among the equally-best, skipping the account just handed out.
+  i=0
+  while [ "$i" -lt "$n" ]; do
+    if [ "${score[$i]}" -eq "$bestv" ] && [ "${cand[$i]##*/}" != "$avoid" ]; then
       ties=$((ties + 1))
-      [ $((RANDOM % ties)) -eq 0 ] && best="$d"   # reservoir-sample among equals
+      [ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
     fi
+    i=$((i + 1))
   done
+  if [ -z "$best" ]; then
+    # The only account at the best score IS the one just used — degraded rotation beats
+    # refusing to pick (and in a two-account pool this is the other half of the
+    # alternation).
+    i=0
+    while [ "$i" -lt "$n" ]; do
+      if [ "${score[$i]}" -eq "$bestv" ]; then
+        ties=$((ties + 1))
+        [ $((RANDOM % ties)) -eq 0 ] && best="${cand[$i]}"
+      fi
+      i=$((i + 1))
+    done
+  fi
   PICK_DIR="$best"
   PICK_SCORE="$bestv"
 }
@@ -293,19 +510,27 @@ else
   sel_log "all-limited fallback=$(basename "$PICK_DIR") weekly=$(fresh_field "$PICK_DIR" weekly_percent || echo '?')%"
 fi
 pick="$PICK_DIR"
-
-# Opportunistic limits refresh: non-blocking, throttled, backgrounded.
+# Remember the pick so the NEXT run does not hand back the same account. An explicit
+# CODEX_ACCOUNT pin deliberately does not: a pin is a caller overriding selection,
+# not a turn in the rotation.
+remember_pick "$pick"
+
+# Opportunistic limits refresh: non-blocking, throttled, backgrounded. The windows are
+# deliberately wide (10m, matching the 5m scheduled pass): the usage endpoint rate-limits
+# its OWN callers, and a fleet of machines polling one account too eagerly earns a 429
+# with Retry-After 3600 — telemetry then goes stale for an hour at a time, which is
+# exactly how every account ends up scoring NEUTRAL.
 kick="$ACC_ROOT/.limits-kick"
 stale=0
 for d in "${valid[@]}"; do
   f="$d/limits.json"
-  if [ ! -f "$f" ] || [ $((now - $(file_mtime "$f"))) -gt 180 ]; then stale=1; break; fi
+  if [ ! -f "$f" ] || [ $((now - $(file_mtime "$f"))) -gt 600 ]; then stale=1; break; fi
 done
 if [ "$stale" = 1 ] && [ -x "$SELF_DIR/codex-accounts" ]; then
   last=0
   [ -f "$kick" ] && last="$(file_mtime "$kick")"
-  if [ $((now - last)) -gt 120 ]; then
-    : > "$kick" 2>/dev/null || true
+  if [ $((now - last)) -gt 600 ]; then
+    : 2>/dev/null > "$kick" || true
     ( "$SELF_DIR/codex-accounts" limits --quiet >/dev/null 2>&1 & ) >/dev/null 2>&1
   fi
 fi
@@ -417,7 +642,7 @@ while :; do
       {
         echo "$now"
         echo "reason=$park_reason soft_until=$park_soft marked_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) detail=$park_detail"
-      } > "$cur/.expired.$$" 2>/dev/null \
+      } 2>/dev/null > "$cur/.expired.$$" \
         && mv -f "$cur/.expired.$$" "$cur/.expired" 2>/dev/null \
         || rm -f "$cur/.expired.$$" 2>/dev/null || true
       sel_log "$(basename "$cur") parked ($park_reason until $park_soft) — see: codex-accounts expired"
@@ -425,7 +650,7 @@ while :; do
       {
         echo $((now + 600))
         echo "bucket=error-cooldown percent=? marked_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) reason=error-cooldown"
-      } > "$cur/.limited.$$" 2>/dev/null \
+      } 2>/dev/null > "$cur/.limited.$$" \
         && mv -f "$cur/.limited.$$" "$cur/.limited" 2>/dev/null \
         || rm -f "$cur/.limited.$$" 2>/dev/null || true
     fi
@@ -441,6 +666,9 @@ while :; do
     if [ -n "$next" ]; then
       sel_log "retry from=$(basename "$cur") to=$(basename "$next") rc=$rc"
       cur="$next"
+      # The account that actually serves the work is the one the next run should rotate
+      # away from — not the one that bounced.
+      remember_pick "$cur"
       attempt=2
       continue
     fi