codex-multi-auth 2.8.4 → 2.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/scripts/codex.js CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
 
3
- import { spawn } from "node:child_process";
3
+ import { execFile, execFileSync, spawn } from "node:child_process";
4
4
  import { createHash, randomBytes } from "node:crypto";
5
5
  import {
6
6
  chmodSync,
@@ -85,6 +85,29 @@ const APP_RUNTIME_HELPER_STATUS_FILE =
85
85
  const APP_RUNTIME_HELPER_OWNER_FILE =
86
86
  RUNTIME_CONSTANTS.APP_RUNTIME_HELPER_OWNER_FILE;
87
87
  const DEFAULT_APP_RUNTIME_HELPER_IDLE_MS = 12 * 60 * 60 * 1000;
88
+ // Absolute ceiling on a helper's life, independent of the idle tracker. The
89
+ // idle reaper depends on activity accounting being correct; any bug there —
90
+ // PID reuse briefly reviving a dead owner is the observed one — previously
91
+ // produced an *unbounded* leak because nothing else bounded the process.
92
+ const DEFAULT_APP_RUNTIME_HELPER_MAX_LIFETIME_MS = 24 * 60 * 60 * 1000;
93
+ // A helper whose launcher is gone is not idle in the same sense as one whose
94
+ // launcher is sitting at a prompt: nobody is coming back to it unless a
95
+ // detached consumer picked it up. The detach grace below hands helpers off
96
+ // optimistically — every launcher that exits cleanly within the window leaves
97
+ // its helper running — so short forwarded commands strand helpers that then
98
+ // hold the full idle timeout with no owner and no traffic. This window is the
99
+ // idle timeout that applies from the moment the owner is confirmed dead, and
100
+ // it only ever fires with zero open client connections, so a consumer that
101
+ // really did take the handoff is never reaped out from under.
102
+ const DEFAULT_APP_RUNTIME_HELPER_DETACHED_IDLE_MS = 15 * 60 * 1000;
103
+ const APP_RUNTIME_HELPER_OWNER_START_TIME_ENV =
104
+ "CODEX_MULTI_AUTH_APP_ROTATION_OWNER_START_TIME_MS";
105
+ // Re-verify owner identity (not just PID liveness) at most this often; a
106
+ // process spawn per tick would cost more than the leak it prevents.
107
+ const APP_RUNTIME_HELPER_OWNER_IDENTITY_RECHECK_MS = 60_000;
108
+ // Status telemetry heartbeat: the tick's job is the timeout check, so status
109
+ // is republished only on change, plus a heartbeat so freshness readers work.
110
+ const APP_RUNTIME_HELPER_STATUS_HEARTBEAT_MS = 60_000;
88
111
  const DEFAULT_APP_RUNTIME_HELPER_DETACH_GRACE_MS = 5_000;
89
112
  const APP_RUNTIME_HELPER_LAUNCH_TIMEOUT_MS = 15_000;
90
113
  const APP_SERVER_SHIM_DIR_NAME = "app-server-shims";
@@ -94,37 +117,49 @@ const DEFAULT_STATUS_QUOTA_REFRESH_INTERVAL_MS = 10 * 60 * 1000;
94
117
  const STATUS_QUOTA_REFRESH_LOCK_STALE_MS = 10 * 60 * 1000;
95
118
  const STATUS_QUOTA_REFRESH_LOCK_DIR = "status-quota-refresh.lock";
96
119
  const STARTUP_UPDATE_NOTICE_TIMED_OUT = Symbol("startup-update-notice-timed-out");
97
- let shadowHomeCleanupBusyFailuresRemaining = Number.parseInt(
98
- process.env.CODEX_MULTI_AUTH_TEST_SHADOW_CLEANUP_BUSY_FAILURES ?? "0",
99
- 10,
100
- );
101
- let shadowHomeCleanupPreflightReadBusyFailuresRemaining = Number.parseInt(
102
- process.env.CODEX_MULTI_AUTH_TEST_SHADOW_PREFLIGHT_READ_BUSY_FAILURES ?? "0",
103
- 10,
104
- );
105
- let shadowHomeSyncLockRecreateStaleCount = Number.parseInt(
106
- process.env.CODEX_MULTI_AUTH_TEST_SHADOW_LOCK_RECREATE_STALE_COUNT ?? "0",
107
- 10,
120
+
121
+ // This wrapper is published (`package.json` ships `scripts/codex.js`), so every
122
+ // fault injector below runs in users' installs. Two guards keep them inert
123
+ // there. The first is an explicit opt-in: a single, greppable switch that must
124
+ // be set alongside any counter, so one stray counter in a shell profile or a CI
125
+ // environment cannot arm anything. The second is a strict parse — `parseInt`
126
+ // happily reads "2abc" as 2 and "1e3" as 1, which is how a value that was never
127
+ // meant to be a count arms an injector — so only a plain run of digits counts
128
+ // and everything else is zero. Zero means "never inject".
129
+ const TEST_FAULT_INJECTION_ENV = "CODEX_MULTI_AUTH_TEST_FAULT_INJECTION";
130
+
131
+ function resolveTestFaultInjectionCount(name, env = process.env) {
132
+ if (env[TEST_FAULT_INJECTION_ENV] !== "1") return 0;
133
+ const raw = (env[name] ?? "").trim();
134
+ if (!/^\d+$/.test(raw)) return 0;
135
+ const parsed = Number.parseInt(raw, 10);
136
+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : 0;
137
+ }
138
+
139
+ let shadowHomeCleanupBusyFailuresRemaining = resolveTestFaultInjectionCount(
140
+ "CODEX_MULTI_AUTH_TEST_SHADOW_CLEANUP_BUSY_FAILURES",
108
141
  );
109
- let shadowHomeSyncMetadataBusyFailuresRemaining = Number.parseInt(
110
- process.env.CODEX_MULTI_AUTH_TEST_SHADOW_SYNC_METADATA_BUSY_FAILURES ?? "0",
111
- 10,
142
+ let shadowHomeCleanupPreflightReadBusyFailuresRemaining =
143
+ resolveTestFaultInjectionCount(
144
+ "CODEX_MULTI_AUTH_TEST_SHADOW_PREFLIGHT_READ_BUSY_FAILURES",
145
+ );
146
+ let shadowHomeSyncLockRecreateStaleCount = resolveTestFaultInjectionCount(
147
+ "CODEX_MULTI_AUTH_TEST_SHADOW_LOCK_RECREATE_STALE_COUNT",
112
148
  );
113
- let shadowHomeSyncLockOwnerWriteFailuresRemaining = Number.parseInt(
114
- process.env.CODEX_MULTI_AUTH_TEST_SHADOW_LOCK_OWNER_WRITE_FAILURES ?? "0",
115
- 10,
149
+ let shadowHomeSyncMetadataBusyFailuresRemaining = resolveTestFaultInjectionCount(
150
+ "CODEX_MULTI_AUTH_TEST_SHADOW_SYNC_METADATA_BUSY_FAILURES",
116
151
  );
152
+ let shadowHomeSyncLockOwnerWriteFailuresRemaining =
153
+ resolveTestFaultInjectionCount(
154
+ "CODEX_MULTI_AUTH_TEST_SHADOW_LOCK_OWNER_WRITE_FAILURES",
155
+ );
117
156
  let appServerShimFileCleanupBusyFailuresRemaining =
118
- Number.parseInt(
119
- process.env.CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_FILE_CLEANUP_BUSY_FAILURES ??
120
- "0",
121
- 10,
122
- ) || 0;
123
- let appServerShimCopyBusyFailuresRemaining =
124
- Number.parseInt(
125
- process.env.CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_COPY_BUSY_FAILURES ?? "0",
126
- 10,
127
- ) || 0;
157
+ resolveTestFaultInjectionCount(
158
+ "CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_FILE_CLEANUP_BUSY_FAILURES",
159
+ );
160
+ let appServerShimCopyBusyFailuresRemaining = resolveTestFaultInjectionCount(
161
+ "CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_COPY_BUSY_FAILURES",
162
+ );
128
163
  const shadowHomeCleanupRetryMarkerDir =
129
164
  (process.env.CODEX_MULTI_AUTH_TEST_SHADOW_RETRY_MARKER_DIR ?? "").trim();
130
165
  let warnedInvalidRuntimeRotationProxyEnv = false;
@@ -3776,10 +3811,21 @@ function installRuntimeRotationAppServerCliShim(forwardedEnv, configArgs = []) {
3776
3811
  return shimDir;
3777
3812
  }
3778
3813
 
3779
- function resolveRuntimeRotationAppHelperStatusPath(env = process.env) {
3814
+ // With a helper PID the path is per-helper, mirroring the owner files below —
3815
+ // N concurrent helpers each publish their own status instead of last-writer-
3816
+ // winning one shared file. Without a PID it is the legacy shared path, kept
3817
+ // only so readers can still see a helper from before this change.
3818
+ function resolveRuntimeRotationAppHelperStatusPath(env = process.env, helperPid) {
3780
3819
  const multiAuthDir =
3781
3820
  resolveOriginalMultiAuthDir(env) ?? join(resolveCodexHomeDir(env), "multi-auth");
3782
- return join(multiAuthDir, APP_RUNTIME_HELPER_STATUS_FILE);
3821
+ const statusFileName =
3822
+ typeof helperPid === "number" && Number.isInteger(helperPid) && helperPid > 0
3823
+ ? APP_RUNTIME_HELPER_STATUS_FILE.replace(
3824
+ /\.json$/i,
3825
+ `.${helperPid}.json`,
3826
+ )
3827
+ : APP_RUNTIME_HELPER_STATUS_FILE;
3828
+ return join(multiAuthDir, statusFileName);
3783
3829
  }
3784
3830
 
3785
3831
  function resolveRuntimeRotationAppHelperOwnerPath(env = process.env, helperPid) {
@@ -3852,6 +3898,96 @@ function resolveRuntimeRotationAppHelperOwnerPid(env = process.env) {
3852
3898
  return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
3853
3899
  }
3854
3900
 
3901
+ function resolveRuntimeRotationAppHelperMaxLifetimeMs(env = process.env) {
3902
+ const parsed = Number.parseInt(
3903
+ env.CODEX_MULTI_AUTH_APP_ROTATION_MAX_LIFETIME_MS ?? "",
3904
+ 10,
3905
+ );
3906
+ // 0 disables the ceiling explicitly; anything unset or invalid gets the
3907
+ // default rather than unbounded life.
3908
+ return Number.isFinite(parsed) && parsed >= 0
3909
+ ? parsed
3910
+ : DEFAULT_APP_RUNTIME_HELPER_MAX_LIFETIME_MS;
3911
+ }
3912
+
3913
+ function resolveRuntimeRotationAppHelperOwnerStartTimeMs(env = process.env) {
3914
+ const parsed = Number.parseInt(
3915
+ env[APP_RUNTIME_HELPER_OWNER_START_TIME_ENV] ?? "",
3916
+ 10,
3917
+ );
3918
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
3919
+ }
3920
+
3921
+ // `lstart` is strftime-formatted and locale-sensitive; Date.parse on a
3922
+ // localized string is implementation-defined and can yield NaN, which would
3923
+ // silently disable the identity check. Both readers pin the C locale so both
3924
+ // sides of every comparison parse the same shape.
3925
+ function parseProcessStartTimeOutput(out) {
3926
+ const trimmed = (out ?? "").trim();
3927
+ if (!trimmed) return null;
3928
+ const parsed = Date.parse(trimmed);
3929
+ return Number.isFinite(parsed) ? parsed : null;
3930
+ }
3931
+
3932
+ // The kernel's start time for a PID, in epoch ms — the identity that survives
3933
+ // PID reuse. Null on platforms without `ps` (Windows) or for a PID that is
3934
+ // already gone; callers must treat null as "identity unknown" and fall back
3935
+ // to bare liveness rather than declaring the process dead. Synchronous —
3936
+ // launcher/sweep use only; the helper's tick uses the async variant below.
3937
+ //
3938
+ // Windows short-circuits rather than spawning: there is no `ps` there, so the
3939
+ // probe could only ever fail, and it is not called once — the launcher probes
3940
+ // itself on every launch and the sweep probes up to `probeBudget` candidates.
3941
+ // Paying a process spawn per probe to learn nothing is the whole cost. Windows
3942
+ // therefore runs on bare liveness, and the 24h lifetime ceiling is what bounds
3943
+ // a leak there.
3944
+ function readProcessStartTimeMs(pid) {
3945
+ if (process.platform === "win32") return null;
3946
+ try {
3947
+ return parseProcessStartTimeOutput(
3948
+ execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], {
3949
+ encoding: "utf8",
3950
+ stdio: ["ignore", "pipe", "ignore"],
3951
+ env: { ...process.env, LC_ALL: "C" },
3952
+ timeout: 2_000,
3953
+ }),
3954
+ );
3955
+ } catch {
3956
+ return null;
3957
+ }
3958
+ }
3959
+
3960
+ // Async variant for the helper's status tick, which runs on the live rotation
3961
+ // proxy's event loop: a wedged `ps` must stall a background probe, never an
3962
+ // in-flight Responses stream. Same parse, same C locale, same null contract,
3963
+ // and the same Windows short-circuit.
3964
+ function readProcessStartTimeMsAsync(pid, onResult) {
3965
+ if (process.platform === "win32") {
3966
+ onResult(null);
3967
+ return;
3968
+ }
3969
+ let child;
3970
+ try {
3971
+ child = execFile(
3972
+ "ps",
3973
+ ["-o", "lstart=", "-p", String(pid)],
3974
+ {
3975
+ encoding: "utf8",
3976
+ env: { ...process.env, LC_ALL: "C" },
3977
+ timeout: 2_000,
3978
+ },
3979
+ (error, stdout) => {
3980
+ onResult(error ? null : parseProcessStartTimeOutput(stdout));
3981
+ },
3982
+ );
3983
+ } catch {
3984
+ onResult(null);
3985
+ return;
3986
+ }
3987
+ // The probe must not keep the helper's event loop referenced on shutdown.
3988
+ child.unref?.();
3989
+ }
3990
+
3855
3991
  function isProcessAlive(pid) {
3856
3992
  try {
3857
3993
  process.kill(pid, 0);
@@ -3861,8 +3997,109 @@ function isProcessAlive(pid) {
3861
3997
  }
3862
3998
  }
3863
3999
 
3864
- function isRuntimeRotationAppHelperOwnerAlive(pid) {
3865
- return isProcessAlive(pid);
4000
+ // `kill(pid, 0)` answers "does *a* process hold this integer", never "is this
4001
+ // still my owner". PIDs recycle, and a helper that mistakes a recycled PID for
4002
+ // its owner pushes its idle deadline forward — a ratchet, because one false
4003
+ // "alive" is never corrected by later true "dead"s, which is how helpers were
4004
+ // observed running 33 hours past a 12-hour timeout. Identity is the owner's
4005
+ // process start time, captured by the launcher at spawn: a recycled PID
4006
+ // necessarily has a later start time, so the match fails and the helper
4007
+ // correctly sees a dead owner. When the start time is unknown (no `ps`, or a
4008
+ // pre-upgrade launcher), behavior degrades to the bare liveness check.
4009
+ //
4010
+ // The verdict is deliberately three-valued. "No owner PID was recorded" and
4011
+ // "the owner is confirmed dead" are different facts, and collapsing them into
4012
+ // one `false` makes a helper launched without an owner — invoked directly, or
4013
+ // spawned by a pre-upgrade launcher that sets no owner PID — start the
4014
+ // detached clock on its very first tick and reap itself while it is still
4015
+ // serving. `unknown` means neither branch fires, which is exactly what the
4016
+ // pre-#664 `if (ownerPid && isAlive(ownerPid))` guard did.
4017
+ const OWNER_ALIVE = "alive";
4018
+ const OWNER_DEAD = "dead";
4019
+ const OWNER_UNKNOWN = "unknown";
4020
+
4021
+ function createRuntimeRotationAppHelperOwnerLivenessCheck(
4022
+ ownerPid,
4023
+ expectedStartTimeMs,
4024
+ recheckIntervalMs = APP_RUNTIME_HELPER_OWNER_IDENTITY_RECHECK_MS,
4025
+ ) {
4026
+ let lastIdentityCheckedAt = 0;
4027
+ let lastIdentityVerdict = true;
4028
+ let probeInFlight = false;
4029
+ return (currentTime) => {
4030
+ if (!ownerPid) {
4031
+ return OWNER_UNKNOWN;
4032
+ }
4033
+ if (!isProcessAlive(ownerPid)) {
4034
+ return OWNER_DEAD;
4035
+ }
4036
+ if (expectedStartTimeMs === null) {
4037
+ return OWNER_ALIVE;
4038
+ }
4039
+ // The probe is asynchronous and single-flight: the tick runs on the live
4040
+ // proxy's event loop, so it always answers from the last verdict and the
4041
+ // probe updates it in the background — a stale verdict is tolerated by
4042
+ // design (one recheck window against a 12h timeout), and single-flight
4043
+ // means a wedged `ps` holds one child, not one per tick.
4044
+ if (
4045
+ !probeInFlight &&
4046
+ currentTime - lastIdentityCheckedAt >= recheckIntervalMs
4047
+ ) {
4048
+ lastIdentityCheckedAt = currentTime;
4049
+ probeInFlight = true;
4050
+ readProcessStartTimeMsAsync(ownerPid, (actualStartTimeMs) => {
4051
+ probeInFlight = false;
4052
+ // A failed read is "identity unknown", not "owner dead": under the
4053
+ // process-table pressure this fix exists for, fork itself can fail,
4054
+ // and declaring a live owner dead would kill the proxy out from
4055
+ // under an active session. Keep the verdict and retry next window.
4056
+ if (actualStartTimeMs !== null) {
4057
+ lastIdentityVerdict = actualStartTimeMs === expectedStartTimeMs;
4058
+ }
4059
+ });
4060
+ }
4061
+ return lastIdentityVerdict ? OWNER_ALIVE : OWNER_DEAD;
4062
+ };
4063
+ }
4064
+
4065
+ function resolveRuntimeRotationAppHelperTickMs(idleTimeoutMs, detachedIdleMs) {
4066
+ const shortestWindowMs =
4067
+ detachedIdleMs > 0 ? Math.min(idleTimeoutMs, detachedIdleMs) : idleTimeoutMs;
4068
+ return Math.min(1_000, Math.max(50, Math.floor(shortestWindowMs / 2)));
4069
+ }
4070
+
4071
+ // `lastIdentityVerdict` starts optimistic, so the first tick reports the owner
4072
+ // alive while the async `ps` probe is still in flight, and a helper adopted by
4073
+ // a recycled owner PID keeps refreshing its activity clock until the first
4074
+ // recheck lands. Against the 12h default that is deliberate and harmless. It
4075
+ // stops being harmless the moment the windows are compressed: a 60s recheck
4076
+ // pinned against a 250ms idle override — how the lifecycle tests run — is
4077
+ // longer than the entire window under test, so whether the verdict ever flips
4078
+ // comes down to probe timing. Scaling the recheck to the shortest window that
4079
+ // can fire makes the flip a property of the code rather than a race, and
4080
+ // production is untouched because both defaults are hours.
4081
+ function resolveRuntimeRotationAppHelperOwnerRecheckMs(
4082
+ idleTimeoutMs,
4083
+ detachedIdleMs,
4084
+ ) {
4085
+ const shortestWindowMs =
4086
+ detachedIdleMs > 0 ? Math.min(idleTimeoutMs, detachedIdleMs) : idleTimeoutMs;
4087
+ return Math.min(
4088
+ APP_RUNTIME_HELPER_OWNER_IDENTITY_RECHECK_MS,
4089
+ Math.max(50, Math.floor(shortestWindowMs / 4)),
4090
+ );
4091
+ }
4092
+
4093
+ function resolveRuntimeRotationAppHelperDetachedIdleMs(env = process.env) {
4094
+ const parsed = Number.parseInt(
4095
+ env.CODEX_MULTI_AUTH_APP_ROTATION_DETACHED_IDLE_MS ?? "",
4096
+ 10,
4097
+ );
4098
+ // 0 disables the detached window explicitly, leaving a stranded helper on
4099
+ // the full idle timeout — the pre-fix behavior, for anyone who depends on it.
4100
+ return Number.isFinite(parsed) && parsed >= 0
4101
+ ? parsed
4102
+ : DEFAULT_APP_RUNTIME_HELPER_DETACHED_IDLE_MS;
3866
4103
  }
3867
4104
 
3868
4105
  function resolveRuntimeRotationAppHelperDetachGraceMs(env = process.env) {
@@ -3900,13 +4137,181 @@ function pickRuntimeRotationAppHelperEnv(env) {
3900
4137
 
3901
4138
  function writeRuntimeRotationAppHelperStatus(payload, env = process.env) {
3902
4139
  try {
3903
- const statusPath = resolveRuntimeRotationAppHelperStatusPath(env);
4140
+ const statusPath = resolveRuntimeRotationAppHelperStatusPath(
4141
+ env,
4142
+ payload?.pid,
4143
+ );
3904
4144
  writeOwnerOnlyJsonFileAtomicSync(statusPath, payload);
3905
4145
  } catch {
3906
4146
  // Best-effort status only; the helper must not fail because telemetry is unavailable.
3907
4147
  }
3908
4148
  }
3909
4149
 
4150
+ let helperMetadataCleanupBusyFailuresRemaining = resolveTestFaultInjectionCount(
4151
+ "CODEX_MULTI_AUTH_TEST_HELPER_METADATA_CLEANUP_BUSY_FAILURES",
4152
+ );
4153
+
4154
+ function maybeThrowSimulatedHelperMetadataFileError() {
4155
+ if (
4156
+ Number.isFinite(helperMetadataCleanupBusyFailuresRemaining) &&
4157
+ helperMetadataCleanupBusyFailuresRemaining > 0
4158
+ ) {
4159
+ helperMetadataCleanupBusyFailuresRemaining -= 1;
4160
+ const error = new Error("simulated EBUSY");
4161
+ error.code = "EBUSY";
4162
+ throw error;
4163
+ }
4164
+ }
4165
+
4166
+ // Windows can hold a transient lock on a file another process just closed, so
4167
+ // every metadata deletion goes through the shared retry rather than a bare
4168
+ // rmSync — a swallowed EBUSY here is how stale files outlive their sweep.
4169
+ function removeHelperMetadataFileWithRetry(targetPath) {
4170
+ try {
4171
+ withSynchronousFileOperationRetry(() => {
4172
+ maybeThrowSimulatedHelperMetadataFileError();
4173
+ rmSync(targetPath, { force: true });
4174
+ });
4175
+ } catch {
4176
+ // Best-effort metadata cleanup only; the next sweep retries.
4177
+ }
4178
+ }
4179
+
4180
+ function removeRuntimeRotationAppHelperOwnerFile(env = process.env, helperPid) {
4181
+ removeHelperMetadataFileWithRetry(
4182
+ resolveRuntimeRotationAppHelperOwnerPath(env, helperPid),
4183
+ );
4184
+ }
4185
+
4186
+ // Owner and status files are written per helper PID and removed on clean
4187
+ // helper exit; a killed helper leaves its files behind. This sweep runs when a
4188
+ // launcher starts the next helper, mirroring the app-server shim-dir sweep:
4189
+ // any per-PID metadata whose helper is no longer alive is stale, as is the
4190
+ // legacy shared status file once the PID recorded inside it is dead. Terminal
4191
+ // status stamps ("idle-timeout", "stopped") therefore survive until the next
4192
+ // helper launch — long enough to be read, without accumulating forever.
4193
+ function sweepStaleRuntimeRotationAppHelperMetadata(env = process.env) {
4194
+ const multiAuthDir =
4195
+ resolveOriginalMultiAuthDir(env) ?? join(resolveCodexHomeDir(env), "multi-auth");
4196
+ let entries = [];
4197
+ try {
4198
+ entries = readdirSync(multiAuthDir, { withFileTypes: true });
4199
+ } catch {
4200
+ return;
4201
+ }
4202
+ // The same filename contract as `runtimeHelperPerPidPattern` in
4203
+ // lib/runtime-constants.ts, re-derived here from the same two constants
4204
+ // rather than imported: this wrapper has to keep working before `dist/` is
4205
+ // built (see `loadRuntimeConstants`), and a sweep that silently matched
4206
+ // nothing because an import failed would delete nothing and report success.
4207
+ // A change to the shape there is a change here.
4208
+ const perPidPattern = (baseName) =>
4209
+ new RegExp(
4210
+ `^${baseName.replace(/\.json$/i, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\.(\\d+)\\.json$`,
4211
+ "i",
4212
+ );
4213
+ const statusPattern = perPidPattern(APP_RUNTIME_HELPER_STATUS_FILE);
4214
+ const ownerPattern = perPidPattern(APP_RUNTIME_HELPER_OWNER_FILE);
4215
+ // A live kill(pid, 0) is not proof the helper is alive — the PID may have
4216
+ // been recycled since a SIGKILLed helper left its files behind, and a
4217
+ // recycled PID would otherwise shield the stale file from every future
4218
+ // sweep. When the file records when its helper started, a current process
4219
+ // whose kernel start time is meaningfully later cannot be that helper.
4220
+ // Identity probes are bounded: results are memoized per PID for the sweep
4221
+ // (the same PID backs both a status and an owner file), and at most a
4222
+ // handful of `ps` spawns run per launch — candidates past the cap are
4223
+ // treated as not-dead and the next launch finishes the work. Dead-PID
4224
+ // files, the overwhelming majority after a leak, never probe at all.
4225
+ const probedStartTimes = new Map();
4226
+ let probeBudget = 20;
4227
+ const probeStartTime = (pid) => {
4228
+ if (probedStartTimes.has(pid)) return probedStartTimes.get(pid);
4229
+ if (probeBudget <= 0) return undefined;
4230
+ probeBudget -= 1;
4231
+ const startTime = readProcessStartTimeMs(pid);
4232
+ probedStartTimes.set(pid, startTime);
4233
+ return startTime;
4234
+ };
4235
+ const isSweepCandidateDead = (pid, filePath) => {
4236
+ if (!isProcessAlive(pid)) return true;
4237
+ let recordedAt = null;
4238
+ try {
4239
+ const parsed = JSON.parse(readFileSync(filePath, "utf8"));
4240
+ if (parsed && typeof parsed === "object") {
4241
+ recordedAt =
4242
+ typeof parsed.startedAt === "number"
4243
+ ? parsed.startedAt
4244
+ : typeof parsed.createdAt === "number"
4245
+ ? parsed.createdAt
4246
+ : null;
4247
+ }
4248
+ } catch {
4249
+ return false;
4250
+ }
4251
+ if (recordedAt === null) return false;
4252
+ const actualStartTimeMs = probeStartTime(pid);
4253
+ if (actualStartTimeMs === null || actualStartTimeMs === undefined) {
4254
+ return false;
4255
+ }
4256
+ return actualStartTimeMs > recordedAt + 60_000;
4257
+ };
4258
+ // Classifying a file as stale and deleting it are two moments, and a PID
4259
+ // freed between them can be handed to a helper starting right now — which
4260
+ // then republishes this exact path before the delete lands, and the sweep
4261
+ // erases a live helper's metadata: invisible to `rotation status`, to
4262
+ // runtime account resolution, and to `unbind-app`, reapable only by its own
4263
+ // timers. Deleting only a file whose mtime still matches what was
4264
+ // classified closes that window; a file rewritten underneath us is by
4265
+ // definition not the one judged dead.
4266
+ const removeIfUnchanged = (entryPath, classifiedMtimeMs) => {
4267
+ if (classifiedMtimeMs !== null) {
4268
+ let currentMtimeMs = null;
4269
+ try {
4270
+ currentMtimeMs = statSync(entryPath).mtimeMs;
4271
+ } catch {
4272
+ return;
4273
+ }
4274
+ if (currentMtimeMs !== classifiedMtimeMs) return;
4275
+ }
4276
+ removeHelperMetadataFileWithRetry(entryPath);
4277
+ };
4278
+ for (const entry of entries) {
4279
+ if (!entry.isFile()) continue;
4280
+ const match =
4281
+ statusPattern.exec(entry.name) ?? ownerPattern.exec(entry.name);
4282
+ if (!match) continue;
4283
+ const pid = Number.parseInt(match[1], 10);
4284
+ if (!Number.isInteger(pid) || pid <= 0) continue;
4285
+ const entryPath = join(multiAuthDir, entry.name);
4286
+ let classifiedMtimeMs = null;
4287
+ try {
4288
+ classifiedMtimeMs = statSync(entryPath).mtimeMs;
4289
+ } catch {
4290
+ continue;
4291
+ }
4292
+ if (!isSweepCandidateDead(pid, entryPath)) continue;
4293
+ removeIfUnchanged(entryPath, classifiedMtimeMs);
4294
+ }
4295
+ const legacyStatusPath = join(multiAuthDir, APP_RUNTIME_HELPER_STATUS_FILE);
4296
+ try {
4297
+ const legacyMtimeMs = statSync(legacyStatusPath).mtimeMs;
4298
+ const parsed = JSON.parse(readFileSync(legacyStatusPath, "utf8"));
4299
+ const legacyPid =
4300
+ parsed && typeof parsed === "object" && Number.isInteger(parsed.pid)
4301
+ ? parsed.pid
4302
+ : null;
4303
+ if (
4304
+ legacyPid === null ||
4305
+ legacyPid <= 0 ||
4306
+ isSweepCandidateDead(legacyPid, legacyStatusPath)
4307
+ ) {
4308
+ removeIfUnchanged(legacyStatusPath, legacyMtimeMs);
4309
+ }
4310
+ } catch {
4311
+ // Missing or unreadable legacy status; nothing to sweep.
4312
+ }
4313
+ }
4314
+
3910
4315
  function writeRuntimeRotationAppHelperOwner(
3911
4316
  identityToken,
3912
4317
  helperPid,
@@ -3941,6 +4346,7 @@ function createRuntimeRotationAppHelperStatus({
3941
4346
  identityToken,
3942
4347
  idleTimeoutMs,
3943
4348
  lastActivityAt,
4349
+ idleExpiresAt,
3944
4350
  state,
3945
4351
  }) {
3946
4352
  const proxyStatus =
@@ -3964,7 +4370,13 @@ function createRuntimeRotationAppHelperStatus({
3964
4370
  updatedAt: Date.now(),
3965
4371
  baseUrl: proxyServer?.baseUrl ?? null,
3966
4372
  idleTimeoutMs,
3967
- idleExpiresAt: lastActivityAt + idleTimeoutMs,
4373
+ // The reaper's real deadline, which is the detached window once the owner
4374
+ // is gone. Reporting the raw idle timeout there would tell `rotation
4375
+ // status` a helper has 12h left when it has minutes.
4376
+ idleExpiresAt:
4377
+ typeof idleExpiresAt === "number"
4378
+ ? idleExpiresAt
4379
+ : lastActivityAt + idleTimeoutMs,
3968
4380
  totalRequests: proxyStatus.totalRequests ?? 0,
3969
4381
  upstreamRequests: proxyStatus.upstreamRequests ?? 0,
3970
4382
  retries: proxyStatus.retries ?? 0,
@@ -3985,21 +4397,97 @@ async function runRuntimeRotationAppHelper(identityToken = "") {
3985
4397
  let closing = false;
3986
4398
  const startedAt = Date.now();
3987
4399
  const idleTimeoutMs = resolveRuntimeRotationAppHelperIdleMs();
4400
+ const maxLifetimeMs = resolveRuntimeRotationAppHelperMaxLifetimeMs();
4401
+ const detachedIdleMs = resolveRuntimeRotationAppHelperDetachedIdleMs();
3988
4402
  const ownerPid = resolveRuntimeRotationAppHelperOwnerPid();
4403
+ const isOwnerAlive = createRuntimeRotationAppHelperOwnerLivenessCheck(
4404
+ ownerPid,
4405
+ resolveRuntimeRotationAppHelperOwnerStartTimeMs(),
4406
+ resolveRuntimeRotationAppHelperOwnerRecheckMs(idleTimeoutMs, detachedIdleMs),
4407
+ );
3989
4408
  let lastActivityAt = startedAt;
3990
4409
  let lastRequestCount = 0;
4410
+ let lastPublishedToken = null;
4411
+ let lastPublishedAt = 0;
4412
+ // When the owner was first confirmed dead. Null while it is alive, and reset
4413
+ // to null if a later probe revives the verdict, so a transient "dead" cannot
4414
+ // ratchet the detached deadline the way the old liveness check ratcheted the
4415
+ // idle one.
4416
+ let ownerGoneSince = null;
4417
+ // A detached consumer holding a socket is evidence the handoff was real,
4418
+ // so it blocks the detached reap even with no requests in flight. The
4419
+ // absence of that evidence is not evidence of absence, and this fails
4420
+ // open on purpose: a proxy that cannot report connections reads as zero,
4421
+ // so the detached window still reaps it on activity alone. Erring the
4422
+ // other way — treating "unknown" as "someone is attached" — would restore
4423
+ // the leak for any shape that stopped answering.
4424
+ const countOpenConnections = () => {
4425
+ if (typeof proxyServer?.getOpenConnectionCount !== "function") return 0;
4426
+ const open = proxyServer.getOpenConnectionCount();
4427
+ // Only a positive count of sockets is evidence of a consumer. Anything
4428
+ // else — negative, fractional, NaN, Infinity — is "unknown", and unknown
4429
+ // degrades exactly the way a missing method does. Comparing a garbage
4430
+ // reading against 0 directly would block the reap forever and silently
4431
+ // restore the leak this exists to close, which is the one direction the
4432
+ // fix cannot afford to fail in.
4433
+ return Number.isSafeInteger(open) && open > 0 ? open : 0;
4434
+ };
4435
+ // The deadline the reaper will actually enforce, which is the earlier of the
4436
+ // idle timeout and — once the owner is gone — the detached window.
4437
+ const resolveIdleDeadline = () =>
4438
+ ownerGoneSince !== null && detachedIdleMs > 0
4439
+ ? Math.min(
4440
+ lastActivityAt + idleTimeoutMs,
4441
+ Math.max(lastActivityAt, ownerGoneSince) + detachedIdleMs,
4442
+ )
4443
+ : lastActivityAt + idleTimeoutMs;
4444
+ // Freshness readers tolerate hours of staleness, but tests run the whole
4445
+ // lifecycle in milliseconds — heartbeat at least once per idle window.
4446
+ //
4447
+ // The detached window counts too. `publishToken` deliberately zeroes
4448
+ // `idleExpiresAt` so a deadline that moves every tick is not a reason to
4449
+ // rewrite the file, which means the published deadline only catches up on a
4450
+ // heartbeat. Once the owner dies the real deadline collapses from the idle
4451
+ // timeout to the detached window, so a heartbeat pinned to the idle window
4452
+ // alone would leave `rotation status` advertising a 12-hour deadline for a
4453
+ // helper that is seconds from exiting — worse under a short
4454
+ // CODEX_MULTI_AUTH_APP_ROTATION_DETACHED_IDLE_MS override, where the helper
4455
+ // can vanish before the file is ever corrected.
4456
+ const statusHeartbeatMs = Math.min(
4457
+ APP_RUNTIME_HELPER_STATUS_HEARTBEAT_MS,
4458
+ idleTimeoutMs,
4459
+ detachedIdleMs > 0 ? detachedIdleMs : Number.POSITIVE_INFINITY,
4460
+ );
3991
4461
 
3992
- const publishStatus = (state) => {
3993
- writeRuntimeRotationAppHelperStatus(
3994
- createRuntimeRotationAppHelperStatus({
3995
- proxyServer,
3996
- startedAt,
3997
- identityToken,
3998
- idleTimeoutMs,
3999
- lastActivityAt,
4000
- state,
4001
- }),
4002
- );
4462
+ const publishStatus = (state, { force = false } = {}) => {
4463
+ const payload = createRuntimeRotationAppHelperStatus({
4464
+ proxyServer,
4465
+ startedAt,
4466
+ identityToken,
4467
+ idleTimeoutMs,
4468
+ lastActivityAt,
4469
+ idleExpiresAt: resolveIdleDeadline(),
4470
+ state,
4471
+ });
4472
+ // `updatedAt` moves every call and `idleExpiresAt` moves every tick the
4473
+ // owner is alive; neither is a reason to rewrite the file. Everything
4474
+ // else changing — state, traffic counters, account fields — is.
4475
+ const publishToken = JSON.stringify({
4476
+ ...payload,
4477
+ updatedAt: 0,
4478
+ idleExpiresAt: 0,
4479
+ });
4480
+ const now = Date.now();
4481
+ if (
4482
+ !force &&
4483
+ publishToken === lastPublishedToken &&
4484
+ now - lastPublishedAt < statusHeartbeatMs
4485
+ ) {
4486
+ return;
4487
+ }
4488
+ lastPublishedToken = publishToken;
4489
+ lastPublishedAt = now;
4490
+ writeRuntimeRotationAppHelperStatus(payload);
4003
4491
  };
4004
4492
 
4005
4493
  const cleanup = async (state = "stopped") => {
@@ -4022,7 +4510,11 @@ async function runRuntimeRotationAppHelper(identityToken = "") {
4022
4510
  try {
4023
4511
  await proxyServer?.close?.();
4024
4512
  } finally {
4025
- publishStatus(state);
4513
+ // The terminal stamp is always written; the next launcher's sweep
4514
+ // removes it once this PID is dead. The owner file has no
4515
+ // post-mortem value, so it goes now.
4516
+ publishStatus(state, { force: true });
4517
+ removeRuntimeRotationAppHelperOwnerFile(process.env, process.pid);
4026
4518
  }
4027
4519
  }
4028
4520
  };
@@ -4095,7 +4587,10 @@ async function runRuntimeRotationAppHelper(identityToken = "") {
4095
4587
  type: "ready",
4096
4588
  pid: process.pid,
4097
4589
  baseUrl: proxyServer.baseUrl,
4098
- statusPath: resolveRuntimeRotationAppHelperStatusPath(),
4590
+ statusPath: resolveRuntimeRotationAppHelperStatusPath(
4591
+ process.env,
4592
+ process.pid,
4593
+ ),
4099
4594
  args: runtimeContext.args ?? [],
4100
4595
  env: pickRuntimeRotationAppHelperEnv(runtimeContext.env),
4101
4596
  })}\n`,
@@ -4108,14 +4603,51 @@ async function runRuntimeRotationAppHelper(identityToken = "") {
4108
4603
  lastRequestCount = requestCount;
4109
4604
  lastActivityAt = currentTime;
4110
4605
  }
4111
- if (ownerPid && isRuntimeRotationAppHelperOwnerAlive(ownerPid)) {
4606
+ const ownerVerdict = isOwnerAlive(currentTime);
4607
+ if (ownerVerdict === OWNER_ALIVE) {
4112
4608
  lastActivityAt = currentTime;
4609
+ ownerGoneSince = null;
4610
+ } else if (ownerVerdict === OWNER_DEAD && ownerGoneSince === null) {
4611
+ ownerGoneSince = currentTime;
4113
4612
  }
4114
4613
  publishStatus("running");
4115
4614
  if (currentTime - lastActivityAt >= idleTimeoutMs) {
4116
4615
  exitAfterCleanup("idle-timeout", 0);
4616
+ } else if (
4617
+ ownerGoneSince !== null &&
4618
+ detachedIdleMs > 0 &&
4619
+ currentTime >= resolveIdleDeadline() &&
4620
+ requestCount === 0 &&
4621
+ countOpenConnections() === 0
4622
+ ) {
4623
+ // The launcher is gone, nothing is connected, nothing has been
4624
+ // proxied for the detached window, and nothing has *ever* been
4625
+ // proxied: this helper was stranded by a launcher that exited, not
4626
+ // handed to a consumer that wants it.
4627
+ //
4628
+ // The never-served gate is what separates the two. An open socket
4629
+ // is not a durable signal — the proxy leaves `keepAliveTimeout` at
4630
+ // Node's 5s default, so a `codex app` session that is merely idle
4631
+ // between turns has zero sockets within seconds, and the socket
4632
+ // check alone would reap the live proxy out from under the desktop
4633
+ // app after the detached window. A helper that has served even one
4634
+ // request was genuinely handed off; from then on the idle timeout
4635
+ // and the lifetime ceiling bound it, exactly as they did before the
4636
+ // detached reap existed.
4637
+ exitAfterCleanup("owner-gone", 0);
4638
+ } else if (
4639
+ maxLifetimeMs > 0 &&
4640
+ currentTime - startedAt >= maxLifetimeMs
4641
+ ) {
4642
+ // The ceiling is deliberately unconditional on activity: it exists
4643
+ // for exactly the case where activity accounting is wrong.
4644
+ exitAfterCleanup("max-lifetime", 0);
4117
4645
  }
4118
- }, Math.min(1_000, Math.max(50, Math.floor(idleTimeoutMs / 2))));
4646
+ // Tick against the shortest window that can fire, so a short detached
4647
+ // window is enforced at its own resolution rather than the idle
4648
+ // timeout's. Both defaults are far above 2s, so production still ticks
4649
+ // once a second.
4650
+ }, resolveRuntimeRotationAppHelperTickMs(idleTimeoutMs, detachedIdleMs));
4119
4651
  } catch (error) {
4120
4652
  process.stdout.write(
4121
4653
  `${JSON.stringify({
@@ -4203,13 +4735,25 @@ function startRuntimeRotationAppHelper(baseContext, options = {}) {
4203
4735
  let stderrBuffer = "";
4204
4736
  let settled = false;
4205
4737
  const identityToken = randomBytes(24).toString("hex");
4738
+ // The launcher states its own identity — PID plus kernel start time — so
4739
+ // the helper's owner-liveness check can tell "my launcher" from a later
4740
+ // process that recycled the PID. An empty value (no `ps` on this
4741
+ // platform) leaves the helper on the bare liveness check.
4742
+ const launcherStartTimeMs = readProcessStartTimeMs(process.pid);
4206
4743
  const helperEnv = {
4207
4744
  ...baseContext.env,
4208
4745
  CODEX_MULTI_AUTH_DIR: resolveRuntimeRotationOriginalMultiAuthDir(
4209
4746
  realCodexHome,
4210
4747
  baseContext.env,
4211
4748
  ),
4749
+ // PID and start time are two halves of one identity and must describe
4750
+ // the same process: both always come from this launcher's own capture,
4751
+ // never from an inherited environment value, which would marry this
4752
+ // PID to another process's start time and make the helper declare its
4753
+ // live owner dead at the first recheck.
4212
4754
  [APP_RUNTIME_HELPER_OWNER_PID_ENV]: String(process.pid),
4755
+ [APP_RUNTIME_HELPER_OWNER_START_TIME_ENV]:
4756
+ launcherStartTimeMs !== null ? String(launcherStartTimeMs) : "",
4213
4757
  [APP_RUNTIME_HELPER_REAL_CODEX_HOME_ENV]: realCodexHome,
4214
4758
  [APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV]:
4215
4759
  options.useCanonicalHome === true ? "1" : "0",
@@ -4230,6 +4774,14 @@ function startRuntimeRotationAppHelper(baseContext, options = {}) {
4230
4774
  },
4231
4775
  );
4232
4776
  writeRuntimeRotationAppHelperOwner(identityToken, helper.pid, helperEnv);
4777
+ // Swept after the spawn, never before it. The sweep is synchronous and
4778
+ // unbounded — a readdir, a readFileSync and an rmSync per candidate, plus
4779
+ // bounded `ps` probes — and the state it exists to clean up (hundreds of
4780
+ // stale files, a loaded process table) is exactly the state that makes it
4781
+ // slow. Running it first put all of that latency in front of `codex app`
4782
+ // and TUI startup; running it here lets the helper boot in parallel with
4783
+ // it. Nothing about spawning depends on the sweep having finished.
4784
+ sweepStaleRuntimeRotationAppHelperMetadata(helperEnv);
4233
4785
  let timeout = null;
4234
4786
  const finish = (result) => {
4235
4787
  if (settled) return;