instar 1.3.1649 → 1.3.1650

package/dist/core/SessionManager.js

@@ -6,6 +6,7 @@
  * and can be monitored/reaped by the server.
  */
 import { isCredentialEnvName, CLAUDE_CONFINED_PERMISSION_ARGS } from './credentialEnvNames.js';
+import { featureEvidence } from '../monitoring/FeatureEvidenceLog.js';
 import { execFileSync, execFile } from 'node:child_process';
 import { isInstarSourceTree } from './SourceTreeGuard.js';
 import { AdmissionRefusedError } from './workAdmission.js';
@@ -44,6 +45,7 @@ import { detectRateLimited, nextIdleThrottleAction, RATE_LIMIT_SETTLED_CAPTURE_L
 import { classifyIdleError } from './IdleErrorClassifier.js';
 import { classifyLoginExpired } from './LoginExpiredSignature.js';
 import { liveTail } from './paneTail.js';
+import { claudeAuthExecPrefix, removeClaudeAuthEnvFile, writeClaudeAuthEnvFile } from './claudeChildAuth.js';
 import { resolveIdleSinceSeed } from './idleSincePersistence.js';
 import { toPaneTailLines, RESOLVER_PREGATE_RE } from '../monitoring/PermissionPromptAutoResolver.js';
 const execFileAsync = promisify(execFile);
@@ -88,6 +90,7 @@ const ageKillGov = governor.for('age-kill-backoff');
  * the lease-holder boundary unchanged.
  */
 const LOCAL_AGE_LIMIT_REAP_CAPABILITY = Symbol('local-age-limit-reap-capability');
+const REROUTED_LIFETIME_REAP_CAPABILITY = Symbol('rerouted-lifetime-reap-capability');
 /**
  * How long a confirmed-move assertion stays valid. A CODE CONSTANT, never config:
  * a safety bound in config is a safety bound an emergency edit can silently remove
@@ -1242,6 +1245,13 @@ export class SessionManager extends EventEmitter {
             // guard counts the degraded-call signal distinctly from a generic error.
             const killed = e.killed === true || e.signal === 'SIGKILL' || /SIGKILL/.test(msg);
             this.degradedTmuxGuard?.observeTmuxCall(Date.now() - t0, killed ? 'killed-client' : 'indeterminate');
+            if (this.tmuxAsyncEnabled) {
+                featureEvidence.recordAtMostEvery('monitoring.tmuxResilience.asyncHotPath', killed ? 'timeout' : 'indeterminate', 300_000, {
+                    kind: killed ? 'fired' : 'error',
+                    reason: killed ? 'bounded-call-timed-out-and-killed' : 'tmux-call-failed-indeterminate',
+                    context: { op: args[0] ?? 'unknown', elapsedMs: Date.now() - t0, timeoutMs: timeout, inFlight: this.tmuxInflightCount },
+                });
+            }
             return { state: 'indeterminate', reason: msg.slice(0, 120) };
         }
     }
@@ -1734,6 +1744,51 @@ rm()  { "${shimRunner}" rm  "$@"; }
     setInteractiveLoginExpiredSink(fn) {
         this.interactiveLoginExpiredSink = fn;
     }
+    /** Put a handler ahead of the sink. When it returns true the signal is handled and goes no further. */
+    prependInteractiveLoginExpiredSink(first) {
+        const rest = this.interactiveLoginExpiredSink;
+        this.interactiveLoginExpiredSink = async (input) => {
+            if (await first(input))
+                return;
+            await rest?.(input);
+        };
+    }
+    /** A restored sign-in key for every later launch. `config` here is a copy, so server code can't reach it. */
+    setAnthropicKey(key) {
+        this.config.anthropicApiKey = key ?? undefined;
+    }
+    /** Forget which sessions were already reported, so a stuck one is offered again on the next check. */
+    clearInteractiveLoginReports() {
+        this.interactiveLoginReportedAt.clear();
+    }
+    /** Unprotected topic sessions showing Claude Code's own login frame right now. */
+    async listSessionsAtLoginScreen() {
+        const stuck = [];
+        for (const session of this.listRunningSessions()) {
+            if ((session.framework ?? 'claude-code') !== 'claude-code' || session.jobSlug)
+                continue;
+            if (this.config.protectedSessions.includes(session.tmuxSession))
+                continue;
+            const pane = (await this.captureOutputMaybeAsync(session.tmuxSession, 40)) || '';
+            const verdict = classifyLoginExpired(pane);
+            if (verdict.loginExpired && verdict.messageLed)
+                stuck.push(session.tmuxSession);
+        }
+        return stuck;
+    }
+    /** The key a session was launched with, as tmux holds it. Read for comparison only: never log it. */
+    sessionLaunchKey(tmuxSession) {
+        try {
+            const line = withSyncOp(() => execFileSync(this.config.tmuxPath, ['show-environment', '-t', `=${tmuxSession}:`, 'CLAUDE_CODE_OAUTH_TOKEN'], {
+                encoding: 'utf-8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'],
+            })).trim();
+            const value = line.startsWith('CLAUDE_CODE_OAUTH_TOKEN=') ? line.slice('CLAUDE_CODE_OAUTH_TOKEN='.length) : '';
+            return value || null;
+        }
+        catch {
+            return null;
+        } // @silent-fallback-ok: unset or unreadable means no launch key, and no key means no refresh
+    }
     setCodexSigninTriggerSink(fn) {
         this.codexSigninTriggerSink = fn;
     }
@@ -2063,6 +2118,15 @@ rm()  { "${shimRunner}" rm  "$@"; }
         }
         const isAuthorizedTopicMovedCloseout = reason.startsWith('topic moved')
             && opts?.localPostTransferCloseout === true;
+        const isAuthorizedReroutedLifetime = reason === 'rerouted-lifetime'
+            && opts?.reroutedLifetimeReapCapability === REROUTED_LIFETIME_REAP_CAPABILITY
+            && session.launchLane === 'rerouted-interactive'
+            && session.completionMode === 'pattern'
+            && typeof session.maxLifetimeMinutes === 'number'
+            && Number.isFinite(session.maxLifetimeMinutes)
+            && session.maxLifetimeMinutes > 0
+            && !!session.startedAt
+            && Date.now() - Date.parse(session.startedAt) > session.maxLifetimeMinutes * 60_000;
         const authorityScope = origin === 'operator'
             ? 'operator'
             : holdsServingLease
@@ -2127,6 +2191,14 @@ rm()  { "${shimRunner}" rm  "$@"; }
                     bypassedReasons.push('recovery-in-flight');
                 if (opts?.bypassActiveProcessKeep)
                     bypassedReasons.push('active-process');
+                // A rerouted one-shot is an interactive REPL, so a live child process can
+                // remain after its hard deadline. The monitor's private capability and
+                // this second expiry check let the hard cap bypass only process-activity
+                // evidence. Protected sessions, user activity, commitments and other
+                // KEEP reasons still veto termination.
+                if (isAuthorizedReroutedLifetime) {
+                    bypassedReasons.push('active-process', 'main-process-active', 'process-uninspectable');
+                }
                 if (opts?.bypassRecentUserMessageForConfirmedMove || opts?.topicTransferClose === true) {
                     bypassedReasons.push('recent-user-message');
                 }
@@ -2608,6 +2680,13 @@ rm()  { "${shimRunner}" rm  "$@"; }
                 // (A) tri-state: ONLY a definitive `false` (dead) marks the session
                 // completed. `'indeterminate'` (slow/timed-out tmux) does NOT — it falls
                 // through to be re-checked next tick rather than reaping a live session.
+                if (alive === 'indeterminate') {
+                    featureEvidence.recordAtMostEvery('monitoring.tmuxResilience.asyncHotPath', `kept:${session.tmuxSession}`, 300_000, { kind: 'held', reason: 'indeterminate-probe-kept-session', context: { session: session.tmuxSession } });
+                }
+                if (alive === false && this.tmuxAsyncEnabled) {
+                    // A definitive-dead verdict is the one a spurious reap would come from; the review grades these rows.
+                    featureEvidence.record('monitoring.tmuxResilience.asyncHotPath', { kind: 'fired', reason: 'probe-classified-session-dead', context: { session: session.tmuxSession } });
+                }
                 if (alive === false) {
                     // CAS re-read after the await: an operator kill (or another killer)
                     // can land mid-probe, so re-read the live record before mutating it —
@@ -2718,12 +2797,14 @@ rm()  { "${shimRunner}" rm  "$@"; }
                         if (lifeElapsedMin > session.maxLifetimeMinutes &&
                             !this.config.protectedSessions.includes(session.tmuxSession) &&
                             !this.isStandingSession(session.tmuxSession)) {
-                            console.warn(`[SessionManager] Rerouted session "${session.name}" exceeded its hard lifetime (${Math.round(lifeElapsedMin)}m > ${session.maxLifetimeMinutes}m) without a completion sentinel. Killing (timeout).`);
-                            const killed = await this.terminateSession(session.id, 'rerouted-lifetime', {
+                            const killed = await this.#terminateSessionInternal(session.id, 'rerouted-lifetime', {
                                 finalStatus: 'killed',
                                 disposition: 'terminal',
+                                origin: 'autonomous',
+                                reroutedLifetimeReapCapability: REROUTED_LIFETIME_REAP_CAPABILITY,
                             });
                             if (killed.terminated) {
+                                console.warn(`[SessionManager] Rerouted session "${session.name}" exceeded its hard lifetime (${Math.round(lifeElapsedMin)}m > ${session.maxLifetimeMinutes}m) without a completion sentinel. Killed (timeout).`);
                                 DegradationReporter.getInstance().report({
                                     feature: 'SessionManager.monitorTick.reroutedLifetime',
                                     primary: 'rerouted-interactive session completes via its sentinel within the lifetime cap',
@@ -2997,11 +3078,15 @@ rm()  { "${shimRunner}" rm  "$@"; }
                             const settledSnapshot = await this.captureOutputMaybeAsync(session.tmuxSession, RATE_LIMIT_SETTLED_CAPTURE_LINES);
                             const { action, nextState } = nextIdleThrottleAction(settledSnapshot, this.idleThrottleSettle.get(session.tmuxSession), now);
                             if (action === 'emit') {
+                                featureEvidence.record('monitoring.idleThrottleSettleGate', { kind: 'fired', reason: 'settled-handed-to-rate-limit-sentinel', context: { session: session.tmuxSession, snapshotRead: settledSnapshot != null } });
                                 this.idleThrottleSettle.delete(session.tmuxSession);
                                 this.emit('rateLimitedAtIdle', session.tmuxSession);
                                 continue; // Settled → sentinel owns recovery.
                             }
                             if (action === 'wait') {
+                                if (!this.idleThrottleSettle.has(session.tmuxSession)) {
+                                    featureEvidence.record('monitoring.idleThrottleSettleGate', { kind: 'held', reason: 'throttle-seen-waiting-to-settle', context: { session: session.tmuxSession } });
+                                }
                                 if (nextState)
                                     this.idleThrottleSettle.set(session.tmuxSession, nextState);
                                 continue; // Not settled yet — re-sample next idle tick; do NOT emit, and
@@ -3010,6 +3095,15 @@ rm()  { "${shimRunner}" rm  "$@"; }
                             }
                             // 'fall-through': no current throttle in the settled window (a stale buffer
                             // line) → clear tracking and proceed to the normal idle machinery below.
+                            if (this.idleThrottleSettle.has(session.tmuxSession) || settledSnapshot == null) {
+                                // The row the gate exists for: a throttle string that never settled gets no hand-off.
+                                if (settledSnapshot == null) {
+                                    featureEvidence.recordAtMostEvery('monitoring.idleThrottleSettleGate', `read-failed:${session.tmuxSession}`, 3_600_000, { kind: 'error', reason: 'pane-read-failed', context: { session: session.tmuxSession } });
+                                }
+                                else {
+                                    featureEvidence.record('monitoring.idleThrottleSettleGate', { kind: 'held', reason: 'throttle-gone-before-settle-no-handoff', context: { session: session.tmuxSession } });
+                                }
+                            }
                             this.idleThrottleSettle.delete(session.tmuxSession);
                         }
                         if (!this.idlePromptSince.has(session.id)) {
@@ -3924,25 +4018,29 @@ rm()  { "${shimRunner}" rm  "$@"; }
     confinedRerouteCapacity() {
         const pressure = this.currentMemoryPressure();
         const running = this.listRunningSessions();
-        if (running.length + this.pendingHeadlessSpawns >= this.config.maxSessions
-            && !this.scheduledOverflowAvailable(running, pressure))
+        // Feedback canaries/builds are not scheduler-proven spawns. The early check
+        // must use their actual gate, not the scheduler's reserved global slot.
+        if (running.length + this.pendingHeadlessSpawns >= this.config.maxSessions)
             return 'saturated';
         if ((this.config.subscriptionPathMode ?? 'off') !== 'force')
             return 'not-rerouted';
-        const maxRerouted = this.config.subscriptionMaxRerouted ?? 3;
-        return this.countReroutedInteractiveSessions() >= maxRerouted
+        return this.countReroutedInteractiveSessions() >= this.rerouteAdmissionCap(false)
             || pressure === 'high' || pressure === 'critical' ? 'saturated' : 'available';
     }
+    rerouteAdmissionCap(scheduledJob) {
+        const maxRerouted = this.config.subscriptionMaxRerouted ?? 3;
+        // A one-slot host cannot reserve without blocking all conversations.
+        return !scheduledJob && maxRerouted > 1 ? maxRerouted - 1 : maxRerouted;
+    }
     evaluateRerouteGate(spawnName, scheduledJob = false) {
         const mode = this.config.subscriptionPathMode ?? 'off';
-        const maxRerouted = this.config.subscriptionMaxRerouted ?? 3;
         const reroutedCount = this.countReroutedInteractiveSessions();
         const pressure = this.currentMemoryPressure();
         const pressureElevated = pressure === 'high' || pressure === 'critical';
         // Keep one of the existing memory-bounded slots for scheduled jobs when
         // conversations would otherwise fill the entire lane. A one-slot host
         // cannot reserve without blocking all conversations, so retain its cap.
-        const admissionCap = !scheduledJob && maxRerouted > 1 ? maxRerouted - 1 : maxRerouted;
+        const admissionCap = this.rerouteAdmissionCap(scheduledJob);
         if (reroutedCount < admissionCap && !pressureElevated) {
             return { allow: true };
         }
@@ -5641,6 +5739,7 @@ rm()  { "${shimRunner}" rm  "$@"; }
         }
         catch (err) {
             console.warn(`[SessionManager] Failed to record build context for "${tmuxSession}": ${err instanceof Error ? err.message : String(err)}`);
+            featureEvidence.recordAtMostEvery('sessions.respawnBuildContext', `capture-failed|${tmuxSession}`, 60 * 60 * 1000, { kind: 'error', reason: 'capture-failed', context: { tmuxSession, errorName: err instanceof Error ? err.name : 'unknown' } });
         }
     }
     /** Async twin of recordBuildContext — async cwd probe, SAME persistence. */
@@ -5655,15 +5754,19 @@ rm()  { "${shimRunner}" rm  "$@"; }
         }
         catch (err) {
             console.warn(`[SessionManager] Failed to record build context for "${tmuxSession}": ${err instanceof Error ? err.message : String(err)}`);
+            featureEvidence.recordAtMostEvery('sessions.respawnBuildContext', `capture-failed|${tmuxSession}`, 60 * 60 * 1000, { kind: 'error', reason: 'capture-failed', context: { tmuxSession, errorName: err instanceof Error ? err.name : 'unknown' } });
         }
     }
     withBuildContextRestoreNote(tmuxSession, initialMessage, resumeSessionId) {
         if (!resumeSessionId || !this.buildContextStore)
             return initialMessage;
         const restore = this.buildContextStore.getRestore(tmuxSession);
-        if (!restore)
+        if (!restore) {
+            featureEvidence.record('sessions.respawnBuildContext', { kind: 'held', reason: 'no-usable-context-on-resume', context: { tmuxSession } });
             return initialMessage;
+        }
         console.log(`[SessionManager] Restoring build context for "${tmuxSession}" from ${restore.entry.currentCwd}`);
+        featureEvidence.record('sessions.respawnBuildContext', { kind: 'fired', reason: 'context-restored-on-resume', context: { tmuxSession, contextAgeMs: Date.now() - restore.entry.updatedAt } });
         return [restore.note, initialMessage].filter(Boolean).join('\n\n');
     }
     /**
@@ -7246,6 +7349,8 @@ rm()  { "${shimRunner}" rm  "$@"; }
         this.linkAgentOwnedMemory('claude-code', {}, this.config.projectDir);
         // Planned once the triage binary is resolved (below); the env slot needs it first.
         let triageThreadline = { transport: 'stdio', reason: 'not-wired', secretEnvFlags: [] };
+        let triageCredentialSource = 'store';
+        let triageAuthFile = null;
         const triageBinaryForPlan = resolveFrameworkBinaryPath({
             framework: 'claude-code',
             frameworkBinaryPaths: this.config.frameworkBinaryPaths,
@@ -7324,6 +7429,12 @@ rm()  { "${shimRunner}" rm  "$@"; }
                     `${this.config.framework ?? 'unset'}). Refusing rather than ` +
                     'spawning another framework\'s binary with Claude flags.');
             }
+            // A key-only agent has no stored login, so triage answered "Not logged in". The key arrives
+            // through a private file, not `-e`, so it stays out of argv (spec cloud-key-internal-calls).
+            triageAuthFile = writeClaudeAuthEnvFile(path.join(this.config.projectDir, '.instar'));
+            triageCredentialSource = triageAuthFile ? 'env' : 'store';
+            if (triageAuthFile)
+                tmuxArgs.push(...claudeAuthExecPrefix(triageAuthFile));
             // CMT-798: triage panes are claude-code sessions too — retained copy.
             tmuxArgs.push(this.resolveLaunchBinary('claude-code', triageBinary));
             // Scoped permissions: allowedTools + permissionMode (NOT --dangerously-skip-permissions)
@@ -7351,6 +7462,7 @@ rm()  { "${shimRunner}" rm  "$@"; }
             }
         }
         catch (err) {
+            removeClaudeAuthEnvFile(triageAuthFile);
             this.revokeOriginSession(triageSessionId);
             throw new Error(`Failed to create triage tmux session: ${this.originSpawnError(err)}`);
         }
@@ -7362,11 +7474,10 @@ rm()  { "${shimRunner}" rm  "$@"; }
             tmuxSession,
             startedAt: new Date().toISOString(),
             maxDurationMinutes: 10,
-            // §2.10 provenance — the triage lane hardcodes an EMPTY `ANTHROPIC_API_KEY=` above
-            // (it always reads the per-CLAUDE_CONFIG_DIR store, never an env token), so 'store' is
-            // the deterministic provenance here. Set explicitly so no claude-code spawn record is
-            // provenance-blind (an env-token lane that forgot the flag would be a lint-caught bug).
-            credentialSource: 'store',
+            // §2.10 provenance — 'env' when the configured sign-in key was handed over above, 'store'
+            // otherwise. Set explicitly so no claude-code spawn record is provenance-blind (an env-token
+            // lane that forgot the flag would be a lint-caught bug).
+            credentialSource: triageCredentialSource,
             ...this.threadlineTransportTag(triageThreadline),
         };
         this.state.saveSession(session);