crewly 1.20.355 → 1.20.358

package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.js

@@ -24,6 +24,17 @@
  *    most every BILLING_PROBE_INTERVAL_MS. A switch-back that fails (the limit
  *    comes straight back) doubles the next probe interval.
  * 7. **Tell the owner once** per event, and once when it is over.
+ * 8. **Prefer the better target.** While the configured runtime is still out,
+ *    an agent on a fallback moves (at an idle boundary, or when it is next
+ *    started) to the earliest available entry of its chain when that is
+ *    earlier than the one it is on — orc on `claude-code@work` goes back to
+ *    `claude-code` once the default login has usage, even though its own
+ *    runtime (DeepSeek, out of credit) is not back.
+ * 9. **Stale marks are cleared** when the reset time passes and a probe
+ *    answers, when the owner signs that login in again
+ *    ({@link RuntimeFallbackService.onRuntimeLogin} /
+ *    {@link RuntimeFallbackService.onAccountLogin}), or when another session
+ *    on that target is seen working ({@link RuntimeFallbackService.reportWorking}).
  *
  * A chain entry may be one of the owner's other Claude Code accounts
  * (`claude-code@work`, issue #942): it is a runtime target of its own — out
@@ -41,6 +52,7 @@
  */
 import { ORCHESTRATOR_SESSION_NAME, RUNTIME_FALLBACK_CONSTANTS, RUNTIME_TYPES } from '../../constants.js';
 import { detectUsageLimit } from './usage-limit-rules.js';
+import { clearUsageLimit } from './usage-limit-state.js';
 import { traceRuntimeBlocked } from '../trace/trace-recorder.js';
 import { accountOf, baseRuntimeOf, parseRuntimeTarget, runtimeTarget } from '../harness/claude-accounts.js';
 import { applySettingsPatch, runtimeLabel, runtimeShortLabel, } from './runtime-fallback.types.js';
@@ -76,6 +88,8 @@ export class RuntimeFallbackService {
     noticeDue = new Map();
     /** Kickoff notes waiting for a session's next launch */
     kickoffNotes = new Map();
+    /** Last working-evidence look per session (throttle) */
+    workingSeen = new Map();
     /** Last switch-back per runtime, to recognise one that failed */
     recentRecoveries = new Map();
     availability = null;
@@ -179,7 +193,7 @@ export class RuntimeFallbackService {
         this.availability = null;
         const target = runtimeTarget(RUNTIME_TYPES.CLAUDE_CODE, account);
         try {
-            await this.recoverSignedOutAccount(target);
+            await this.recheckAfterLogin(target);
         }
         finally {
             // The owner is away: a freshly signed-in account must be usable at once,
@@ -254,27 +268,94 @@ export class RuntimeFallbackService {
         }
     }
     /**
-     * If the account was marked signed out, probe it and bring it back.
+     * The owner signed a harness's default login in again (the re-login flow
+     * finished for it). A mark on that runtime — signed out, or out of usage:
+     * the new login may be another account, or the same one with usage bought
+     * — is probed now instead of at its reset time, and cleared when the
+     * probe answers.
      *
-     * @param target - `claude-code@<account>`
+     * @param runtime - Runtime id (`claude-code`, `codex-cli`, …; the harness id)
+     * @returns Resolves when done; never rejects
+     *
+     * @example
+     * ```ts
+     * relogin.setHarnessLoginHandler((harnessId) => service.onRuntimeLogin(harnessId));
+     * ```
      */
-    async recoverSignedOutAccount(target) {
+    async onRuntimeLogin(runtime) {
+        this.availability = null;
+        try {
+            await this.recheckAfterLogin(runtime);
+        }
+        catch (err) {
+            this.logger.warn('Re-checking a runtime after a sign-in failed', { runtime, error: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    /**
+     * A login of a runtime target was signed in again: if it is marked (signed
+     * out or out of usage), probe it and bring it back when it answers.
+     *
+     * @param target - Runtime target (`claude-code`, `claude-code@<account>`)
+     */
+    async recheckAfterLogin(target) {
         const entry = this.state.exhausted[target];
-        if (!entry || entry.kind !== 'login')
+        if (!entry)
             return;
         const result = await this.deps.probe(target).catch(() => 'unknown');
         entry.lastProbeAt = new Date(this.now()).toISOString();
         if (result === 'available') {
+            this.logger.info('Signed in again and the probe answers — clearing the out-of-usage mark', { runtime: target, kind: entry.kind ?? 'usage_limit' });
             await this.recover(target);
             return;
         }
-        if (result === 'limited') {
+        if (result === 'limited' && entry.kind === 'login') {
             // Signed in again, but that account is out of usage too.
             entry.kind = 'usage_limit';
             entry.ruleId = 'probe_limited';
         }
         this.save();
     }
+    /**
+     * A session's model just answered with a tool call (Claude Code
+     * PreToolUse hook). A model cannot do that while its account is out of
+     * usage, so a mark on the target the session runs on is stale: it is
+     * cleared (agents moved off it come back at their idle boundary). Marks
+     * newer than WORKING_EVIDENCE_GRACE_MS are kept (a turn already streaming
+     * when the limit hit proves nothing). Looked at most once a minute per
+     * session, and only while a Claude Code target is marked.
+     *
+     * @param sessionName - Session the hook came from
+     */
+    reportWorking(sessionName) {
+        if (!this.state.settings.enabled || this.isExempt(sessionName))
+            return;
+        if (!Object.keys(this.state.exhausted).some((r) => baseRuntimeOf(r) === RUNTIME_TYPES.CLAUDE_CODE))
+            return;
+        if (this.switching.has(sessionName))
+            return;
+        const now = this.now();
+        if (now - (this.workingSeen.get(sessionName) ?? Number.NEGATIVE_INFINITY) < C.WORKING_EVIDENCE_THROTTLE_MS)
+            return;
+        this.workingSeen.set(sessionName, now);
+        void this.onWorking(sessionName).catch((err) => this.logger.warn('Runtime fallback could not use a working signal', { sessionName, error: err instanceof Error ? err.message : String(err) }));
+    }
+    /**
+     * Clear the mark on the target a working session runs on.
+     *
+     * @param sessionName - Session
+     */
+    async onWorking(sessionName) {
+        const target = this.state.overrides[sessionName]?.runtime ?? (await this.deps.getAgent(sessionName).catch(() => null))?.primary;
+        if (!target || baseRuntimeOf(target) !== RUNTIME_TYPES.CLAUDE_CODE)
+            return;
+        const entry = this.state.exhausted[target];
+        if (!entry)
+            return;
+        if (this.now() - Date.parse(entry.since) < C.WORKING_EVIDENCE_GRACE_MS)
+            return;
+        this.logger.info('A session is working on a runtime marked out of usage — clearing the mark', { runtime: target, sessionName, since: entry.since });
+        await this.recover(target);
+    }
     /**
      * Take the note for a session's next kickoff.
      *
@@ -356,10 +437,23 @@ export class RuntimeFallbackService {
             }
             else if (existing.revertPending && !this.state.exhausted[existing.primary]) {
                 // A stopped agent being started after its runtime came back.
-                this.finishRevertOffline(input.sessionName, existing);
+                await this.finishRevertOffline(input.sessionName, existing);
                 return configured;
             }
+            else if (!this.state.exhausted[existing.primary]) {
+                // Its runtime is back but not yet marked for revert: the next tick reverts it.
+                return this.decision(existing.runtime);
+            }
             else {
+                // Starting anyway: on the best target of its chain, not on one that ran
+                // out meanwhile or below one that came back.
+                const agent = await this.launchAgent(input);
+                const better = await this.betterTarget(agent, existing.runtime, true);
+                if (better) {
+                    await this.recordSwitch(agent, existing.runtime, better, null, this.state.exhausted[existing.runtime] ? 'switch' : 'upgrade');
+                    this.logger.info('Starting an agent on a better fallback runtime', { sessionName: input.sessionName, from: existing.runtime, runtime: better });
+                    return this.decision(better);
+                }
                 return this.decision(existing.runtime);
             }
         }
@@ -373,14 +467,7 @@ export class RuntimeFallbackService {
         }
         if (input.isOrchestrator && !this.state.settings.orcFollows)
             return configured;
-        const agent = (await this.deps.getAgent(input.sessionName).catch(() => null)) ?? {
-            sessionName: input.sessionName,
-            name: input.sessionName,
-            primary: input.configured,
-            memberId: input.memberId,
-            teamId: input.teamId,
-            isOrchestrator: input.isOrchestrator,
-        };
+        const agent = await this.launchAgent(input);
         const target = await this.pickFallback(agent, input.configured);
         if (!target) {
             this.markNoFallback(input.configured);
@@ -394,6 +481,22 @@ export class RuntimeFallbackService {
         });
         return this.decision(target);
     }
+    /**
+     * The agent behind a launch (from the launch input when it is not known).
+     *
+     * @param input - Launch input
+     * @returns Agent
+     */
+    async launchAgent(input) {
+        return ((await this.deps.getAgent(input.sessionName).catch(() => null)) ?? {
+            sessionName: input.sessionName,
+            name: input.sessionName,
+            primary: input.configured,
+            memberId: input.memberId,
+            teamId: input.teamId,
+            isOrchestrator: input.isOrchestrator,
+        });
+    }
     /**
      * The configured runtime of a launching agent is not installed / not signed
      * in on this machine: record it like a signed-out account so the agent starts
@@ -510,8 +613,11 @@ export class RuntimeFallbackService {
         }
         else {
             const confirmed = await this.confirm(runtime);
-            if (!confirmed)
+            if (!confirmed) {
+                // The probe answered normally: the session is not out of usage either.
+                clearUsageLimit(sessionName);
                 return;
+            }
             if (!this.state.exhausted[runtime]) {
                 // The limit came straight back after a switch-back: back off the
                 // next probe, and do not tell the owner a second time.
@@ -590,6 +696,10 @@ export class RuntimeFallbackService {
         }
         finally {
             this.switching.delete(sessionName);
+            // On another runtime now (or back on one that has usage again): the
+            // session's own usage-limit state no longer holds.
+            if (outcome === 'switched' || outcome === 'reverted')
+                clearUsageLimit(sessionName);
             await this.afterSwitch(sessionName, outcome, opts.flushAfter);
         }
     }
@@ -653,18 +763,26 @@ export class RuntimeFallbackService {
      * @param from - Runtime it leaves (out of usage)
      * @param target - Runtime it goes to
      * @param handover - Handover file, when written
+     * @param direction - `switch` (`from` ran out) or `upgrade` (`target` ranks higher in the chain)
      */
-    async recordSwitch(agent, from, target, handover = null) {
+    async recordSwitch(agent, from, target, handover = null, direction = 'switch') {
         const sessionName = agent.sessionName;
         const existing = this.state.overrides[sessionName];
         const exhausted = this.state.exhausted[from];
-        // The primary's conversation is kept for the switch back; the fallback
-        // runtime starts fresh.
-        const primarySessionId = existing?.primarySessionId ?? (from === agent.primary ? this.deps.conversation.get(sessionName) : undefined);
+        const currentId = this.deps.conversation.get(sessionName);
+        // The primary's conversation is kept for the switch back. Between two
+        // Claude Code targets (the default login and another account) the
+        // conversation itself travels: its transcript is copied into the target
+        // account's config dir and resumed there (crewly#1344). Any other hop
+        // starts fresh with the handover.
+        const primarySessionId = existing?.primarySessionId ?? (from === agent.primary ? currentId : undefined);
+        const carried = currentId ? await this.carryConversation(sessionName, from, target, currentId) : false;
         this.deps.conversation.clear(sessionName);
+        if (carried)
+            this.deps.conversation.set(sessionName, currentId);
         if (target === agent.primary) {
             delete this.state.overrides[sessionName];
-            if (primarySessionId)
+            if (!carried && primarySessionId)
                 this.deps.conversation.set(sessionName, primarySessionId);
         }
         else {
@@ -685,9 +803,37 @@ export class RuntimeFallbackService {
         }
         this.save();
         const workItem = await this.deps.getActiveWorkItem(sessionName).catch(() => null);
-        this.kickoffNotes.set(sessionName, this.switchNote(from, target, exhausted?.until, handover, workItem));
+        this.kickoffNotes.set(sessionName, direction === 'upgrade'
+            ? this.upgradeNote(agent.primary, from, target, handover, workItem, carried)
+            : this.switchNote(from, target, exhausted?.until, handover, workItem, carried));
+        if (carried)
+            this.logger.info('Conversation carried to the target Claude Code account', { sessionName, from, to: target, conversationId: currentId });
         this.scheduleNotice(from);
     }
+    /**
+     * Whether the conversation can continue on the target: both targets run
+     * Claude Code (different accounts or the default login) and the wiring
+     * copied its transcript there. Never throws.
+     *
+     * @param sessionName - Session
+     * @param from - Target it leaves
+     * @param to - Target it goes to
+     * @param conversationId - Its conversation on `from`
+     * @returns True when `--resume <conversationId>` works on `to`
+     */
+    async carryConversation(sessionName, from, to, conversationId) {
+        if (!this.deps.carryConversation)
+            return false;
+        if (baseRuntimeOf(from) !== RUNTIME_TYPES.CLAUDE_CODE || baseRuntimeOf(to) !== RUNTIME_TYPES.CLAUDE_CODE)
+            return false;
+        try {
+            return await this.deps.carryConversation({ sessionName, from, to, conversationId });
+        }
+        catch (err) {
+            this.logger.warn('Could not carry the conversation to the target account; it starts fresh with the handover', { sessionName, from, to, error: err instanceof Error ? err.message : String(err) });
+            return false;
+        }
+    }
     /**
      * Wait until the agent is not mid-turn (bounded: a usage limit ends the turn anyway).
      *
@@ -731,6 +877,33 @@ export class RuntimeFallbackService {
         const own = agent.memberId ? this.state.settings.memberChains[agent.memberId] : undefined;
         return own && own.length > 0 ? own : this.state.settings.chain;
     }
+    /**
+     * Where an agent whose configured runtime is out should be instead of
+     * `current`: the earliest available entry of its chain (never its
+     * configured runtime, never one marked out of usage), when that ranks
+     * before `current` — or, with `orAnyWhenCurrentOut`, any available entry
+     * when `current` itself is marked out.
+     *
+     * @param agent - Agent (its configured runtime is out)
+     * @param current - Runtime target it is on
+     * @param orAnyWhenCurrentOut - Also move off a `current` that ran out to a lower-ranked entry
+     * @returns Better target, or null to stay
+     */
+    async betterTarget(agent, current, orAnyWhenCurrentOut) {
+        const best = await this.pickFallback(agent, agent.primary);
+        if (!best || best === current)
+            return null;
+        const chain = this.chainFor(agent);
+        const rank = (r) => {
+            const i = chain.indexOf(r);
+            return i < 0 ? Number.POSITIVE_INFINITY : i;
+        };
+        if (rank(best) < rank(current))
+            return best;
+        if (orAnyWhenCurrentOut && this.state.exhausted[current])
+            return best;
+        return null;
+    }
     /**
      * Whether some runtime could take over (synchronous; uses the cached
      * availability, optimistic when nothing is cached yet).
@@ -894,11 +1067,17 @@ export class RuntimeFallbackService {
         }
         await this.revertIdle();
     }
-    /** Revert every pending override whose agent is idle (or not running). */
+    /**
+     * Revert every pending override whose agent is idle (or not running), and
+     * move an idle agent whose configured runtime is still out to a better
+     * entry of its chain ({@link betterTarget}).
+     */
     async revertIdle() {
         for (const [sessionName, override] of Object.entries(this.state.overrides)) {
-            if (this.state.exhausted[override.primary])
+            if (this.state.exhausted[override.primary]) {
+                await this.upgradeIfBetter(sessionName, override);
                 continue;
+            }
             if (!override.revertPending) {
                 // State from before a restart whose primary is no longer marked out of usage.
                 override.revertPending = true;
@@ -907,7 +1086,7 @@ export class RuntimeFallbackService {
             if (this.switching.has(sessionName))
                 continue;
             if (!this.deps.isLive(sessionName)) {
-                this.finishRevertOffline(sessionName, override);
+                await this.finishRevertOffline(sessionName, override);
                 continue;
             }
             if (await this.deps.isBusy(sessionName).catch(() => true))
@@ -915,20 +1094,100 @@ export class RuntimeFallbackService {
             await this.revertSession(sessionName);
         }
     }
+    /**
+     * Move an idle, running agent to a better entry of its chain. A stopped
+     * agent is left alone: its next start picks the best target.
+     *
+     * @param sessionName - Session
+     * @param override - Its override (the configured runtime is out)
+     */
+    async upgradeIfBetter(sessionName, override) {
+        if (this.switching.has(sessionName) || this.isExempt(sessionName) || !this.deps.isLive(sessionName))
+            return;
+        // Cheap filter before reading the agent: some other chain entry must be open.
+        const { chain, memberChains } = this.state.settings;
+        const open = [...chain, ...Object.values(memberChains).flat()].some((r) => r !== override.runtime && r !== override.primary && !this.state.exhausted[r]);
+        if (!open)
+            return;
+        const agent = await this.deps.getAgent(sessionName).catch(() => null);
+        if (!agent || agent.primary !== override.primary)
+            return;
+        if (agent.isOrchestrator && !this.state.settings.orcFollows)
+            return;
+        const better = await this.betterTarget(agent, override.runtime, false);
+        if (!better)
+            return;
+        if (await this.deps.isBusy(sessionName).catch(() => true))
+            return;
+        if (this.switching.has(sessionName))
+            return;
+        this.switching.add(sessionName);
+        let outcome = 'skipped';
+        try {
+            outcome = await this.doUpgrade(agent, override.runtime, better);
+        }
+        catch (err) {
+            this.logger.warn('Moving an agent to a better fallback runtime failed', { sessionName, error: err instanceof Error ? err.message : String(err) });
+            outcome = 'failed';
+        }
+        finally {
+            this.switching.delete(sessionName);
+            await this.afterSwitch(sessionName, outcome, false);
+        }
+    }
+    /**
+     * Hand over and relaunch an idle agent on a better target.
+     *
+     * @param agent - Agent
+     * @param from - Target it is on
+     * @param target - Better target
+     * @returns Outcome
+     */
+    async doUpgrade(agent, from, target) {
+        const sessionName = agent.sessionName;
+        // Last look right before: anything that started since means "not now".
+        if (await this.deps.isBusy(sessionName).catch(() => true))
+            return 'skipped';
+        const handover = await this.deps
+            .writeHandover({
+            sessionName,
+            from,
+            to: target,
+            direction: 'upgrade',
+            conversationId: this.deps.conversation.get(sessionName) ?? null,
+            workItem: await this.deps.getActiveWorkItem(sessionName).catch(() => null),
+        })
+            .catch(() => null);
+        await this.recordSwitch(agent, from, target, handover, 'upgrade');
+        this.logger.info('Moving an idle agent to a better fallback runtime (its own runtime is still out)', { sessionName, from, to: target, primary: agent.primary });
+        const ok = await this.deps.relaunch(agent).catch(() => false);
+        if (!ok) {
+            this.logger.warn('Relaunch on the better fallback runtime failed (the override stays; the next start uses it)', { sessionName, runtime: target });
+            return 'failed';
+        }
+        return 'switched';
+    }
     /**
      * Clear the override of an agent that is not running.
      *
      * @param sessionName - Session
      * @param override - Its override
      */
-    finishRevertOffline(sessionName, override) {
+    async finishRevertOffline(sessionName, override) {
         delete this.state.overrides[sessionName];
-        if (override.primarySessionId)
+        const currentId = this.deps.conversation.get(sessionName);
+        // The conversation that went on on the fallback account comes back with
+        // the agent when both sides are Claude Code (crewly#1344); otherwise the
+        // primary's own conversation from before the switch is resumed.
+        const carried = currentId ? await this.carryConversation(sessionName, override.runtime, override.primary, currentId) : false;
+        if (carried)
+            this.deps.conversation.set(sessionName, currentId);
+        else if (override.primarySessionId)
             this.deps.conversation.set(sessionName, override.primarySessionId);
         else
             this.deps.conversation.clear(sessionName);
         this.save();
-        this.logger.info('Fallback ended for a stopped agent (its next start uses its own runtime)', { sessionName, runtime: override.primary });
+        this.logger.info('Fallback ended for a stopped agent (its next start uses its own runtime)', { sessionName, runtime: override.primary, carried });
     }
     /**
      * Move an idle agent back to its configured runtime.
@@ -973,12 +1232,18 @@ export class RuntimeFallbackService {
             })
                 .catch(() => null);
             delete this.state.overrides[sessionName];
-            if (override.primarySessionId)
+            const currentId = this.deps.conversation.get(sessionName);
+            // Both sides Claude Code: the conversation that went on while on the
+            // fallback account comes back with the agent (crewly#1344).
+            const carried = currentId ? await this.carryConversation(sessionName, override.runtime, override.primary, currentId) : false;
+            if (carried)
+                this.deps.conversation.set(sessionName, currentId);
+            else if (override.primarySessionId)
                 this.deps.conversation.set(sessionName, override.primarySessionId);
             else
                 this.deps.conversation.clear(sessionName);
             this.save();
-            this.kickoffNotes.set(sessionName, this.revertNote(override, handover, workItem));
+            this.kickoffNotes.set(sessionName, this.revertNote(override, handover, workItem, carried));
             this.logger.info('Switching agent back to its own runtime at an idle boundary', { sessionName, from: override.runtime, to: override.primary });
             const ok = await this.deps.relaunch(agent).catch(() => false);
             if (!ok) {
@@ -1137,7 +1402,15 @@ export class RuntimeFallbackService {
             ...(tz ? { timeZone: tz } : {}),
         }).format(new Date(epochMs));
     }
-    switchNote(from, to, until, handover, workItem) {
+    /** The second sentence of a kickoff note: whether the conversation came along. */
+    continuity(handover, carried) {
+        if (carried)
+            return 'Your conversation came with you — carry on where you were; nothing was lost.';
+        return handover
+            ? `This is a fresh conversation: after registering, read ${handover} once — it holds the end of your previous conversation.`
+            : 'This is a fresh conversation: your tasks, teams and wiki are all still in Crewly.';
+    }
+    switchNote(from, to, until, handover, workItem, carried = false) {
         const reset = until ? ` (it resets ~${this.formatTime(Date.parse(until))})` : '';
         const kind = this.state.exhausted[from]?.kind;
         const parts = [
@@ -1148,20 +1421,29 @@ export class RuntimeFallbackService {
                     : kind === 'login'
                         ? `Crewly moved you from ${this.label(from)} to ${this.label(to)} because the ${this.label(from)} login expired.`
                         : `Crewly moved you from ${this.label(from)} to ${this.label(to)} because ${this.label(from)} hit its usage limit${reset}; you will be moved back when it resets.`,
-            handover
-                ? `This is a fresh conversation: after registering, read ${handover} once — it holds the end of your previous conversation.`
-                : 'This is a fresh conversation: your tasks, teams and wiki are all still in Crewly.',
+            this.continuity(handover, carried),
+        ];
+        if (workItem)
+            parts.push(`Then continue WorkItem ${workItem.id} ("${workItem.title}") where you left off.`);
+        return parts.join(' ');
+    }
+    upgradeNote(primary, from, to, handover, workItem, carried = false) {
+        const parts = [
+            `${this.label(to)} has usage again, so Crewly moved you there from ${this.label(from)}; ${this.label(primary)} is still unavailable and you will be moved back to it once it is.`,
+            this.continuity(handover, carried),
         ];
         if (workItem)
             parts.push(`Then continue WorkItem ${workItem.id} ("${workItem.title}") where you left off.`);
         return parts.join(' ');
     }
-    revertNote(override, handover, workItem) {
+    revertNote(override, handover, workItem, carried = false) {
         const parts = [
             `${this.label(override.primary)} is available again, so Crewly moved you back from ${this.label(override.runtime)}.`,
-            handover
-                ? `Work went on while you were away: read ${handover} once for what happened since.`
-                : 'Work went on while you were away: check your WorkItems and wiki for what happened since.',
+            carried
+                ? 'Your conversation came back with you — carry on where you were.'
+                : handover
+                    ? `Work went on while you were away: read ${handover} once for what happened since.`
+                    : 'Work went on while you were away: check your WorkItems and wiki for what happened since.',
         ];
         if (workItem)
             parts.push(`Then continue WorkItem ${workItem.id} ("${workItem.title}").`);