crewly 1.20.355 → 1.20.356
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/roles/team-leader/prompt.md +1 -1
- package/config/roles/team-leader/tl-addon.md +1 -1
- package/config/skills/team-leader/delegate-task/SKILL.md +23 -1
- package/config/skills/team-leader/delegate-task/execute.sh +44 -1
- package/config/skills/team-leader/delegate-task/execute.test.sh +95 -1
- package/dist/backend/backend/src/constants.d.ts +65 -0
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +65 -0
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/controllers/agent-hooks/agent-hooks.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/agent-hooks/agent-hooks.controller.js +10 -0
- package/dist/backend/backend/src/controllers/agent-hooks/agent-hooks.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/orchestrator/orchestrator.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/orchestrator/orchestrator.controller.js +7 -0
- package/dist/backend/backend/src/controllers/orchestrator/orchestrator.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/team/team.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/team/team.controller.js +5 -0
- package/dist/backend/backend/src/controllers/team/team.controller.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts +15 -0
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +68 -11
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts +11 -0
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.js +73 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.d.ts +9 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js +60 -4
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/resource-mode-protection.d.ts +22 -0
- package/dist/backend/backend/src/services/agent/resource-mode-protection.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/resource-mode-protection.js +33 -0
- package/dist/backend/backend/src/services/agent/resource-mode-protection.js.map +1 -1
- package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts +50 -3
- package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/resource-mode.service.js +92 -10
- package/dist/backend/backend/src/services/agent/resource-mode.service.js.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-relogin.service.d.ts +23 -0
- package/dist/backend/backend/src/services/harness/harness-relogin.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-relogin.service.js +47 -6
- package/dist/backend/backend/src/services/harness/harness-relogin.service.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts +14 -2
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js +21 -4
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.js +10 -1
- package/dist/backend/backend/src/services/messaging/queue-processor.service.js.map +1 -1
- package/dist/backend/backend/src/services/open-items/open-items.service.d.ts +8 -0
- package/dist/backend/backend/src/services/open-items/open-items.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/open-items/open-items.service.js +33 -0
- package/dist/backend/backend/src/services/open-items/open-items.service.js.map +1 -1
- package/dist/backend/backend/src/services/open-items/open-items.wiring.js +2 -2
- package/dist/backend/backend/src/services/open-items/open-items.wiring.js.map +1 -1
- package/dist/backend/backend/src/services/open-items/question-card-gate.d.ts +49 -0
- package/dist/backend/backend/src/services/open-items/question-card-gate.d.ts.map +1 -0
- package/dist/backend/backend/src/services/open-items/question-card-gate.js +77 -0
- package/dist/backend/backend/src/services/open-items/question-card-gate.js.map +1 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.js +10 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-heartbeat-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.d.ts +2 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.js +17 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-restart.service.js.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.d.ts +7 -0
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.js +22 -10
- package/dist/backend/backend/src/services/orchestrator/orchestrator-status.service.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.d.ts +16 -0
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.d.ts.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.js +23 -0
- package/dist/backend/backend/src/services/runtime-fallback/effective-runtime.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.d.ts +2 -1
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.d.ts.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.js +3 -2
- package/dist/backend/backend/src/services/runtime-fallback/reset-time.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.d.ts +101 -5
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.js +247 -19
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.d.ts.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.js +10 -2
- package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.wiring.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.d.ts +78 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.d.ts.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.js +159 -1
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-rules.js.map +1 -1
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-screen.d.ts +63 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-screen.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-screen.js +104 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-screen.js.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-state.d.ts +166 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-state.d.ts.map +1 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-state.js +207 -0
- package/dist/backend/backend/src/services/runtime-fallback/usage-limit-state.js.map +1 -0
- package/dist/backend/backend/src/services/v3/workitem-dispatch.subscriber.d.ts.map +1 -1
- package/dist/backend/backend/src/services/v3/workitem-dispatch.subscriber.js +13 -0
- package/dist/backend/backend/src/services/v3/workitem-dispatch.subscriber.js.map +1 -1
- package/dist/backend/backend/src/services/workflow/scheduler.service.d.ts +22 -0
- package/dist/backend/backend/src/services/workflow/scheduler.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/workflow/scheduler.service.js +35 -1
- package/dist/backend/backend/src/services/workflow/scheduler.service.js.map +1 -1
- package/dist/backend/backend/src/types/index.d.ts +12 -0
- package/dist/backend/backend/src/types/index.d.ts.map +1 -1
- package/dist/backend/backend/src/types/index.js.map +1 -1
- package/dist/backend/backend/src/types/team-template.types.d.ts +1 -0
- package/dist/backend/backend/src/types/team-template.types.d.ts.map +1 -1
- package/dist/backend/backend/src/types/team-template.types.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +65 -0
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +65 -0
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts +11 -0
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.js +73 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.d.ts +9 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js +60 -4
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts +50 -3
- package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/resource-mode.service.js +92 -10
- package/dist/cli/backend/src/services/agent/resource-mode.service.js.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-relogin.service.d.ts +23 -0
- package/dist/cli/backend/src/services/harness/harness-relogin.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-relogin.service.js +47 -6
- package/dist/cli/backend/src/services/harness/harness-relogin.service.js.map +1 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.js +10 -1
- package/dist/cli/backend/src/services/messaging/queue-processor.service.js.map +1 -1
- package/dist/cli/backend/src/services/open-items/open-items.service.d.ts +8 -0
- package/dist/cli/backend/src/services/open-items/open-items.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/open-items/open-items.service.js +33 -0
- package/dist/cli/backend/src/services/open-items/open-items.service.js.map +1 -1
- package/dist/cli/backend/src/services/open-items/question-card-gate.d.ts +49 -0
- package/dist/cli/backend/src/services/open-items/question-card-gate.d.ts.map +1 -0
- package/dist/cli/backend/src/services/open-items/question-card-gate.js +77 -0
- package/dist/cli/backend/src/services/open-items/question-card-gate.js.map +1 -0
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.d.ts +2 -0
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.js +17 -0
- package/dist/cli/backend/src/services/orchestrator/orchestrator-restart.service.js.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.d.ts +7 -0
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.js +22 -10
- package/dist/cli/backend/src/services/orchestrator/orchestrator-status.service.js.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.d.ts +16 -0
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.d.ts.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.js +23 -0
- package/dist/cli/backend/src/services/runtime-fallback/effective-runtime.js.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/reset-time.d.ts +2 -1
- package/dist/cli/backend/src/services/runtime-fallback/reset-time.d.ts.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/reset-time.js +3 -2
- package/dist/cli/backend/src/services/runtime-fallback/reset-time.js.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-rules.d.ts +78 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-rules.d.ts.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-rules.js +159 -1
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-rules.js.map +1 -1
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-screen.d.ts +63 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-screen.d.ts.map +1 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-screen.js +104 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-screen.js.map +1 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-state.d.ts +166 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-state.d.ts.map +1 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-state.js +207 -0
- package/dist/cli/backend/src/services/runtime-fallback/usage-limit-state.js.map +1 -0
- package/dist/cli/backend/src/types/index.d.ts +12 -0
- package/dist/cli/backend/src/types/index.d.ts.map +1 -1
- package/dist/cli/backend/src/types/index.js.map +1 -1
- package/dist/cli/backend/src/types/team-template.types.d.ts +1 -0
- package/dist/cli/backend/src/types/team-template.types.d.ts.map +1 -1
- package/dist/cli/backend/src/types/team-template.types.js.map +1 -1
- package/package.json +1 -1
|
@@ -24,6 +24,17 @@
|
|
|
24
24
|
* most every BILLING_PROBE_INTERVAL_MS. A switch-back that fails (the limit
|
|
25
25
|
* comes straight back) doubles the next probe interval.
|
|
26
26
|
* 7. **Tell the owner once** per event, and once when it is over.
|
|
27
|
+
* 8. **Prefer the better target.** While the configured runtime is still out,
|
|
28
|
+
* an agent on a fallback moves (at an idle boundary, or when it is next
|
|
29
|
+
* started) to the earliest available entry of its chain when that is
|
|
30
|
+
* earlier than the one it is on — orc on `claude-code@work` goes back to
|
|
31
|
+
* `claude-code` once the default login has usage, even though its own
|
|
32
|
+
* runtime (DeepSeek, out of credit) is not back.
|
|
33
|
+
* 9. **Stale marks are cleared** when the reset time passes and a probe
|
|
34
|
+
* answers, when the owner signs that login in again
|
|
35
|
+
* ({@link RuntimeFallbackService.onRuntimeLogin} /
|
|
36
|
+
* {@link RuntimeFallbackService.onAccountLogin}), or when another session
|
|
37
|
+
* on that target is seen working ({@link RuntimeFallbackService.reportWorking}).
|
|
27
38
|
*
|
|
28
39
|
* A chain entry may be one of the owner's other Claude Code accounts
|
|
29
40
|
* (`claude-code@work`, issue #942): it is a runtime target of its own — out
|
|
@@ -65,7 +76,12 @@ export interface HandoverRequest {
|
|
|
65
76
|
sessionName: string;
|
|
66
77
|
from: string;
|
|
67
78
|
to: string;
|
|
68
|
-
|
|
79
|
+
/**
|
|
80
|
+
* `switch`: `from` ran out; `revert`: back to the configured runtime;
|
|
81
|
+
* `upgrade`: `to` is earlier in the fallback order and has usage again
|
|
82
|
+
* (the configured runtime is still out)
|
|
83
|
+
*/
|
|
84
|
+
direction: 'switch' | 'revert' | 'upgrade';
|
|
69
85
|
/** Conversation id on `from`, when known */
|
|
70
86
|
conversationId: string | null;
|
|
71
87
|
/** WorkItem the agent is on */
|
|
@@ -169,6 +185,8 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
169
185
|
private readonly noticeDue;
|
|
170
186
|
/** Kickoff notes waiting for a session's next launch */
|
|
171
187
|
private readonly kickoffNotes;
|
|
188
|
+
/** Last working-evidence look per session (throttle) */
|
|
189
|
+
private readonly workingSeen;
|
|
172
190
|
/** Last switch-back per runtime, to recognise one that failed */
|
|
173
191
|
private readonly recentRecoveries;
|
|
174
192
|
private availability;
|
|
@@ -241,11 +259,46 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
241
259
|
*/
|
|
242
260
|
private reevaluateExhausted;
|
|
243
261
|
/**
|
|
244
|
-
*
|
|
262
|
+
* The owner signed a harness's default login in again (the re-login flow
|
|
263
|
+
* finished for it). A mark on that runtime — signed out, or out of usage:
|
|
264
|
+
* the new login may be another account, or the same one with usage bought
|
|
265
|
+
* — is probed now instead of at its reset time, and cleared when the
|
|
266
|
+
* probe answers.
|
|
245
267
|
*
|
|
246
|
-
* @param
|
|
268
|
+
* @param runtime - Runtime id (`claude-code`, `codex-cli`, …; the harness id)
|
|
269
|
+
* @returns Resolves when done; never rejects
|
|
270
|
+
*
|
|
271
|
+
* @example
|
|
272
|
+
* ```ts
|
|
273
|
+
* relogin.setHarnessLoginHandler((harnessId) => service.onRuntimeLogin(harnessId));
|
|
274
|
+
* ```
|
|
275
|
+
*/
|
|
276
|
+
onRuntimeLogin(runtime: string): Promise<void>;
|
|
277
|
+
/**
|
|
278
|
+
* A login of a runtime target was signed in again: if it is marked (signed
|
|
279
|
+
* out or out of usage), probe it and bring it back when it answers.
|
|
280
|
+
*
|
|
281
|
+
* @param target - Runtime target (`claude-code`, `claude-code@<account>`)
|
|
247
282
|
*/
|
|
248
|
-
private
|
|
283
|
+
private recheckAfterLogin;
|
|
284
|
+
/**
|
|
285
|
+
* A session's model just answered with a tool call (Claude Code
|
|
286
|
+
* PreToolUse hook). A model cannot do that while its account is out of
|
|
287
|
+
* usage, so a mark on the target the session runs on is stale: it is
|
|
288
|
+
* cleared (agents moved off it come back at their idle boundary). Marks
|
|
289
|
+
* newer than WORKING_EVIDENCE_GRACE_MS are kept (a turn already streaming
|
|
290
|
+
* when the limit hit proves nothing). Looked at most once a minute per
|
|
291
|
+
* session, and only while a Claude Code target is marked.
|
|
292
|
+
*
|
|
293
|
+
* @param sessionName - Session the hook came from
|
|
294
|
+
*/
|
|
295
|
+
reportWorking(sessionName: string): void;
|
|
296
|
+
/**
|
|
297
|
+
* Clear the mark on the target a working session runs on.
|
|
298
|
+
*
|
|
299
|
+
* @param sessionName - Session
|
|
300
|
+
*/
|
|
301
|
+
private onWorking;
|
|
249
302
|
/**
|
|
250
303
|
* Take the note for a session's next kickoff.
|
|
251
304
|
*
|
|
@@ -279,6 +332,13 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
279
332
|
* @returns The decision
|
|
280
333
|
*/
|
|
281
334
|
resolveLaunch(input: LaunchRuntimeInput): Promise<LaunchRuntimeDecision>;
|
|
335
|
+
/**
|
|
336
|
+
* The agent behind a launch (from the launch input when it is not known).
|
|
337
|
+
*
|
|
338
|
+
* @param input - Launch input
|
|
339
|
+
* @returns Agent
|
|
340
|
+
*/
|
|
341
|
+
private launchAgent;
|
|
282
342
|
/**
|
|
283
343
|
* The configured runtime of a launching agent is not installed / not signed
|
|
284
344
|
* in on this machine: record it like a signed-out account so the agent starts
|
|
@@ -358,6 +418,7 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
358
418
|
* @param from - Runtime it leaves (out of usage)
|
|
359
419
|
* @param target - Runtime it goes to
|
|
360
420
|
* @param handover - Handover file, when written
|
|
421
|
+
* @param direction - `switch` (`from` ran out) or `upgrade` (`target` ranks higher in the chain)
|
|
361
422
|
*/
|
|
362
423
|
private recordSwitch;
|
|
363
424
|
/**
|
|
@@ -382,6 +443,19 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
382
443
|
* @returns Chain
|
|
383
444
|
*/
|
|
384
445
|
private chainFor;
|
|
446
|
+
/**
|
|
447
|
+
* Where an agent whose configured runtime is out should be instead of
|
|
448
|
+
* `current`: the earliest available entry of its chain (never its
|
|
449
|
+
* configured runtime, never one marked out of usage), when that ranks
|
|
450
|
+
* before `current` — or, with `orAnyWhenCurrentOut`, any available entry
|
|
451
|
+
* when `current` itself is marked out.
|
|
452
|
+
*
|
|
453
|
+
* @param agent - Agent (its configured runtime is out)
|
|
454
|
+
* @param current - Runtime target it is on
|
|
455
|
+
* @param orAnyWhenCurrentOut - Also move off a `current` that ran out to a lower-ranked entry
|
|
456
|
+
* @returns Better target, or null to stay
|
|
457
|
+
*/
|
|
458
|
+
private betterTarget;
|
|
385
459
|
/**
|
|
386
460
|
* Whether some runtime could take over (synchronous; uses the cached
|
|
387
461
|
* availability, optimistic when nothing is cached yet).
|
|
@@ -428,8 +502,29 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
428
502
|
* @param runtime - Runtime
|
|
429
503
|
*/
|
|
430
504
|
recover(runtime: string): Promise<void>;
|
|
431
|
-
/**
|
|
505
|
+
/**
|
|
506
|
+
* Revert every pending override whose agent is idle (or not running), and
|
|
507
|
+
* move an idle agent whose configured runtime is still out to a better
|
|
508
|
+
* entry of its chain ({@link betterTarget}).
|
|
509
|
+
*/
|
|
432
510
|
private revertIdle;
|
|
511
|
+
/**
|
|
512
|
+
* Move an idle, running agent to a better entry of its chain. A stopped
|
|
513
|
+
* agent is left alone: its next start picks the best target.
|
|
514
|
+
*
|
|
515
|
+
* @param sessionName - Session
|
|
516
|
+
* @param override - Its override (the configured runtime is out)
|
|
517
|
+
*/
|
|
518
|
+
private upgradeIfBetter;
|
|
519
|
+
/**
|
|
520
|
+
* Hand over and relaunch an idle agent on a better target.
|
|
521
|
+
*
|
|
522
|
+
* @param agent - Agent
|
|
523
|
+
* @param from - Target it is on
|
|
524
|
+
* @param target - Better target
|
|
525
|
+
* @returns Outcome
|
|
526
|
+
*/
|
|
527
|
+
private doUpgrade;
|
|
433
528
|
/**
|
|
434
529
|
* Clear the override of an agent that is not running.
|
|
435
530
|
*
|
|
@@ -475,6 +570,7 @@ export declare class RuntimeFallbackService implements RuntimeFallbackHooks {
|
|
|
475
570
|
private notify;
|
|
476
571
|
private formatTime;
|
|
477
572
|
private switchNote;
|
|
573
|
+
private upgradeNote;
|
|
478
574
|
private revertNote;
|
|
479
575
|
/**
|
|
480
576
|
* Badge data for an agent running on a fallback.
|
package/dist/backend/backend/src/services/runtime-fallback/runtime-fallback.service.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-fallback.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/runtime-fallback/runtime-fallback.service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"runtime-fallback.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/runtime-fallback/runtime-fallback.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAOH,OAAO,KAAK,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AACnI,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAIN,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,uBAAuB,EAE5B,KAAK,eAAe,EACpB,MAAM,6BAA6B,CAAC;AAOrC;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,aAAa,CAAC;AAE9E,qDAAqD;AACrD,MAAM,WAAW,iBAAiB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,OAAO,CAAC;CACxB;AAED,yDAAyD;AACzD,MAAM,WAAW,eAAe;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC3C,4CAA4C;IAC5C,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,+BAA+B;IAC/B,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAC/C;AAED,sBAAsB;AACtB,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAChE;AAED,4BAA4B;AAC5B,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC5C;AAED,+BAA+B;AAC/B,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,oBAAoB,CAAC;IAC5B,4DAA4D;IAC5D,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC;IACrE,iHAAiH;IACjH,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5D,yDAAyD;IACzD,MAAM,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC;IACzC,yDAAyD;IACzD,MAAM,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,iDAAiD;IACjD,eAAe,EAAE,CAAC,QAAQ,EAAE,uBAAuB,KAAK,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACvF,0FAA0F;IAC1F,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD,gDAAgD;IAChD,aAAa,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChE,+BAA+B;IAC/B,iBAAiB,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAC1F,0CAA0C;IAC1C,YAAY,EAAE;QACb,GAAG,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;QACjD,GAAG,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;QAC/C,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;KACrC,CAAC;IACF,4EAA4E;IAC5E,QAAQ,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzD,6DAA6D;IAC7D,SAAS,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,gDAAgD;IAChD,WAAW,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,0BAA0B;IAC1B,WAAW,EAAE,MAAM,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,qBAAqB,GAAG,IAAI,CAAC;IAC9C,wDAAwD;IACxD,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC;IAC5C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,uCAAuC;AACvC,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,uBAAuB,CAAC;IAClC,QAAQ,EAAE,KAAK,CAAC,mBAAmB,GAAG;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC9D,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,SAAS,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,mCAAmC;AACnC,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,GAAG,QAAQ,CAAC;AAc3F,oCAAoC;AACpC,qBAAa,sBAAuB,YAAW,oBAAoB;IA4BtD,OAAO,CAAC,QAAQ,CAAC,IAAI;IA3BjC,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAClE,kEAAkE;IAClE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,+CAA+C;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA+B;IAC1D,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,wDAAwD;IACxD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAC1D,wDAAwD;IACxD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,iEAAiE;IACjE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA+E;IAChH,OAAO,CAAC,YAAY,CAA4D;IAChF,OAAO,CAAC,KAAK,CAA+B;IAC5C,OAAO,CAAC,OAAO,CAAS;IAExB;;OAEG;gBAC0B,IAAI,EAAE,mBAAmB;IAStD,sEAAsE;IACtE,KAAK,IAAI,IAAI;IAQb,qBAAqB;IACrB,IAAI,IAAI,IAAI;IAOZ,uCAAuC;IACvC,WAAW,IAAI,uBAAuB;IAItC;;;;;;OAMG;IACH,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,uBAAuB;IAOvD,0EAA0E;IAC1E,sBAAsB,IAAI,IAAI;IAM9B;;;;;OAKG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAK/C;;;;;OAKG;IACH,UAAU,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAK9C;;;;;;;;OAQG;IACH,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAU/C;;;;;;OAMG;IACG,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBpD;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW;IA8BnB;;;OAGG;YACW,mBAAmB;IAajC;;;;;;;;;;;;;;OAcG;IACG,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASpD;;;;;OAKG;YACW,iBAAiB;IAkB/B;;;;;;;;;;OAUG;IACH,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAYxC;;;;OAIG;YACW,SAAS;IAUvB;;;;;OAKG;IACH,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAMnD;;;;;;;;;OASG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,mBAAmB,GAAG,OAAO;IAiBtG;;;;;;OAMG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO;IAYzE;;;;;OAKG;IACG,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAoD9E;;;;;OAKG;YACW,WAAW;IAazB;;;;;;;OAOG;YACW,mBAAmB;IAoBjC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ;IAKhB;;;;;;OAMG;YACW,kBAAkB;IAyBhC;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IAazB;;;;;;OAMG;YACW,YAAY;IAqD1B;;;;;OAKG;IACH,OAAO,CAAC,OAAO;IAmBf;;;;;;OAMG;IACG,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,gBAAgB,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,CAAC;IAoB1H;;;;;;;;OAQG;YACW,WAAW;YAMX,QAAQ;IAkCtB;;;;;;;;;OASG;YACW,YAAY;IAwC1B;;;;OAIG;YACW,gBAAgB;IAS9B;;;;;;;OAOG;YACW,YAAY;IAU1B;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IAKhB;;;;;;;;;;;OAWG;YACW,YAAY;IAa1B;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IASxB;;;;;OAKG;IACG,eAAe,CAAC,KAAK,UAAQ,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAapE,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB3B;;;;OAIG;YACW,aAAa;IA+B3B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;;;OAIG;IACG,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAyB7C;;;;OAIG;YACW,UAAU;IAqBxB;;;;;;OAMG;YACW,eAAe;IA0B7B;;;;;;;OAOG;YACW,SAAS;IAwBvB;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAQ3B;;;;;OAKG;IACG,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;YAalD,QAAQ;IAoCtB,OAAO,CAAC,cAAc;IAQtB,4DAA4D;IACtD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YAwBrB,eAAe;IAkB7B;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IAgB3B;;;;;;OAMG;YACW,iBAAiB;IAmB/B;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;YAYF,MAAM;IAapB,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAmBlB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,UAAU;IAalB;;;;;OAKG;IACH,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAa7D;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAclD,OAAO,CAAC,KAAK;IAIb,OAAO,CAAC,QAAQ;IAUhB,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,IAAI;CAOZ;AAKD;;;;GAIG;AACH,wBAAgB,yBAAyB,IAAI,sBAAsB,GAAG,IAAI,CAEzE;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,sBAAsB,GAAG,IAAI,GAAG,IAAI,CAEtF"}
|
|
@@ -24,6 +24,17 @@
|
|
|
24
24
|
* most every BILLING_PROBE_INTERVAL_MS. A switch-back that fails (the limit
|
|
25
25
|
* comes straight back) doubles the next probe interval.
|
|
26
26
|
* 7. **Tell the owner once** per event, and once when it is over.
|
|
27
|
+
* 8. **Prefer the better target.** While the configured runtime is still out,
|
|
28
|
+
* an agent on a fallback moves (at an idle boundary, or when it is next
|
|
29
|
+
* started) to the earliest available entry of its chain when that is
|
|
30
|
+
* earlier than the one it is on — orc on `claude-code@work` goes back to
|
|
31
|
+
* `claude-code` once the default login has usage, even though its own
|
|
32
|
+
* runtime (DeepSeek, out of credit) is not back.
|
|
33
|
+
* 9. **Stale marks are cleared** when the reset time passes and a probe
|
|
34
|
+
* answers, when the owner signs that login in again
|
|
35
|
+
* ({@link RuntimeFallbackService.onRuntimeLogin} /
|
|
36
|
+
* {@link RuntimeFallbackService.onAccountLogin}), or when another session
|
|
37
|
+
* on that target is seen working ({@link RuntimeFallbackService.reportWorking}).
|
|
27
38
|
*
|
|
28
39
|
* A chain entry may be one of the owner's other Claude Code accounts
|
|
29
40
|
* (`claude-code@work`, issue #942): it is a runtime target of its own — out
|
|
@@ -41,6 +52,7 @@
|
|
|
41
52
|
*/
|
|
42
53
|
import { ORCHESTRATOR_SESSION_NAME, RUNTIME_FALLBACK_CONSTANTS, RUNTIME_TYPES } from '../../constants.js';
|
|
43
54
|
import { detectUsageLimit } from './usage-limit-rules.js';
|
|
55
|
+
import { clearUsageLimit } from './usage-limit-state.js';
|
|
44
56
|
import { traceRuntimeBlocked } from '../trace/trace-recorder.js';
|
|
45
57
|
import { accountOf, baseRuntimeOf, parseRuntimeTarget, runtimeTarget } from '../harness/claude-accounts.js';
|
|
46
58
|
import { applySettingsPatch, runtimeLabel, runtimeShortLabel, } from './runtime-fallback.types.js';
|
|
@@ -76,6 +88,8 @@ export class RuntimeFallbackService {
|
|
|
76
88
|
noticeDue = new Map();
|
|
77
89
|
/** Kickoff notes waiting for a session's next launch */
|
|
78
90
|
kickoffNotes = new Map();
|
|
91
|
+
/** Last working-evidence look per session (throttle) */
|
|
92
|
+
workingSeen = new Map();
|
|
79
93
|
/** Last switch-back per runtime, to recognise one that failed */
|
|
80
94
|
recentRecoveries = new Map();
|
|
81
95
|
availability = null;
|
|
@@ -179,7 +193,7 @@ export class RuntimeFallbackService {
|
|
|
179
193
|
this.availability = null;
|
|
180
194
|
const target = runtimeTarget(RUNTIME_TYPES.CLAUDE_CODE, account);
|
|
181
195
|
try {
|
|
182
|
-
await this.
|
|
196
|
+
await this.recheckAfterLogin(target);
|
|
183
197
|
}
|
|
184
198
|
finally {
|
|
185
199
|
// The owner is away: a freshly signed-in account must be usable at once,
|
|
@@ -254,27 +268,94 @@ export class RuntimeFallbackService {
|
|
|
254
268
|
}
|
|
255
269
|
}
|
|
256
270
|
/**
|
|
257
|
-
*
|
|
271
|
+
* The owner signed a harness's default login in again (the re-login flow
|
|
272
|
+
* finished for it). A mark on that runtime — signed out, or out of usage:
|
|
273
|
+
* the new login may be another account, or the same one with usage bought
|
|
274
|
+
* — is probed now instead of at its reset time, and cleared when the
|
|
275
|
+
* probe answers.
|
|
258
276
|
*
|
|
259
|
-
* @param
|
|
277
|
+
* @param runtime - Runtime id (`claude-code`, `codex-cli`, …; the harness id)
|
|
278
|
+
* @returns Resolves when done; never rejects
|
|
279
|
+
*
|
|
280
|
+
* @example
|
|
281
|
+
* ```ts
|
|
282
|
+
* relogin.setHarnessLoginHandler((harnessId) => service.onRuntimeLogin(harnessId));
|
|
283
|
+
* ```
|
|
260
284
|
*/
|
|
261
|
-
async
|
|
285
|
+
async onRuntimeLogin(runtime) {
|
|
286
|
+
this.availability = null;
|
|
287
|
+
try {
|
|
288
|
+
await this.recheckAfterLogin(runtime);
|
|
289
|
+
}
|
|
290
|
+
catch (err) {
|
|
291
|
+
this.logger.warn('Re-checking a runtime after a sign-in failed', { runtime, error: err instanceof Error ? err.message : String(err) });
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* A login of a runtime target was signed in again: if it is marked (signed
|
|
296
|
+
* out or out of usage), probe it and bring it back when it answers.
|
|
297
|
+
*
|
|
298
|
+
* @param target - Runtime target (`claude-code`, `claude-code@<account>`)
|
|
299
|
+
*/
|
|
300
|
+
async recheckAfterLogin(target) {
|
|
262
301
|
const entry = this.state.exhausted[target];
|
|
263
|
-
if (!entry
|
|
302
|
+
if (!entry)
|
|
264
303
|
return;
|
|
265
304
|
const result = await this.deps.probe(target).catch(() => 'unknown');
|
|
266
305
|
entry.lastProbeAt = new Date(this.now()).toISOString();
|
|
267
306
|
if (result === 'available') {
|
|
307
|
+
this.logger.info('Signed in again and the probe answers — clearing the out-of-usage mark', { runtime: target, kind: entry.kind ?? 'usage_limit' });
|
|
268
308
|
await this.recover(target);
|
|
269
309
|
return;
|
|
270
310
|
}
|
|
271
|
-
if (result === 'limited') {
|
|
311
|
+
if (result === 'limited' && entry.kind === 'login') {
|
|
272
312
|
// Signed in again, but that account is out of usage too.
|
|
273
313
|
entry.kind = 'usage_limit';
|
|
274
314
|
entry.ruleId = 'probe_limited';
|
|
275
315
|
}
|
|
276
316
|
this.save();
|
|
277
317
|
}
|
|
318
|
+
/**
|
|
319
|
+
* A session's model just answered with a tool call (Claude Code
|
|
320
|
+
* PreToolUse hook). A model cannot do that while its account is out of
|
|
321
|
+
* usage, so a mark on the target the session runs on is stale: it is
|
|
322
|
+
* cleared (agents moved off it come back at their idle boundary). Marks
|
|
323
|
+
* newer than WORKING_EVIDENCE_GRACE_MS are kept (a turn already streaming
|
|
324
|
+
* when the limit hit proves nothing). Looked at most once a minute per
|
|
325
|
+
* session, and only while a Claude Code target is marked.
|
|
326
|
+
*
|
|
327
|
+
* @param sessionName - Session the hook came from
|
|
328
|
+
*/
|
|
329
|
+
reportWorking(sessionName) {
|
|
330
|
+
if (!this.state.settings.enabled || this.isExempt(sessionName))
|
|
331
|
+
return;
|
|
332
|
+
if (!Object.keys(this.state.exhausted).some((r) => baseRuntimeOf(r) === RUNTIME_TYPES.CLAUDE_CODE))
|
|
333
|
+
return;
|
|
334
|
+
if (this.switching.has(sessionName))
|
|
335
|
+
return;
|
|
336
|
+
const now = this.now();
|
|
337
|
+
if (now - (this.workingSeen.get(sessionName) ?? Number.NEGATIVE_INFINITY) < C.WORKING_EVIDENCE_THROTTLE_MS)
|
|
338
|
+
return;
|
|
339
|
+
this.workingSeen.set(sessionName, now);
|
|
340
|
+
void this.onWorking(sessionName).catch((err) => this.logger.warn('Runtime fallback could not use a working signal', { sessionName, error: err instanceof Error ? err.message : String(err) }));
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Clear the mark on the target a working session runs on.
|
|
344
|
+
*
|
|
345
|
+
* @param sessionName - Session
|
|
346
|
+
*/
|
|
347
|
+
async onWorking(sessionName) {
|
|
348
|
+
const target = this.state.overrides[sessionName]?.runtime ?? (await this.deps.getAgent(sessionName).catch(() => null))?.primary;
|
|
349
|
+
if (!target || baseRuntimeOf(target) !== RUNTIME_TYPES.CLAUDE_CODE)
|
|
350
|
+
return;
|
|
351
|
+
const entry = this.state.exhausted[target];
|
|
352
|
+
if (!entry)
|
|
353
|
+
return;
|
|
354
|
+
if (this.now() - Date.parse(entry.since) < C.WORKING_EVIDENCE_GRACE_MS)
|
|
355
|
+
return;
|
|
356
|
+
this.logger.info('A session is working on a runtime marked out of usage — clearing the mark', { runtime: target, sessionName, since: entry.since });
|
|
357
|
+
await this.recover(target);
|
|
358
|
+
}
|
|
278
359
|
/**
|
|
279
360
|
* Take the note for a session's next kickoff.
|
|
280
361
|
*
|
|
@@ -359,7 +440,20 @@ export class RuntimeFallbackService {
|
|
|
359
440
|
this.finishRevertOffline(input.sessionName, existing);
|
|
360
441
|
return configured;
|
|
361
442
|
}
|
|
443
|
+
else if (!this.state.exhausted[existing.primary]) {
|
|
444
|
+
// Its runtime is back but not yet marked for revert: the next tick reverts it.
|
|
445
|
+
return this.decision(existing.runtime);
|
|
446
|
+
}
|
|
362
447
|
else {
|
|
448
|
+
// Starting anyway: on the best target of its chain, not on one that ran
|
|
449
|
+
// out meanwhile or below one that came back.
|
|
450
|
+
const agent = await this.launchAgent(input);
|
|
451
|
+
const better = await this.betterTarget(agent, existing.runtime, true);
|
|
452
|
+
if (better) {
|
|
453
|
+
await this.recordSwitch(agent, existing.runtime, better, null, this.state.exhausted[existing.runtime] ? 'switch' : 'upgrade');
|
|
454
|
+
this.logger.info('Starting an agent on a better fallback runtime', { sessionName: input.sessionName, from: existing.runtime, runtime: better });
|
|
455
|
+
return this.decision(better);
|
|
456
|
+
}
|
|
363
457
|
return this.decision(existing.runtime);
|
|
364
458
|
}
|
|
365
459
|
}
|
|
@@ -373,14 +467,7 @@ export class RuntimeFallbackService {
|
|
|
373
467
|
}
|
|
374
468
|
if (input.isOrchestrator && !this.state.settings.orcFollows)
|
|
375
469
|
return configured;
|
|
376
|
-
const agent =
|
|
377
|
-
sessionName: input.sessionName,
|
|
378
|
-
name: input.sessionName,
|
|
379
|
-
primary: input.configured,
|
|
380
|
-
memberId: input.memberId,
|
|
381
|
-
teamId: input.teamId,
|
|
382
|
-
isOrchestrator: input.isOrchestrator,
|
|
383
|
-
};
|
|
470
|
+
const agent = await this.launchAgent(input);
|
|
384
471
|
const target = await this.pickFallback(agent, input.configured);
|
|
385
472
|
if (!target) {
|
|
386
473
|
this.markNoFallback(input.configured);
|
|
@@ -394,6 +481,22 @@ export class RuntimeFallbackService {
|
|
|
394
481
|
});
|
|
395
482
|
return this.decision(target);
|
|
396
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* The agent behind a launch (from the launch input when it is not known).
|
|
486
|
+
*
|
|
487
|
+
* @param input - Launch input
|
|
488
|
+
* @returns Agent
|
|
489
|
+
*/
|
|
490
|
+
async launchAgent(input) {
|
|
491
|
+
return ((await this.deps.getAgent(input.sessionName).catch(() => null)) ?? {
|
|
492
|
+
sessionName: input.sessionName,
|
|
493
|
+
name: input.sessionName,
|
|
494
|
+
primary: input.configured,
|
|
495
|
+
memberId: input.memberId,
|
|
496
|
+
teamId: input.teamId,
|
|
497
|
+
isOrchestrator: input.isOrchestrator,
|
|
498
|
+
});
|
|
499
|
+
}
|
|
397
500
|
/**
|
|
398
501
|
* The configured runtime of a launching agent is not installed / not signed
|
|
399
502
|
* in on this machine: record it like a signed-out account so the agent starts
|
|
@@ -510,8 +613,11 @@ export class RuntimeFallbackService {
|
|
|
510
613
|
}
|
|
511
614
|
else {
|
|
512
615
|
const confirmed = await this.confirm(runtime);
|
|
513
|
-
if (!confirmed)
|
|
616
|
+
if (!confirmed) {
|
|
617
|
+
// The probe answered normally: the session is not out of usage either.
|
|
618
|
+
clearUsageLimit(sessionName);
|
|
514
619
|
return;
|
|
620
|
+
}
|
|
515
621
|
if (!this.state.exhausted[runtime]) {
|
|
516
622
|
// The limit came straight back after a switch-back: back off the
|
|
517
623
|
// next probe, and do not tell the owner a second time.
|
|
@@ -590,6 +696,10 @@ export class RuntimeFallbackService {
|
|
|
590
696
|
}
|
|
591
697
|
finally {
|
|
592
698
|
this.switching.delete(sessionName);
|
|
699
|
+
// On another runtime now (or back on one that has usage again): the
|
|
700
|
+
// session's own usage-limit state no longer holds.
|
|
701
|
+
if (outcome === 'switched' || outcome === 'reverted')
|
|
702
|
+
clearUsageLimit(sessionName);
|
|
593
703
|
await this.afterSwitch(sessionName, outcome, opts.flushAfter);
|
|
594
704
|
}
|
|
595
705
|
}
|
|
@@ -653,8 +763,9 @@ export class RuntimeFallbackService {
|
|
|
653
763
|
* @param from - Runtime it leaves (out of usage)
|
|
654
764
|
* @param target - Runtime it goes to
|
|
655
765
|
* @param handover - Handover file, when written
|
|
766
|
+
* @param direction - `switch` (`from` ran out) or `upgrade` (`target` ranks higher in the chain)
|
|
656
767
|
*/
|
|
657
|
-
async recordSwitch(agent, from, target, handover = null) {
|
|
768
|
+
async recordSwitch(agent, from, target, handover = null, direction = 'switch') {
|
|
658
769
|
const sessionName = agent.sessionName;
|
|
659
770
|
const existing = this.state.overrides[sessionName];
|
|
660
771
|
const exhausted = this.state.exhausted[from];
|
|
@@ -685,7 +796,7 @@ export class RuntimeFallbackService {
|
|
|
685
796
|
}
|
|
686
797
|
this.save();
|
|
687
798
|
const workItem = await this.deps.getActiveWorkItem(sessionName).catch(() => null);
|
|
688
|
-
this.kickoffNotes.set(sessionName, this.switchNote(from, target, exhausted?.until, handover, workItem));
|
|
799
|
+
this.kickoffNotes.set(sessionName, direction === 'upgrade' ? this.upgradeNote(agent.primary, from, target, handover, workItem) : this.switchNote(from, target, exhausted?.until, handover, workItem));
|
|
689
800
|
this.scheduleNotice(from);
|
|
690
801
|
}
|
|
691
802
|
/**
|
|
@@ -731,6 +842,33 @@ export class RuntimeFallbackService {
|
|
|
731
842
|
const own = agent.memberId ? this.state.settings.memberChains[agent.memberId] : undefined;
|
|
732
843
|
return own && own.length > 0 ? own : this.state.settings.chain;
|
|
733
844
|
}
|
|
845
|
+
/**
|
|
846
|
+
* Where an agent whose configured runtime is out should be instead of
|
|
847
|
+
* `current`: the earliest available entry of its chain (never its
|
|
848
|
+
* configured runtime, never one marked out of usage), when that ranks
|
|
849
|
+
* before `current` — or, with `orAnyWhenCurrentOut`, any available entry
|
|
850
|
+
* when `current` itself is marked out.
|
|
851
|
+
*
|
|
852
|
+
* @param agent - Agent (its configured runtime is out)
|
|
853
|
+
* @param current - Runtime target it is on
|
|
854
|
+
* @param orAnyWhenCurrentOut - Also move off a `current` that ran out to a lower-ranked entry
|
|
855
|
+
* @returns Better target, or null to stay
|
|
856
|
+
*/
|
|
857
|
+
async betterTarget(agent, current, orAnyWhenCurrentOut) {
|
|
858
|
+
const best = await this.pickFallback(agent, agent.primary);
|
|
859
|
+
if (!best || best === current)
|
|
860
|
+
return null;
|
|
861
|
+
const chain = this.chainFor(agent);
|
|
862
|
+
const rank = (r) => {
|
|
863
|
+
const i = chain.indexOf(r);
|
|
864
|
+
return i < 0 ? Number.POSITIVE_INFINITY : i;
|
|
865
|
+
};
|
|
866
|
+
if (rank(best) < rank(current))
|
|
867
|
+
return best;
|
|
868
|
+
if (orAnyWhenCurrentOut && this.state.exhausted[current])
|
|
869
|
+
return best;
|
|
870
|
+
return null;
|
|
871
|
+
}
|
|
734
872
|
/**
|
|
735
873
|
* Whether some runtime could take over (synchronous; uses the cached
|
|
736
874
|
* availability, optimistic when nothing is cached yet).
|
|
@@ -894,11 +1032,17 @@ export class RuntimeFallbackService {
|
|
|
894
1032
|
}
|
|
895
1033
|
await this.revertIdle();
|
|
896
1034
|
}
|
|
897
|
-
/**
|
|
1035
|
+
/**
|
|
1036
|
+
* Revert every pending override whose agent is idle (or not running), and
|
|
1037
|
+
* move an idle agent whose configured runtime is still out to a better
|
|
1038
|
+
* entry of its chain ({@link betterTarget}).
|
|
1039
|
+
*/
|
|
898
1040
|
async revertIdle() {
|
|
899
1041
|
for (const [sessionName, override] of Object.entries(this.state.overrides)) {
|
|
900
|
-
if (this.state.exhausted[override.primary])
|
|
1042
|
+
if (this.state.exhausted[override.primary]) {
|
|
1043
|
+
await this.upgradeIfBetter(sessionName, override);
|
|
901
1044
|
continue;
|
|
1045
|
+
}
|
|
902
1046
|
if (!override.revertPending) {
|
|
903
1047
|
// State from before a restart whose primary is no longer marked out of usage.
|
|
904
1048
|
override.revertPending = true;
|
|
@@ -915,6 +1059,79 @@ export class RuntimeFallbackService {
|
|
|
915
1059
|
await this.revertSession(sessionName);
|
|
916
1060
|
}
|
|
917
1061
|
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Move an idle, running agent to a better entry of its chain. A stopped
|
|
1064
|
+
* agent is left alone: its next start picks the best target.
|
|
1065
|
+
*
|
|
1066
|
+
* @param sessionName - Session
|
|
1067
|
+
* @param override - Its override (the configured runtime is out)
|
|
1068
|
+
*/
|
|
1069
|
+
async upgradeIfBetter(sessionName, override) {
|
|
1070
|
+
if (this.switching.has(sessionName) || this.isExempt(sessionName) || !this.deps.isLive(sessionName))
|
|
1071
|
+
return;
|
|
1072
|
+
// Cheap filter before reading the agent: some other chain entry must be open.
|
|
1073
|
+
const { chain, memberChains } = this.state.settings;
|
|
1074
|
+
const open = [...chain, ...Object.values(memberChains).flat()].some((r) => r !== override.runtime && r !== override.primary && !this.state.exhausted[r]);
|
|
1075
|
+
if (!open)
|
|
1076
|
+
return;
|
|
1077
|
+
const agent = await this.deps.getAgent(sessionName).catch(() => null);
|
|
1078
|
+
if (!agent || agent.primary !== override.primary)
|
|
1079
|
+
return;
|
|
1080
|
+
if (agent.isOrchestrator && !this.state.settings.orcFollows)
|
|
1081
|
+
return;
|
|
1082
|
+
const better = await this.betterTarget(agent, override.runtime, false);
|
|
1083
|
+
if (!better)
|
|
1084
|
+
return;
|
|
1085
|
+
if (await this.deps.isBusy(sessionName).catch(() => true))
|
|
1086
|
+
return;
|
|
1087
|
+
if (this.switching.has(sessionName))
|
|
1088
|
+
return;
|
|
1089
|
+
this.switching.add(sessionName);
|
|
1090
|
+
let outcome = 'skipped';
|
|
1091
|
+
try {
|
|
1092
|
+
outcome = await this.doUpgrade(agent, override.runtime, better);
|
|
1093
|
+
}
|
|
1094
|
+
catch (err) {
|
|
1095
|
+
this.logger.warn('Moving an agent to a better fallback runtime failed', { sessionName, error: err instanceof Error ? err.message : String(err) });
|
|
1096
|
+
outcome = 'failed';
|
|
1097
|
+
}
|
|
1098
|
+
finally {
|
|
1099
|
+
this.switching.delete(sessionName);
|
|
1100
|
+
await this.afterSwitch(sessionName, outcome, false);
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
/**
|
|
1104
|
+
* Hand over and relaunch an idle agent on a better target.
|
|
1105
|
+
*
|
|
1106
|
+
* @param agent - Agent
|
|
1107
|
+
* @param from - Target it is on
|
|
1108
|
+
* @param target - Better target
|
|
1109
|
+
* @returns Outcome
|
|
1110
|
+
*/
|
|
1111
|
+
async doUpgrade(agent, from, target) {
|
|
1112
|
+
const sessionName = agent.sessionName;
|
|
1113
|
+
// Last look right before: anything that started since means "not now".
|
|
1114
|
+
if (await this.deps.isBusy(sessionName).catch(() => true))
|
|
1115
|
+
return 'skipped';
|
|
1116
|
+
const handover = await this.deps
|
|
1117
|
+
.writeHandover({
|
|
1118
|
+
sessionName,
|
|
1119
|
+
from,
|
|
1120
|
+
to: target,
|
|
1121
|
+
direction: 'upgrade',
|
|
1122
|
+
conversationId: this.deps.conversation.get(sessionName) ?? null,
|
|
1123
|
+
workItem: await this.deps.getActiveWorkItem(sessionName).catch(() => null),
|
|
1124
|
+
})
|
|
1125
|
+
.catch(() => null);
|
|
1126
|
+
await this.recordSwitch(agent, from, target, handover, 'upgrade');
|
|
1127
|
+
this.logger.info('Moving an idle agent to a better fallback runtime (its own runtime is still out)', { sessionName, from, to: target, primary: agent.primary });
|
|
1128
|
+
const ok = await this.deps.relaunch(agent).catch(() => false);
|
|
1129
|
+
if (!ok) {
|
|
1130
|
+
this.logger.warn('Relaunch on the better fallback runtime failed (the override stays; the next start uses it)', { sessionName, runtime: target });
|
|
1131
|
+
return 'failed';
|
|
1132
|
+
}
|
|
1133
|
+
return 'switched';
|
|
1134
|
+
}
|
|
918
1135
|
/**
|
|
919
1136
|
* Clear the override of an agent that is not running.
|
|
920
1137
|
*
|
|
@@ -1156,6 +1373,17 @@ export class RuntimeFallbackService {
|
|
|
1156
1373
|
parts.push(`Then continue WorkItem ${workItem.id} ("${workItem.title}") where you left off.`);
|
|
1157
1374
|
return parts.join(' ');
|
|
1158
1375
|
}
|
|
1376
|
+
upgradeNote(primary, from, to, handover, workItem) {
|
|
1377
|
+
const parts = [
|
|
1378
|
+
`${this.label(to)} has usage again, so Crewly moved you there from ${this.label(from)}; ${this.label(primary)} is still unavailable and you will be moved back to it once it is.`,
|
|
1379
|
+
handover
|
|
1380
|
+
? `This is a fresh conversation: after registering, read ${handover} once — it holds the end of your previous conversation.`
|
|
1381
|
+
: 'This is a fresh conversation: your tasks, teams and wiki are all still in Crewly.',
|
|
1382
|
+
];
|
|
1383
|
+
if (workItem)
|
|
1384
|
+
parts.push(`Then continue WorkItem ${workItem.id} ("${workItem.title}") where you left off.`);
|
|
1385
|
+
return parts.join(' ');
|
|
1386
|
+
}
|
|
1159
1387
|
revertNote(override, handover, workItem) {
|
|
1160
1388
|
const parts = [
|
|
1161
1389
|
`${this.label(override.primary)} is available again, so Crewly moved you back from ${this.label(override.runtime)}.`,
|