crewly 1.20.172 → 1.20.173
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/backend/src/constants.d.ts +29 -1
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +32 -1
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +34 -2
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js +17 -2
- package/dist/backend/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-exec.utils.d.ts.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-exec.utils.js +1 -0
- package/dist/backend/backend/src/services/harness/harness-exec.utils.js.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-login-probe.d.ts +65 -0
- package/dist/backend/backend/src/services/harness/harness-login-probe.d.ts.map +1 -0
- package/dist/backend/backend/src/services/harness/harness-login-probe.js +99 -0
- package/dist/backend/backend/src/services/harness/harness-login-probe.js.map +1 -0
- package/dist/backend/backend/src/services/harness/harness-relogin.service.d.ts +349 -113
- package/dist/backend/backend/src/services/harness/harness-relogin.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/harness/harness-relogin.service.js +792 -278
- package/dist/backend/backend/src/services/harness/harness-relogin.service.js.map +1 -1
- package/dist/backend/backend/src/services/harness/harness.types.d.ts +2 -0
- package/dist/backend/backend/src/services/harness/harness.types.d.ts.map +1 -1
- package/dist/backend/backend/src/services/harness/relogin-state.store.d.ts +61 -0
- package/dist/backend/backend/src/services/harness/relogin-state.store.d.ts.map +1 -0
- package/dist/backend/backend/src/services/harness/relogin-state.store.js +92 -0
- package/dist/backend/backend/src/services/harness/relogin-state.store.js.map +1 -0
- package/dist/backend/backend/src/services/messaging/owner-message-watchdog.service.d.ts +26 -1
- package/dist/backend/backend/src/services/messaging/owner-message-watchdog.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/owner-message-watchdog.service.js +77 -2
- package/dist/backend/backend/src/services/messaging/owner-message-watchdog.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-relogin-dm.service.d.ts +42 -19
- package/dist/backend/backend/src/services/slack/slack-relogin-dm.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-relogin-dm.service.js +69 -31
- package/dist/backend/backend/src/services/slack/slack-relogin-dm.service.js.map +1 -1
- package/dist/backend/backend/src/services/v3/request-status-update.subscriber.d.ts +1 -1
- package/dist/backend/backend/src/services/v3/request-status-update.subscriber.js +15 -15
- package/dist/backend/backend/src/services/v3/request-status-update.subscriber.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +29 -1
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +32 -1
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js +17 -2
- package/dist/cli/backend/src/services/agent/oauth-relogin-monitor.service.js.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-exec.utils.d.ts.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-exec.utils.js +1 -0
- package/dist/cli/backend/src/services/harness/harness-exec.utils.js.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-login-probe.d.ts +65 -0
- package/dist/cli/backend/src/services/harness/harness-login-probe.d.ts.map +1 -0
- package/dist/cli/backend/src/services/harness/harness-login-probe.js +99 -0
- package/dist/cli/backend/src/services/harness/harness-login-probe.js.map +1 -0
- package/dist/cli/backend/src/services/harness/harness-relogin.service.d.ts +349 -113
- package/dist/cli/backend/src/services/harness/harness-relogin.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/harness/harness-relogin.service.js +792 -278
- package/dist/cli/backend/src/services/harness/harness-relogin.service.js.map +1 -1
- package/dist/cli/backend/src/services/harness/harness.types.d.ts +2 -0
- package/dist/cli/backend/src/services/harness/harness.types.d.ts.map +1 -1
- package/dist/cli/backend/src/services/harness/relogin-state.store.d.ts +61 -0
- package/dist/cli/backend/src/services/harness/relogin-state.store.d.ts.map +1 -0
- package/dist/cli/backend/src/services/harness/relogin-state.store.js +92 -0
- package/dist/cli/backend/src/services/harness/relogin-state.store.js.map +1 -0
- package/dist/cli/backend/src/services/messaging/owner-message-watchdog.service.d.ts +26 -1
- package/dist/cli/backend/src/services/messaging/owner-message-watchdog.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/messaging/owner-message-watchdog.service.js +77 -2
- package/dist/cli/backend/src/services/messaging/owner-message-watchdog.service.js.map +1 -1
- package/frontend/dist/assets/{index-cb23748e.js → index-36e85ac8.js} +258 -258
- package/frontend/dist/index.html +1 -1
- package/package.json +1 -1
|
@@ -1,64 +1,71 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Harness re-login coordinator —
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* -
|
|
43
|
-
*
|
|
44
|
-
*
|
|
2
|
+
* Harness re-login coordinator — re-login from the owner's phone, with no
|
|
3
|
+
* agent involved.
|
|
4
|
+
*
|
|
5
|
+
* The instance backend (never an agent: every agent may be stuck on the very
|
|
6
|
+
* login that expired) notices that a harness's login is gone, tells the owner
|
|
7
|
+
* once, runs the harness's login itself when the owner replies, relays the
|
|
8
|
+
* link and the code, and brings the agents back with the messages that
|
|
9
|
+
* waited for them.
|
|
10
|
+
*
|
|
11
|
+
* Flow per harness (at most one at a time):
|
|
12
|
+
*
|
|
13
|
+
* 1. **Detect.** {@link HarnessReloginService.reportExpiry} — an agent's
|
|
14
|
+
* output or screen matched an expiry rule (login-expiry-rules.ts, via the
|
|
15
|
+
* OAuth re-login monitor), or the periodic check
|
|
16
|
+
* ({@link HarnessReloginService.checkHarnesses}) found a harness in use
|
|
17
|
+
* signed out. The periodic check covers every harness the orchestrator or
|
|
18
|
+
* a team member is configured on, so it works with zero agents running.
|
|
19
|
+
* 2. **Confirm.** The harness's status command, then — because Claude Code's
|
|
20
|
+
* stored credential survives an expiry — a live probe
|
|
21
|
+
* (harness-login-probe.ts). Only a confirmed `logged_out` goes on;
|
|
22
|
+
* `unknown` never does (`codex login` revokes a working login).
|
|
23
|
+
* 3. **Tell the owner once.** A stored API key is used silently if Crewly has
|
|
24
|
+
* one. Otherwise one DM — "Claude Code on <machine> is signed out, so N
|
|
25
|
+
* agents can't work. Reply `login` …" — then re-reminders with backoff
|
|
26
|
+
* (3 h, doubling, 24 h cap), persisted so restarts do not re-send it. No
|
|
27
|
+
* login is started yet: a link sent while the owner is away expires unused.
|
|
28
|
+
* 4. **The owner replies `login`** (or 「重新登录」, `relogin claude`, …) in
|
|
29
|
+
* the DM. The Slack bridge offers owner DMs to
|
|
30
|
+
* {@link HarnessReloginService.handleOwnerReply} before anything else, so
|
|
31
|
+
* this works with no agent awake. The login broker runs the harness's
|
|
32
|
+
* login in its own PTY (`claude setup-token` / `codex login --device-auth`)
|
|
33
|
+
* and the link (and Codex's one-time code) is sent back.
|
|
34
|
+
* 5. **The code.** For Claude the owner replies with the code from the page;
|
|
35
|
+
* it is typed into the waiting login. A rejected code is reported once; an
|
|
36
|
+
* expired link puts the flow back to "signed out" ("reply `login` for a
|
|
37
|
+
* new link").
|
|
38
|
+
* 6. **Success** is confirmed with the probe, then the stuck agents are
|
|
39
|
+
* restarted (conversation resumed) and the owner messages that waited are
|
|
40
|
+
* re-delivered ({@link HarnessReloginDeps.onLoginRestored}). One DM says so.
|
|
41
|
+
*
|
|
42
|
+
* Owner-requested logins ({@link HarnessReloginService.startOwnerLogin}:
|
|
43
|
+
* 「重新登录 claude」 / "relogin codex" / the orchestrator's `harness-login`
|
|
44
|
+
* skill) start the broker at once, even while logged in (account switch).
|
|
45
|
+
*
|
|
46
|
+
* Secrets: Claude's code is passed to the broker and never stored, logged
|
|
47
|
+
* or echoed; DMs are built from {@link LoginSession} snapshots, which never
|
|
48
|
+
* contain a token, and the screen text is redacted again.
|
|
45
49
|
*
|
|
46
50
|
* @module services/harness/harness-relogin.service
|
|
47
51
|
*/
|
|
52
|
+
import * as os from 'os';
|
|
48
53
|
import { HARNESS_CONSTANTS } from '../../constants.js';
|
|
49
54
|
import { LoggerService } from '../core/logger.service.js';
|
|
50
55
|
import { getHarnessCredentialsStore } from './harness-credentials.store.js';
|
|
56
|
+
import { createHarnessLoginProbe } from './harness-login-probe.js';
|
|
51
57
|
import { getBrokerLoginMethod, getHarnessDefinition } from './harness-registry.js';
|
|
52
58
|
import { getHarnessService } from './harness.service.js';
|
|
53
59
|
import { SILENT_HARNESS_LOGGER, isHarnessId, isTerminalLoginState, } from './harness.types.js';
|
|
54
60
|
import { LOGIN_BROKER_EVENTS } from './login-broker.service.js';
|
|
55
61
|
import { redactSecrets } from './login-rules.js';
|
|
56
62
|
import { parseOwnerLoginRequest } from './owner-login-request.js';
|
|
57
|
-
|
|
63
|
+
import { MemoryReloginStateStore, createDefaultReloginStateStore } from './relogin-state.store.js';
|
|
64
|
+
/** A detected expiry: not forced, standard wording, this machine's DM. */
|
|
58
65
|
const EXPIRY_FLOW = { trigger: 'expiry', switchAccount: false, replyTarget: null };
|
|
59
66
|
/** Harnesses the coordinator can log in (they have a broker method). */
|
|
60
67
|
function hasBrokerLogin(harnessId) {
|
|
61
|
-
return getBrokerLoginMethod(harnessId) !== undefined;
|
|
68
|
+
return isHarnessId(harnessId) && getBrokerLoginMethod(harnessId) !== undefined;
|
|
62
69
|
}
|
|
63
70
|
/**
|
|
64
71
|
* Display name of a harness.
|
|
@@ -69,6 +76,37 @@ function hasBrokerLogin(harnessId) {
|
|
|
69
76
|
function displayName(harnessId) {
|
|
70
77
|
return getHarnessDefinition(harnessId)?.displayName ?? harnessId;
|
|
71
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* The word the owner types after `relogin` for a harness.
|
|
81
|
+
*
|
|
82
|
+
* @param harnessId - Harness id
|
|
83
|
+
* @returns e.g. "claude"
|
|
84
|
+
*/
|
|
85
|
+
export function harnessCommandWord(harnessId) {
|
|
86
|
+
if (harnessId === HARNESS_CONSTANTS.IDS.CLAUDE_CODE)
|
|
87
|
+
return 'claude';
|
|
88
|
+
if (harnessId === HARNESS_CONSTANTS.IDS.CODEX_CLI)
|
|
89
|
+
return 'codex';
|
|
90
|
+
if (harnessId === HARNESS_CONSTANTS.IDS.ANTIGRAVITY_CLI)
|
|
91
|
+
return 'antigravity';
|
|
92
|
+
return 'gemini';
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Normalise a short reply for keyword matching: trim, lower-case, drop
|
|
96
|
+
* surrounding quotes / backticks and trailing punctuation.
|
|
97
|
+
*
|
|
98
|
+
* @param text - Owner reply
|
|
99
|
+
* @returns Normalised text
|
|
100
|
+
*/
|
|
101
|
+
function normaliseKeyword(text) {
|
|
102
|
+
return text
|
|
103
|
+
.trim()
|
|
104
|
+
.replace(/^[`'"「“]+|[`'"」”]+$/g, '')
|
|
105
|
+
.replace(/[\s.!?。!?~~]+$/u, '')
|
|
106
|
+
.trim()
|
|
107
|
+
.toLowerCase()
|
|
108
|
+
.replace(/\s+/g, ' ');
|
|
109
|
+
}
|
|
72
110
|
/**
|
|
73
111
|
* Whether a reply asks to start the login over.
|
|
74
112
|
*
|
|
@@ -76,9 +114,27 @@ function displayName(harnessId) {
|
|
|
76
114
|
* @returns True for `relogin`, `re-login` or `重新登录`
|
|
77
115
|
*/
|
|
78
116
|
export function isRetryKeyword(text) {
|
|
79
|
-
const normalized = text
|
|
117
|
+
const normalized = normaliseKeyword(text);
|
|
80
118
|
return HARNESS_CONSTANTS.RELOGIN.RETRY_KEYWORDS.some((keyword) => keyword.toLowerCase() === normalized);
|
|
81
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* Whether a bare reply asks to sign in (`login`, `sign in`, 「登录」, the retry keywords).
|
|
122
|
+
*
|
|
123
|
+
* @param text - Owner reply
|
|
124
|
+
* @returns True when the whole reply is a login keyword
|
|
125
|
+
*
|
|
126
|
+
* @example
|
|
127
|
+
* ```ts
|
|
128
|
+
* isLoginKeyword('Login!'); // true
|
|
129
|
+
* isLoginKeyword('login to gmail'); // false
|
|
130
|
+
* ```
|
|
131
|
+
*/
|
|
132
|
+
export function isLoginKeyword(text) {
|
|
133
|
+
if (typeof text !== 'string')
|
|
134
|
+
return false;
|
|
135
|
+
const normalized = normaliseKeyword(text);
|
|
136
|
+
return [...HARNESS_CONSTANTS.RELOGIN.LOGIN_KEYWORDS, ...HARNESS_CONSTANTS.RELOGIN.RETRY_KEYWORDS].some((keyword) => keyword.toLowerCase() === normalized);
|
|
137
|
+
}
|
|
82
138
|
/**
|
|
83
139
|
* Strip formatting Slack clients add around a pasted value (backticks).
|
|
84
140
|
*
|
|
@@ -100,18 +156,26 @@ export function looksLikeAuthCode(text) {
|
|
|
100
156
|
return text.length >= CODE_MIN_LENGTH && text.length <= CODE_MAX_LENGTH && !/\s/.test(text);
|
|
101
157
|
}
|
|
102
158
|
/**
|
|
103
|
-
* "
|
|
159
|
+
* "Ella, Rex and 3 more" — capped list of agent names.
|
|
104
160
|
*
|
|
105
|
-
* @param agents -
|
|
106
|
-
* @returns
|
|
161
|
+
* @param agents - Agent names
|
|
162
|
+
* @returns List text ('' when empty)
|
|
107
163
|
*/
|
|
108
164
|
export function describeWaitingAgents(agents) {
|
|
109
165
|
if (agents.length === 0)
|
|
110
|
-
return '
|
|
166
|
+
return '';
|
|
111
167
|
const max = HARNESS_CONSTANTS.RELOGIN.DM_MAX_LISTED_AGENTS;
|
|
112
|
-
const listed = agents.slice(0, max).join('
|
|
113
|
-
|
|
114
|
-
|
|
168
|
+
const listed = agents.slice(0, max).join(', ');
|
|
169
|
+
return agents.length > max ? `${listed} and ${agents.length - max} more` : listed;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* "1 agent" / "3 agents".
|
|
173
|
+
*
|
|
174
|
+
* @param n - Count
|
|
175
|
+
* @returns Phrase
|
|
176
|
+
*/
|
|
177
|
+
function agentCount(n) {
|
|
178
|
+
return `${n} agent${n === 1 ? '' : 's'}`;
|
|
115
179
|
}
|
|
116
180
|
/**
|
|
117
181
|
* Keep text safe for a DM: redact secrets, drop code fences, cap length (tail).
|
|
@@ -125,7 +189,7 @@ function safeForDm(text, max) {
|
|
|
125
189
|
return redacted.length > max ? redacted.slice(redacted.length - max) : redacted;
|
|
126
190
|
}
|
|
127
191
|
/** Retry hint appended to failure DMs. */
|
|
128
|
-
const RETRY_HINT = '
|
|
192
|
+
const RETRY_HINT = 'Reply `login` to try again.';
|
|
129
193
|
/**
|
|
130
194
|
* Prefix an owner reply needs before it is typed into an unrecognised login
|
|
131
195
|
* screen. Without it, any one-line DM to the orc during the login would have
|
|
@@ -133,7 +197,7 @@ const RETRY_HINT = '回复「重新登录」(或 `relogin`)再试一次。';
|
|
|
133
197
|
*/
|
|
134
198
|
export const SCREEN_REPLY_PREFIXES = ['输入', 'input'];
|
|
135
199
|
/**
|
|
136
|
-
* The text after a screen-reply prefix ("
|
|
200
|
+
* The text after a screen-reply prefix ("input 1" → "1"), or null.
|
|
137
201
|
*
|
|
138
202
|
* @param reply - Owner reply (already unwrapped)
|
|
139
203
|
* @returns Text to type, or null when the reply has no prefix
|
|
@@ -147,41 +211,66 @@ export function stripScreenReplyPrefix(reply) {
|
|
|
147
211
|
return null;
|
|
148
212
|
}
|
|
149
213
|
/**
|
|
150
|
-
* First line of a
|
|
214
|
+
* First line of a link DM.
|
|
151
215
|
*
|
|
152
216
|
* @param harnessId - Harness
|
|
153
|
-
* @param
|
|
154
|
-
* @param wording - Owner-requested / account switch
|
|
217
|
+
* @param wording - Account switch / machine
|
|
155
218
|
* @returns Slack mrkdwn line
|
|
156
219
|
*/
|
|
157
|
-
function loginDmHeader(harnessId,
|
|
220
|
+
function loginDmHeader(harnessId, wording) {
|
|
221
|
+
const name = displayName(harnessId);
|
|
222
|
+
const where = wording.machine ? ` on ${wording.machine}` : '';
|
|
223
|
+
return wording.switchAccount ? `*Switch the ${name} account${where}*` : `*Sign in to ${name}${where}*`;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* The first "signed out" DM, a re-reminder, or the note after a link expired unused.
|
|
227
|
+
*
|
|
228
|
+
* @param harnessId - Harness
|
|
229
|
+
* @param machine - This machine's name
|
|
230
|
+
* @param agents - Names of the agents that run on it
|
|
231
|
+
* @param kind - `notice` (first), `reminder`, or `link_expired`
|
|
232
|
+
* @returns Slack mrkdwn text
|
|
233
|
+
*
|
|
234
|
+
* @example
|
|
235
|
+
* ```ts
|
|
236
|
+
* formatSignedOutDm('claude-code', 'iriss-air.lan', ['Ella', 'Rex'], 'notice');
|
|
237
|
+
* // "*Claude Code on iriss-air.lan is signed out*, so 2 agents can't work (Ella, Rex).\nReply `login` …"
|
|
238
|
+
* ```
|
|
239
|
+
*/
|
|
240
|
+
export function formatSignedOutDm(harnessId, machine, agents, kind = 'notice') {
|
|
158
241
|
const name = displayName(harnessId);
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
242
|
+
const reply = `Reply \`login\` here to sign in from your phone (or \`relogin ${harnessCommandWord(harnessId)}\`).`;
|
|
243
|
+
const listed = describeWaitingAgents(agents);
|
|
244
|
+
const who = agents.length > 0 ? `, so ${agentCount(agents.length)} can't work (${listed})` : '';
|
|
245
|
+
if (kind === 'link_expired') {
|
|
246
|
+
return `The sign-in link for ${name} on ${machine} expired before it was used${agents.length > 0 ? `; ${agentCount(agents.length)} still waiting` : ''}. Reply \`login\` when you're ready and I'll send a fresh one (it works for 15 minutes).`;
|
|
247
|
+
}
|
|
248
|
+
if (kind === 'reminder')
|
|
249
|
+
return `Reminder: ${name} on ${machine} is still signed out${who}.\n${reply}`;
|
|
250
|
+
return `*${name} on ${machine} is signed out*${who}.\n${reply}`;
|
|
162
251
|
}
|
|
163
252
|
/**
|
|
164
253
|
* DM with the sign-in link (and Codex's one-time code on its own line).
|
|
165
254
|
*
|
|
166
255
|
* @param session - Broker session (never contains a secret)
|
|
167
|
-
* @param
|
|
168
|
-
* @param wording -
|
|
256
|
+
* @param _waiting - Agents waiting on the login (unused; kept for callers)
|
|
257
|
+
* @param wording - Account switch / machine
|
|
169
258
|
* @returns Slack mrkdwn text
|
|
170
259
|
*/
|
|
171
|
-
export function formatLinkDm(session,
|
|
172
|
-
const lines = [loginDmHeader(session.harnessId,
|
|
260
|
+
export function formatLinkDm(session, _waiting = [], wording = {}) {
|
|
261
|
+
const lines = [loginDmHeader(session.harnessId, wording), ''];
|
|
173
262
|
if (session.method === 'device') {
|
|
174
|
-
lines.push('1.
|
|
263
|
+
lines.push('1. Open this link on your phone:', session.url ?? '', '', '2. Enter this code:', session.userCode ?? '', '');
|
|
175
264
|
if (wording.switchAccount)
|
|
176
|
-
lines.push('
|
|
177
|
-
lines.push('
|
|
265
|
+
lines.push('To use a different account, switch to it on that page first.');
|
|
266
|
+
lines.push('Finish signing in on your phone; Crewly continues by itself.');
|
|
178
267
|
}
|
|
179
268
|
else {
|
|
180
|
-
lines.push('1.
|
|
269
|
+
lines.push('1. Open this link on your phone and approve:', session.url ?? '', '');
|
|
181
270
|
if (wording.switchAccount)
|
|
182
|
-
lines.push('
|
|
183
|
-
lines.push('2.
|
|
184
|
-
lines.push('Crewly
|
|
271
|
+
lines.push(' To use a different account, switch to it on that page before approving.', '');
|
|
272
|
+
lines.push('2. Reply here with the code the page shows (just the code).', '');
|
|
273
|
+
lines.push('The link works for 15 minutes. Crewly enters the code and the agents carry on.');
|
|
185
274
|
}
|
|
186
275
|
return lines.join('\n');
|
|
187
276
|
}
|
|
@@ -189,20 +278,20 @@ export function formatLinkDm(session, waiting, wording = {}) {
|
|
|
189
278
|
* DM for a login screen the broker did not recognise.
|
|
190
279
|
*
|
|
191
280
|
* @param session - Broker session
|
|
192
|
-
* @param
|
|
193
|
-
* @param wording -
|
|
281
|
+
* @param _waiting - Agents waiting on the login (unused; kept for callers)
|
|
282
|
+
* @param wording - Account switch / machine
|
|
194
283
|
* @returns Slack mrkdwn text
|
|
195
284
|
*/
|
|
196
|
-
export function formatScreenDm(session,
|
|
285
|
+
export function formatScreenDm(session, _waiting = [], wording = {}) {
|
|
197
286
|
const screen = safeForDm(session.screen, HARNESS_CONSTANTS.RELOGIN.DM_SCREEN_MAX_CHARS) || '(empty screen)';
|
|
198
287
|
return [
|
|
199
|
-
loginDmHeader(session.harnessId,
|
|
288
|
+
loginDmHeader(session.harnessId, wording),
|
|
200
289
|
'',
|
|
201
|
-
|
|
290
|
+
"Crewly started the sign-in but doesn't recognise its screen. It shows:",
|
|
202
291
|
'```',
|
|
203
292
|
screen,
|
|
204
293
|
'```',
|
|
205
|
-
'
|
|
294
|
+
'To type into it, reply `input <text>` (for example `input 1`). Other messages go to the Orc as usual.',
|
|
206
295
|
].join('\n');
|
|
207
296
|
}
|
|
208
297
|
/**
|
|
@@ -213,22 +302,34 @@ export function formatScreenDm(session, waiting, wording = {}) {
|
|
|
213
302
|
*/
|
|
214
303
|
export function formatRejectedDm(session) {
|
|
215
304
|
const reason = safeForDm(session.message ?? '', HARNESS_CONSTANTS.RELOGIN.DM_MESSAGE_MAX_CHARS);
|
|
216
|
-
const what = session.method === 'subscription' ? '
|
|
217
|
-
return
|
|
305
|
+
const what = session.method === 'subscription' ? 'code' : 'input';
|
|
306
|
+
return `That ${what} didn't work${reason ? ` (${reason})` : ''}. Reply with it again, or reply \`login\` for a fresh link.`;
|
|
218
307
|
}
|
|
219
308
|
/**
|
|
220
|
-
*
|
|
309
|
+
* "; 2 waiting messages re-delivered".
|
|
310
|
+
*
|
|
311
|
+
* @param redelivered - Count
|
|
312
|
+
* @returns Clause ('' for none)
|
|
313
|
+
*/
|
|
314
|
+
function redeliveredClause(redelivered) {
|
|
315
|
+
if (redelivered <= 0)
|
|
316
|
+
return '';
|
|
317
|
+
return `; ${redelivered} waiting message${redelivered === 1 ? '' : 's'} re-delivered`;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* DM after a login came back and the agents were restarted.
|
|
221
321
|
*
|
|
222
322
|
* @param harnessId - Harness
|
|
223
323
|
* @param result - Resume result
|
|
324
|
+
* @param redelivered - Owner messages re-delivered to them
|
|
325
|
+
* @param machine - This machine's name
|
|
224
326
|
* @returns Slack mrkdwn text
|
|
225
327
|
*/
|
|
226
|
-
export function formatSuccessDm(harnessId, result) {
|
|
227
|
-
const
|
|
228
|
-
let text =
|
|
229
|
-
if (result.failed.length > 0)
|
|
230
|
-
text += `
|
|
231
|
-
}
|
|
328
|
+
export function formatSuccessDm(harnessId, result, redelivered = 0, machine) {
|
|
329
|
+
const where = machine ? ` on ${machine}` : '';
|
|
330
|
+
let text = `Done: ${displayName(harnessId)}${where} is signed in again. ${agentCount(result.resumed.length)} resumed${redeliveredClause(redelivered)}.`;
|
|
331
|
+
if (result.failed.length > 0)
|
|
332
|
+
text += ` Could not restart: ${result.failed.join(', ')}.`;
|
|
232
333
|
return text;
|
|
233
334
|
}
|
|
234
335
|
/**
|
|
@@ -236,18 +337,19 @@ export function formatSuccessDm(harnessId, result) {
|
|
|
236
337
|
*
|
|
237
338
|
* @param harnessId - Harness
|
|
238
339
|
* @param result - Which agents were restarted onto the new login
|
|
340
|
+
* @param redelivered - Owner messages re-delivered to them
|
|
239
341
|
* @returns Slack mrkdwn text
|
|
240
342
|
*/
|
|
241
|
-
export function formatOwnerSuccessDm(harnessId, result) {
|
|
242
|
-
let text =
|
|
343
|
+
export function formatOwnerSuccessDm(harnessId, result, redelivered = 0) {
|
|
344
|
+
let text = `Done: ${displayName(harnessId)} is signed in.`;
|
|
243
345
|
if (result.resumed.length > 0)
|
|
244
|
-
text += ` ${result.resumed.length}
|
|
346
|
+
text += ` ${agentCount(result.resumed.length)} restarted on the new login${redeliveredClause(redelivered)}.`;
|
|
245
347
|
if (result.failed.length > 0)
|
|
246
|
-
text += `
|
|
348
|
+
text += ` Could not restart: ${result.failed.join(', ')}.`;
|
|
247
349
|
return text;
|
|
248
350
|
}
|
|
249
351
|
/** Harness names the owner can type, for the "which one?" reply. */
|
|
250
|
-
const WHICH_HARNESS_HINT = '
|
|
352
|
+
const WHICH_HARNESS_HINT = 'Reply `relogin claude` or `relogin codex` (to switch accounts: `switch claude account`).';
|
|
251
353
|
/**
|
|
252
354
|
* Reply to a login request that named no harness Crewly can log in.
|
|
253
355
|
*
|
|
@@ -255,7 +357,7 @@ const WHICH_HARNESS_HINT = '回复「重新登录 claude」或「重新登录 co
|
|
|
255
357
|
* @returns Slack mrkdwn text
|
|
256
358
|
*/
|
|
257
359
|
export function formatWhichHarnessDm(name) {
|
|
258
|
-
return name ? `Crewly
|
|
360
|
+
return name ? `Crewly doesn't run "${name}". Which one should I sign in? ${WHICH_HARNESS_HINT}` : `Which one should I sign in? ${WHICH_HARNESS_HINT}`;
|
|
259
361
|
}
|
|
260
362
|
/**
|
|
261
363
|
* Reply for a harness that has no link login.
|
|
@@ -266,15 +368,15 @@ export function formatWhichHarnessDm(name) {
|
|
|
266
368
|
export function formatNoBrokerLoginDm(harnessId) {
|
|
267
369
|
const name = displayName(harnessId);
|
|
268
370
|
if (harnessId === HARNESS_CONSTANTS.IDS.ANTIGRAVITY_CLI) {
|
|
269
|
-
return `${name}
|
|
371
|
+
return `${name} uses a Gemini API key, not a sign-in link. Run \`crewly login antigravity\` on the machine, or enter the key in Crewly Setup → Sign in.`;
|
|
270
372
|
}
|
|
271
373
|
if (harnessId === HARNESS_CONSTANTS.IDS.GEMINI_CLI) {
|
|
272
|
-
return `${name}
|
|
374
|
+
return `${name} is enterprise-only; Crewly can't sign it in. ${WHICH_HARNESS_HINT}`;
|
|
273
375
|
}
|
|
274
|
-
return `${name}
|
|
376
|
+
return `${name} has no sign-in link Crewly can use. ${WHICH_HARNESS_HINT}`;
|
|
275
377
|
}
|
|
276
378
|
/**
|
|
277
|
-
* DM after the login failed
|
|
379
|
+
* DM after the login failed (sent once per failure).
|
|
278
380
|
*
|
|
279
381
|
* @param harnessId - Harness
|
|
280
382
|
* @param reason - Broker message (redacted here)
|
|
@@ -282,41 +384,56 @@ export function formatNoBrokerLoginDm(harnessId) {
|
|
|
282
384
|
*/
|
|
283
385
|
export function formatFailureDm(harnessId, reason) {
|
|
284
386
|
const detail = safeForDm(reason ?? '', HARNESS_CONSTANTS.RELOGIN.DM_MESSAGE_MAX_CHARS);
|
|
285
|
-
|
|
387
|
+
const clause = detail ? `: ${detail.replace(/[.!?。!?]+$/u, '')}.` : '.';
|
|
388
|
+
return `Signing in to ${displayName(harnessId)} didn't finish${clause} ${RETRY_HINT}`;
|
|
286
389
|
}
|
|
287
390
|
/**
|
|
288
|
-
* Re-reminder text for a login whose last link expired unused.
|
|
289
|
-
* link: a new one is made when the owner replies, so it is fresh when used.
|
|
391
|
+
* Re-reminder text for a login whose last link expired unused.
|
|
290
392
|
*
|
|
291
393
|
* @param harnessId - Harness
|
|
292
394
|
* @param waiting - Agents waiting on the login
|
|
395
|
+
* @param machine - This machine's name
|
|
293
396
|
* @returns DM text
|
|
294
397
|
*/
|
|
295
|
-
export function formatReminderDm(harnessId, waiting) {
|
|
296
|
-
|
|
297
|
-
return `${displayName(harnessId)} still isn't signed in again, and the last sign-in link has expired. ${who}When it suits you, reply \`relogin\` and I'll send a new link right away (valid for 15 minutes).`;
|
|
398
|
+
export function formatReminderDm(harnessId, waiting, machine = os.hostname()) {
|
|
399
|
+
return formatSignedOutDm(harnessId, machine, waiting, 'link_expired');
|
|
298
400
|
}
|
|
299
|
-
/** Coordinates
|
|
401
|
+
/** Coordinates re-login flows, one per harness. */
|
|
300
402
|
export class HarnessReloginService {
|
|
301
403
|
broker;
|
|
302
404
|
credentials;
|
|
303
405
|
apiKeys;
|
|
304
406
|
checkLoginState;
|
|
407
|
+
verifyLogin;
|
|
305
408
|
getOrcHarness;
|
|
409
|
+
listAgents;
|
|
410
|
+
sessionNeedsLogin;
|
|
411
|
+
onLoginRestored;
|
|
412
|
+
machineName;
|
|
413
|
+
state;
|
|
306
414
|
notifier;
|
|
307
415
|
resumer;
|
|
308
416
|
now;
|
|
309
417
|
logger;
|
|
310
418
|
flows = new Map();
|
|
419
|
+
/** Harnesses whose expiry is being confirmed → sessions reported meanwhile */
|
|
420
|
+
confirming = new Map();
|
|
421
|
+
/** Harnesses whose re-reminder is being prepared */
|
|
422
|
+
reminding = new Set();
|
|
311
423
|
/** Expiry reports are ignored until then (resumed transcripts repeat old errors) */
|
|
312
424
|
quietUntil = new Map();
|
|
313
425
|
/** Last silent API-key recovery per harness */
|
|
314
426
|
silentKeyAt = new Map();
|
|
315
427
|
/** Sessions restarted after a login: screen-sweep reports are ignored until live output reports again */
|
|
316
428
|
mutedScreens = new Set();
|
|
317
|
-
/**
|
|
318
|
-
|
|
429
|
+
/** Last probe result per harness */
|
|
430
|
+
probeCache = new Map();
|
|
431
|
+
/** Probes in flight per harness */
|
|
432
|
+
probeInFlight = new Map();
|
|
433
|
+
/** Agents configured on this machine, from the last {@link listAgents} */
|
|
434
|
+
agents = [];
|
|
319
435
|
statusTimer = null;
|
|
436
|
+
firstCheckTimer = null;
|
|
320
437
|
statusCheckInFlight = false;
|
|
321
438
|
/**
|
|
322
439
|
* @param deps - Dependencies
|
|
@@ -326,7 +443,13 @@ export class HarnessReloginService {
|
|
|
326
443
|
this.credentials = deps.credentials;
|
|
327
444
|
this.apiKeys = deps.apiKeys;
|
|
328
445
|
this.checkLoginState = deps.checkLoginState;
|
|
446
|
+
this.verifyLogin = deps.verifyLogin ?? null;
|
|
329
447
|
this.getOrcHarness = deps.getOrcHarness;
|
|
448
|
+
this.listAgents = deps.listAgents ?? null;
|
|
449
|
+
this.sessionNeedsLogin = deps.sessionNeedsLogin ?? null;
|
|
450
|
+
this.onLoginRestored = deps.onLoginRestored ?? null;
|
|
451
|
+
this.machineName = deps.machineName ?? (() => os.hostname());
|
|
452
|
+
this.state = deps.state ?? new MemoryReloginStateStore();
|
|
330
453
|
this.notifier = deps.notifier ?? null;
|
|
331
454
|
this.resumer = deps.resumer ?? null;
|
|
332
455
|
this.now = deps.now ?? Date.now;
|
|
@@ -350,6 +473,30 @@ export class HarnessReloginService {
|
|
|
350
473
|
setResumer(resumer) {
|
|
351
474
|
this.resumer = resumer;
|
|
352
475
|
}
|
|
476
|
+
/**
|
|
477
|
+
* Set what re-delivers the owner messages that waited for a login.
|
|
478
|
+
*
|
|
479
|
+
* @param handler - Handler, or null
|
|
480
|
+
*/
|
|
481
|
+
setLoginRestoredHandler(handler) {
|
|
482
|
+
this.onLoginRestored = handler ?? null;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Set how the configured agents (running or not) are listed.
|
|
486
|
+
*
|
|
487
|
+
* @param lister - Lister, or null
|
|
488
|
+
*/
|
|
489
|
+
setAgentLister(lister) {
|
|
490
|
+
this.listAgents = lister ?? null;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Set the per-session sign-in flag lookup (OAuth monitor).
|
|
494
|
+
*
|
|
495
|
+
* @param lookup - Lookup, or null
|
|
496
|
+
*/
|
|
497
|
+
setSessionNeedsLogin(lookup) {
|
|
498
|
+
this.sessionNeedsLogin = lookup ?? null;
|
|
499
|
+
}
|
|
353
500
|
/**
|
|
354
501
|
* Report an expired harness login.
|
|
355
502
|
*
|
|
@@ -375,43 +522,66 @@ export class HarnessReloginService {
|
|
|
375
522
|
this.mutedScreens.delete(sessionName);
|
|
376
523
|
}
|
|
377
524
|
const existing = this.flows.get(harnessId);
|
|
378
|
-
if (existing) {
|
|
525
|
+
if (existing && (existing.phase === 'running' || existing.remind)) {
|
|
379
526
|
if (sessionName)
|
|
380
527
|
existing.stuck.add(sessionName);
|
|
381
|
-
if (existing.phase === '
|
|
382
|
-
this.
|
|
383
|
-
if (existing.lastOutcome === 'timed_out')
|
|
384
|
-
void this.remind(existing);
|
|
385
|
-
else
|
|
386
|
-
this.restart(existing);
|
|
387
|
-
}
|
|
528
|
+
if (existing.phase === 'signed_out')
|
|
529
|
+
void this.maybeRemind(existing);
|
|
388
530
|
return true;
|
|
389
531
|
}
|
|
390
|
-
this.
|
|
391
|
-
|
|
532
|
+
const confirming = this.confirming.get(harnessId);
|
|
533
|
+
if (confirming) {
|
|
534
|
+
if (sessionName)
|
|
535
|
+
confirming.add(sessionName);
|
|
536
|
+
return true;
|
|
537
|
+
}
|
|
538
|
+
// An owner flow that failed (e.g. an account switch) was not a confirmed
|
|
539
|
+
// expiry; this report is — confirm it like any other.
|
|
540
|
+
const stuck = new Set(existing ? existing.stuck : []);
|
|
541
|
+
if (sessionName)
|
|
542
|
+
stuck.add(sessionName);
|
|
543
|
+
this.confirming.set(harnessId, stuck);
|
|
544
|
+
this.logger.warn('Harness login looks expired — confirming before telling the owner', { harnessId, sessionName, source });
|
|
545
|
+
void this.confirmExpiry(harnessId);
|
|
392
546
|
return true;
|
|
393
547
|
}
|
|
394
548
|
/**
|
|
395
549
|
* Offer an owner's Slack DM to the re-login coordinator.
|
|
396
550
|
*
|
|
397
|
-
* Consumed, in this order:
|
|
398
|
-
* login code or screen reply for a
|
|
399
|
-
*
|
|
551
|
+
* Consumed, in this order: a login keyword (`login`, 「重新登录」…) while a
|
|
552
|
+
* harness is signed out or signing in; a login code or screen reply for a
|
|
553
|
+
* running flow (only in the conversation the link went to); and finally an
|
|
554
|
+
* owner login request (「重新登录 claude」, "relogin codex" — see
|
|
400
555
|
* {@link parseOwnerLoginRequest}), which starts a forced login answered in
|
|
401
556
|
* `target`.
|
|
402
557
|
*
|
|
558
|
+
* In another agent's bot DM (`scope: 'agent'`) only a request that names a
|
|
559
|
+
* harness, or a reply to a flow, is taken — the rest is that agent's mail.
|
|
560
|
+
*
|
|
403
561
|
* @param text - Reply text (never logged)
|
|
404
562
|
* @param target - Conversation the owner wrote in (answers go there)
|
|
563
|
+
* @param scope - `orc` (orc / master DM) or `agent` (another agent's bot DM)
|
|
405
564
|
* @returns True when consumed; the caller must then NOT pass it on as a chat message
|
|
406
565
|
*/
|
|
407
|
-
handleOwnerReply(text, target = null) {
|
|
566
|
+
handleOwnerReply(text, target = null, scope = 'orc') {
|
|
408
567
|
if (typeof text !== 'string')
|
|
409
568
|
return false;
|
|
410
|
-
if (this.flows.size > 0 && this.consumeFlowReply(text))
|
|
569
|
+
if (this.flows.size > 0 && this.consumeFlowReply(text, target))
|
|
411
570
|
return true;
|
|
571
|
+
// `login` in an agent's DM while that agent sits at a sign-in screen:
|
|
572
|
+
// sign in its harness (the watchdog's note asks for exactly this).
|
|
573
|
+
if (scope === 'agent' && isLoginKeyword(text) && target?.agentSession) {
|
|
574
|
+
const harnessId = this.harnessOfSession(target.agentSession);
|
|
575
|
+
if (harnessId && hasBrokerLogin(harnessId) && (this.isSignedOut(harnessId) || this.sessionNeedsLogin?.(target.agentSession))) {
|
|
576
|
+
this.startOwnerLogin(harnessId, { replyTarget: target, requestedBy: 'owner_dm' });
|
|
577
|
+
return true;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
412
580
|
const request = parseOwnerLoginRequest(text);
|
|
413
581
|
if (!request)
|
|
414
582
|
return false;
|
|
583
|
+
if (scope === 'agent' && request.kind !== 'harness')
|
|
584
|
+
return false;
|
|
415
585
|
void this.handleOwnerLoginRequest(request, target);
|
|
416
586
|
return true;
|
|
417
587
|
}
|
|
@@ -430,29 +600,41 @@ export class HarnessReloginService {
|
|
|
430
600
|
return { status: 'no_broker_login', harnessId, message: formatNoBrokerLoginDm(harnessId) };
|
|
431
601
|
}
|
|
432
602
|
const existing = this.flows.get(harnessId);
|
|
433
|
-
const
|
|
434
|
-
if (existing) {
|
|
435
|
-
this.clearScreenTimer(existing);
|
|
436
|
-
this.cancelOwnSession(existing);
|
|
437
|
-
if (this.flows.get(harnessId) === existing)
|
|
438
|
-
this.flows.delete(harnessId);
|
|
439
|
-
}
|
|
440
|
-
// The owner asked: a quiet period after the last login must not swallow it.
|
|
441
|
-
this.quietUntil.delete(harnessId);
|
|
603
|
+
const restarted = Boolean(existing && existing.phase === 'running');
|
|
442
604
|
this.logger.info('Owner-requested login started', {
|
|
443
605
|
harnessId,
|
|
444
606
|
requestedBy: options.requestedBy,
|
|
445
607
|
switchAccount: options.switchAccount === true,
|
|
446
|
-
restarted
|
|
608
|
+
restarted,
|
|
447
609
|
answersInThread: Boolean(options.replyTarget),
|
|
448
610
|
});
|
|
449
|
-
this.
|
|
611
|
+
this.replaceAndStart(harnessId, existing ?? null, {
|
|
450
612
|
trigger: 'owner',
|
|
451
613
|
switchAccount: options.switchAccount === true,
|
|
452
|
-
replyTarget: options.replyTarget ?? null,
|
|
614
|
+
replyTarget: options.replyTarget ?? existing?.replyTarget ?? null,
|
|
453
615
|
});
|
|
454
616
|
const dmAvailable = this.notifier ? (this.notifier.isAvailable?.() ?? true) : false;
|
|
455
|
-
return { status:
|
|
617
|
+
return { status: restarted ? 'restarted' : 'started', harnessId, dmAvailable };
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Harnesses confirmed signed out on this machine (persisted).
|
|
621
|
+
*
|
|
622
|
+
* @param harnessId - Harness
|
|
623
|
+
* @returns True while the owner has not signed it in again
|
|
624
|
+
*/
|
|
625
|
+
isSignedOut(harnessId) {
|
|
626
|
+
return this.state.get(harnessId).signedOutSince !== undefined;
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* The signed-out harness an agent is waiting on, if any — for the
|
|
630
|
+
* owner-message watchdog, which must not wake an agent onto a dead login.
|
|
631
|
+
*
|
|
632
|
+
* @param sessionName - Agent session
|
|
633
|
+
* @returns The harness, or null when its harness is not signed out
|
|
634
|
+
*/
|
|
635
|
+
signedOutHarnessOf(sessionName) {
|
|
636
|
+
const harnessId = this.harnessOfSession(sessionName);
|
|
637
|
+
return harnessId && this.isSignedOut(harnessId) ? harnessId : null;
|
|
456
638
|
}
|
|
457
639
|
/**
|
|
458
640
|
* Act on a parsed owner login request from the DM.
|
|
@@ -470,26 +652,33 @@ export class HarnessReloginService {
|
|
|
470
652
|
await this.notify(result.message, target);
|
|
471
653
|
}
|
|
472
654
|
/**
|
|
473
|
-
* Offer a DM to the
|
|
474
|
-
*
|
|
655
|
+
* Offer a DM to the flows: a login keyword, a login code, or a reply to
|
|
656
|
+
* an unrecognised screen.
|
|
475
657
|
*
|
|
476
658
|
* @param text - Reply text (never logged)
|
|
659
|
+
* @param target - Conversation it was written in
|
|
477
660
|
* @returns True when consumed
|
|
478
661
|
*/
|
|
479
|
-
consumeFlowReply(text) {
|
|
480
|
-
if (
|
|
662
|
+
consumeFlowReply(text, target) {
|
|
663
|
+
if (isLoginKeyword(text)) {
|
|
481
664
|
const flows = [...this.flows.values()];
|
|
482
|
-
const
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
665
|
+
for (const flow of flows) {
|
|
666
|
+
this.replaceAndStart(flow.harnessId, flow, {
|
|
667
|
+
trigger: flow.trigger,
|
|
668
|
+
switchAccount: flow.switchAccount,
|
|
669
|
+
replyTarget: target ?? flow.replyTarget,
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
this.logger.info('Sign-in started on the owner\'s reply', { harnessIds: flows.map((flow) => flow.harnessId) });
|
|
487
673
|
return true;
|
|
488
674
|
}
|
|
489
675
|
const reply = unwrapReply(text);
|
|
490
676
|
for (const flow of this.flows.values()) {
|
|
491
677
|
if (flow.phase !== 'running' || !flow.sessionId)
|
|
492
678
|
continue;
|
|
679
|
+
// The code belongs in the conversation the link went to.
|
|
680
|
+
if (flow.linkChannelId && target && target.channelId !== flow.linkChannelId)
|
|
681
|
+
continue;
|
|
493
682
|
const session = this.readSession(flow.sessionId);
|
|
494
683
|
if (!session || isTerminalLoginState(session.state))
|
|
495
684
|
continue;
|
|
@@ -532,32 +721,40 @@ export class HarnessReloginService {
|
|
|
532
721
|
return { harnessId, sessionId: flow.sessionId, startedAt: new Date(flow.startedAt).toISOString() };
|
|
533
722
|
}
|
|
534
723
|
/**
|
|
535
|
-
* Check
|
|
536
|
-
*
|
|
537
|
-
*
|
|
538
|
-
*
|
|
724
|
+
* Check every harness in use on this machine — the orchestrator's and every
|
|
725
|
+
* configured agent's, running or not — and report the ones that are signed
|
|
726
|
+
* out. A harness the status command calls logged in is probed for real now
|
|
727
|
+
* and then ({@link HARNESS_CONSTANTS.RELOGIN.PROBE_INTERVAL_MS}), because a
|
|
728
|
+
* stored Claude credential survives its expiry. Signed-out harnesses get
|
|
729
|
+
* their (backed-off) re-reminder here.
|
|
539
730
|
*
|
|
540
731
|
* @returns Resolves when the check is done; never rejects
|
|
541
732
|
*/
|
|
542
|
-
async
|
|
733
|
+
async checkHarnesses() {
|
|
543
734
|
if (this.statusCheckInFlight)
|
|
544
735
|
return;
|
|
545
736
|
this.statusCheckInFlight = true;
|
|
546
737
|
try {
|
|
547
|
-
const orc = await this.getOrcHarness();
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
const
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
738
|
+
const orc = await this.getOrcHarness().catch(() => null);
|
|
739
|
+
const agents = await this.refreshAgents();
|
|
740
|
+
const configured = new Map();
|
|
741
|
+
for (const agent of agents) {
|
|
742
|
+
if (hasBrokerLogin(agent.harnessId))
|
|
743
|
+
configured.set(agent.harnessId, (configured.get(agent.harnessId) ?? 0) + 1);
|
|
744
|
+
}
|
|
745
|
+
const harnesses = new Set(configured.keys());
|
|
746
|
+
if (orc && hasBrokerLogin(orc))
|
|
747
|
+
harnesses.add(orc);
|
|
748
|
+
for (const harnessId of harnesses) {
|
|
749
|
+
const flow = this.flows.get(harnessId);
|
|
750
|
+
if (flow?.phase === 'running' || this.confirming.has(harnessId))
|
|
751
|
+
continue;
|
|
752
|
+
if (flow?.phase === 'signed_out' && flow.remind) {
|
|
753
|
+
await this.maybeRemind(flow);
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
await this.checkOne(harnessId, (configured.get(harnessId) ?? 0) > 0 || harnessId === orc);
|
|
554
757
|
}
|
|
555
|
-
if (state !== 'logged_out')
|
|
556
|
-
return;
|
|
557
|
-
const running = this.resumer?.listSessions(orc) ?? [];
|
|
558
|
-
if (!this.seenLoggedIn.has(orc) && running.length === 0)
|
|
559
|
-
return;
|
|
560
|
-
this.reportExpiry({ harnessId: orc, sessionName: null, source: 'status' });
|
|
561
758
|
}
|
|
562
759
|
catch (error) {
|
|
563
760
|
this.logger.warn('Harness status check failed', { error: error instanceof Error ? error.message : String(error) });
|
|
@@ -567,35 +764,263 @@ export class HarnessReloginService {
|
|
|
567
764
|
}
|
|
568
765
|
}
|
|
569
766
|
/**
|
|
570
|
-
*
|
|
767
|
+
* Compatibility name of {@link checkHarnesses}.
|
|
768
|
+
*
|
|
769
|
+
* @returns Resolves when the check is done
|
|
770
|
+
*/
|
|
771
|
+
async checkOrcHarness() {
|
|
772
|
+
await this.checkHarnesses();
|
|
773
|
+
}
|
|
774
|
+
/**
|
|
775
|
+
* One harness of the periodic check.
|
|
776
|
+
*
|
|
777
|
+
* @param harnessId - Harness
|
|
778
|
+
* @param inUse - The orchestrator or a configured agent runs on it
|
|
779
|
+
*/
|
|
780
|
+
async checkOne(harnessId, inUse) {
|
|
781
|
+
const passive = await this.checkLoginState(harnessId).catch(() => 'unknown');
|
|
782
|
+
if (passive === 'logged_out') {
|
|
783
|
+
const running = this.resumer?.listSessions(harnessId) ?? [];
|
|
784
|
+
// A machine that was never logged in is onboarding's job, not a re-login.
|
|
785
|
+
if (this.state.get(harnessId).seenLoggedInAt === undefined && running.length === 0 && !inUse)
|
|
786
|
+
return;
|
|
787
|
+
this.reportExpiry({ harnessId, sessionName: null, source: 'status' });
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
if (passive === 'logged_in')
|
|
791
|
+
this.markSeenLoggedIn(harnessId);
|
|
792
|
+
if (!this.verifyLogin || !inUse)
|
|
793
|
+
return;
|
|
794
|
+
const last = this.probeCache.get(harnessId);
|
|
795
|
+
if (last && this.now() - last.at < HARNESS_CONSTANTS.RELOGIN.PROBE_INTERVAL_MS)
|
|
796
|
+
return;
|
|
797
|
+
const probed = await this.probe(harnessId, true);
|
|
798
|
+
if (probed === 'logged_out')
|
|
799
|
+
this.reportExpiry({ harnessId, sessionName: null, source: 'status' });
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Start the periodic check (first run shortly after boot, so a machine
|
|
803
|
+
* that comes up signed out says so without waiting a full interval).
|
|
571
804
|
*
|
|
572
805
|
* @param intervalMs - Cadence (defaults to STATUS_CHECK_INTERVAL_MS)
|
|
806
|
+
* @param firstDelayMs - Delay of the first check (default 60 s)
|
|
573
807
|
*/
|
|
574
|
-
start(intervalMs = HARNESS_CONSTANTS.RELOGIN.STATUS_CHECK_INTERVAL_MS) {
|
|
808
|
+
start(intervalMs = HARNESS_CONSTANTS.RELOGIN.STATUS_CHECK_INTERVAL_MS, firstDelayMs = 60_000) {
|
|
575
809
|
if (this.statusTimer)
|
|
576
810
|
return;
|
|
577
|
-
this.statusTimer = setInterval(() => void this.
|
|
811
|
+
this.statusTimer = setInterval(() => void this.checkHarnesses(), intervalMs);
|
|
578
812
|
this.statusTimer.unref?.();
|
|
813
|
+
this.firstCheckTimer = setTimeout(() => void this.checkHarnesses(), firstDelayMs);
|
|
814
|
+
this.firstCheckTimer.unref?.();
|
|
579
815
|
}
|
|
580
816
|
/** Stop the periodic check and forget every flow's timers. */
|
|
581
817
|
stop() {
|
|
582
818
|
if (this.statusTimer)
|
|
583
819
|
clearInterval(this.statusTimer);
|
|
820
|
+
if (this.firstCheckTimer)
|
|
821
|
+
clearTimeout(this.firstCheckTimer);
|
|
584
822
|
this.statusTimer = null;
|
|
823
|
+
this.firstCheckTimer = null;
|
|
585
824
|
for (const flow of this.flows.values())
|
|
586
825
|
this.clearScreenTimer(flow);
|
|
587
826
|
}
|
|
588
827
|
/**
|
|
589
|
-
*
|
|
828
|
+
* Confirm a suspected expiry, then tell the owner (or use a stored key).
|
|
829
|
+
*
|
|
830
|
+
* @param harnessId - Harness
|
|
831
|
+
*/
|
|
832
|
+
async confirmExpiry(harnessId) {
|
|
833
|
+
try {
|
|
834
|
+
const state = await this.currentState(harnessId);
|
|
835
|
+
const stuck = this.confirming.get(harnessId) ?? new Set();
|
|
836
|
+
if (state !== 'logged_out') {
|
|
837
|
+
// An agent's "401" is not proof the login expired — a bad API key
|
|
838
|
+
// or text typed into the sign-in screen gives one too, and
|
|
839
|
+
// `codex login` revokes a working login when it starts (2026-09-26,
|
|
840
|
+
// Nova). `unknown` (probe failed) is not proof either.
|
|
841
|
+
this.quietUntil.set(harnessId, this.now() + HARNESS_CONSTANTS.RELOGIN.NOT_EXPIRED_QUIET_MS);
|
|
842
|
+
this.logger.warn(state === 'logged_in'
|
|
843
|
+
? 'Re-login skipped: the harness is still signed in (the error was not an expired login)'
|
|
844
|
+
: 'Re-login skipped: the harness could not be confirmed as signed out', { harnessId, state, stuck: [...stuck] });
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
this.markSignedOut(harnessId);
|
|
848
|
+
// Names for the notice ("so 2 agents can't work (Crewly Orc, Ella)").
|
|
849
|
+
await this.refreshAgents();
|
|
850
|
+
const current = this.flows.get(harnessId);
|
|
851
|
+
if (current && (current.phase === 'running' || current.remind)) {
|
|
852
|
+
for (const name of stuck)
|
|
853
|
+
current.stuck.add(name);
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
const flow = this.newFlow(harnessId, stuck, EXPIRY_FLOW, 'signed_out');
|
|
857
|
+
flow.remind = true;
|
|
858
|
+
if (current) {
|
|
859
|
+
flow.replyTarget = current.replyTarget;
|
|
860
|
+
flow.lastOutcome = current.lastOutcome;
|
|
861
|
+
}
|
|
862
|
+
this.flows.set(harnessId, flow);
|
|
863
|
+
if (await this.tryStoredApiKey(flow))
|
|
864
|
+
return;
|
|
865
|
+
if (this.flows.get(harnessId) !== flow)
|
|
866
|
+
return;
|
|
867
|
+
this.logger.warn('Harness is signed out — telling the owner', { harnessId, stuck: [...flow.stuck] });
|
|
868
|
+
await this.sendSignedOutNotice(flow, 'notice');
|
|
869
|
+
}
|
|
870
|
+
catch (error) {
|
|
871
|
+
this.logger.warn('Confirming a harness expiry failed', { harnessId, error: error instanceof Error ? error.message : String(error) });
|
|
872
|
+
}
|
|
873
|
+
finally {
|
|
874
|
+
this.confirming.delete(harnessId);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Login state: the status command, then the live probe when the status
|
|
879
|
+
* command cannot see an expiry (it says logged in / unknown).
|
|
880
|
+
*
|
|
881
|
+
* @param harnessId - Harness
|
|
882
|
+
* @param fresh - Ignore a cached probe result
|
|
883
|
+
* @returns The best-known state
|
|
884
|
+
*/
|
|
885
|
+
async currentState(harnessId, fresh = false) {
|
|
886
|
+
const passive = await this.checkLoginState(harnessId).catch(() => 'unknown');
|
|
887
|
+
if (passive === 'logged_out')
|
|
888
|
+
return 'logged_out';
|
|
889
|
+
const probed = await this.probe(harnessId, fresh);
|
|
890
|
+
return probed === 'unknown' ? passive : probed;
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* Run (or reuse) the live probe.
|
|
894
|
+
*
|
|
895
|
+
* @param harnessId - Harness
|
|
896
|
+
* @param fresh - Ignore a cached result
|
|
897
|
+
* @returns Probe result (`unknown` when there is no probe)
|
|
898
|
+
*/
|
|
899
|
+
async probe(harnessId, fresh) {
|
|
900
|
+
if (!this.verifyLogin)
|
|
901
|
+
return 'unknown';
|
|
902
|
+
const cached = this.probeCache.get(harnessId);
|
|
903
|
+
if (!fresh && cached && this.now() - cached.at < HARNESS_CONSTANTS.RELOGIN.PROBE_CACHE_MS)
|
|
904
|
+
return cached.state;
|
|
905
|
+
const inFlight = this.probeInFlight.get(harnessId);
|
|
906
|
+
if (inFlight)
|
|
907
|
+
return inFlight;
|
|
908
|
+
const verify = this.verifyLogin;
|
|
909
|
+
const run = (async () => {
|
|
910
|
+
try {
|
|
911
|
+
const state = await verify(harnessId).catch(() => 'unknown');
|
|
912
|
+
this.probeCache.set(harnessId, { at: this.now(), state });
|
|
913
|
+
if (state === 'logged_in')
|
|
914
|
+
this.markSeenLoggedIn(harnessId);
|
|
915
|
+
this.logger.info('Harness sign-in probe', { harnessId, state });
|
|
916
|
+
return state;
|
|
917
|
+
}
|
|
918
|
+
finally {
|
|
919
|
+
this.probeInFlight.delete(harnessId);
|
|
920
|
+
}
|
|
921
|
+
})();
|
|
922
|
+
this.probeInFlight.set(harnessId, run);
|
|
923
|
+
return run;
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Re-remind the owner of a signed-out harness once its backoff elapsed. If
|
|
927
|
+
* the harness is signed in again by then (the owner did it on the
|
|
928
|
+
* machine or in Setup), the flow finishes and the agents resume instead.
|
|
929
|
+
*
|
|
930
|
+
* @param flow - A signed-out flow
|
|
931
|
+
*/
|
|
932
|
+
async maybeRemind(flow) {
|
|
933
|
+
const { harnessId } = flow;
|
|
934
|
+
if (!flow.remind || this.reminding.has(harnessId) || !this.noticeDue(harnessId))
|
|
935
|
+
return;
|
|
936
|
+
this.reminding.add(harnessId);
|
|
937
|
+
try {
|
|
938
|
+
const state = await this.currentState(harnessId);
|
|
939
|
+
if (this.flows.get(harnessId) !== flow || flow.phase !== 'signed_out')
|
|
940
|
+
return;
|
|
941
|
+
if (state === 'logged_in') {
|
|
942
|
+
this.logger.info('Re-login: the harness is signed in again; resuming the waiting agents', { harnessId });
|
|
943
|
+
await this.succeed(flow, true);
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
if (state !== 'logged_out')
|
|
947
|
+
return;
|
|
948
|
+
this.logger.info('Re-login: reminding the owner', { harnessId, lastOutcome: flow.lastOutcome });
|
|
949
|
+
await this.sendSignedOutNotice(flow, flow.lastOutcome === 'timed_out' ? 'link_expired' : 'reminder');
|
|
950
|
+
}
|
|
951
|
+
finally {
|
|
952
|
+
this.reminding.delete(harnessId);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
/**
|
|
956
|
+
* Whether the backoff allows another "signed out" DM for a harness.
|
|
957
|
+
*
|
|
958
|
+
* @param harnessId - Harness
|
|
959
|
+
* @returns True when no notice was sent yet, or the backoff elapsed
|
|
960
|
+
*/
|
|
961
|
+
noticeDue(harnessId) {
|
|
962
|
+
const record = this.state.get(harnessId);
|
|
963
|
+
if (record.lastNoticeAt === undefined)
|
|
964
|
+
return true;
|
|
965
|
+
const count = Math.max(1, record.noticeCount ?? 1);
|
|
966
|
+
const { REMIND_BACKOFF_BASE_MS, REMIND_BACKOFF_MAX_MS } = HARNESS_CONSTANTS.RELOGIN;
|
|
967
|
+
const gap = Math.min(REMIND_BACKOFF_BASE_MS * 2 ** (count - 1), REMIND_BACKOFF_MAX_MS);
|
|
968
|
+
return this.now() - record.lastNoticeAt >= gap;
|
|
969
|
+
}
|
|
970
|
+
/**
|
|
971
|
+
* Send the "signed out" DM (first notice or re-reminder), within the backoff.
|
|
972
|
+
*
|
|
973
|
+
* @param flow - The flow
|
|
974
|
+
* @param kind - Which text
|
|
975
|
+
*/
|
|
976
|
+
async sendSignedOutNotice(flow, kind) {
|
|
977
|
+
const { harnessId } = flow;
|
|
978
|
+
if (!this.noticeDue(harnessId)) {
|
|
979
|
+
this.logger.debug('Re-login: owner already told; next reminder after the backoff', { harnessId });
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
const record = this.state.get(harnessId);
|
|
983
|
+
// A restart forgets the flow but not the notice: the next one is a reminder.
|
|
984
|
+
const text = formatSignedOutDm(harnessId, this.machineName(), this.agentNamesOn(flow), kind === 'notice' && record.lastNoticeAt ? 'reminder' : kind);
|
|
985
|
+
const delivered = await this.dm(flow, text);
|
|
986
|
+
if (!delivered)
|
|
987
|
+
return;
|
|
988
|
+
this.state.update(harnessId, { lastNoticeAt: this.now(), noticeCount: (record.noticeCount ?? 0) + 1 });
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* Replace a harness's flow with a new one and start its broker session.
|
|
992
|
+
*
|
|
993
|
+
* @param harnessId - Harness
|
|
994
|
+
* @param previous - The flow it replaces (its stuck agents are kept), or null
|
|
995
|
+
* @param options - Trigger, wording and reply target
|
|
996
|
+
*/
|
|
997
|
+
replaceAndStart(harnessId, previous, options) {
|
|
998
|
+
if (previous) {
|
|
999
|
+
this.clearScreenTimer(previous);
|
|
1000
|
+
this.cancelOwnSession(previous);
|
|
1001
|
+
if (this.flows.get(harnessId) === previous)
|
|
1002
|
+
this.flows.delete(harnessId);
|
|
1003
|
+
}
|
|
1004
|
+
// The owner asked: a quiet period after the last login must not swallow it.
|
|
1005
|
+
this.quietUntil.delete(harnessId);
|
|
1006
|
+
const flow = this.newFlow(harnessId, new Set(previous?.stuck ?? []), options, 'running');
|
|
1007
|
+
flow.remind = previous?.remind ?? false;
|
|
1008
|
+
this.flows.set(harnessId, flow);
|
|
1009
|
+
this.startBrokerSession(flow);
|
|
1010
|
+
}
|
|
1011
|
+
/**
|
|
1012
|
+
* A new flow record.
|
|
590
1013
|
*
|
|
591
1014
|
* @param harnessId - Harness
|
|
592
1015
|
* @param stuck - Sessions known to be stuck
|
|
593
|
-
* @param options - Trigger, wording and reply target
|
|
1016
|
+
* @param options - Trigger, wording and reply target
|
|
1017
|
+
* @param phase - Initial phase
|
|
1018
|
+
* @returns The flow (not registered)
|
|
594
1019
|
*/
|
|
595
|
-
|
|
596
|
-
|
|
1020
|
+
newFlow(harnessId, stuck, options, phase) {
|
|
1021
|
+
return {
|
|
597
1022
|
harnessId,
|
|
598
|
-
phase
|
|
1023
|
+
phase,
|
|
599
1024
|
sessionId: null,
|
|
600
1025
|
startedAt: this.now(),
|
|
601
1026
|
stuck,
|
|
@@ -608,45 +1033,11 @@ export class HarnessReloginService {
|
|
|
608
1033
|
trigger: options.trigger,
|
|
609
1034
|
switchAccount: options.switchAccount,
|
|
610
1035
|
replyTarget: options.replyTarget,
|
|
1036
|
+
linkChannelId: null,
|
|
611
1037
|
lastOutcome: null,
|
|
1038
|
+
remind: false,
|
|
1039
|
+
restartAll: true,
|
|
612
1040
|
};
|
|
613
|
-
// Registered synchronously: this is the per-harness debounce.
|
|
614
|
-
this.flows.set(harnessId, flow);
|
|
615
|
-
void this.run(flow);
|
|
616
|
-
}
|
|
617
|
-
/**
|
|
618
|
-
* Body of a flow (async part of {@link begin}).
|
|
619
|
-
*
|
|
620
|
-
* @param flow - The flow
|
|
621
|
-
*/
|
|
622
|
-
async run(flow) {
|
|
623
|
-
// The owner asked for this login (possibly to switch accounts): no
|
|
624
|
-
// "still logged in" check and no silent API key — they want the link.
|
|
625
|
-
if (flow.trigger === 'owner') {
|
|
626
|
-
this.startBrokerSession(flow);
|
|
627
|
-
return;
|
|
628
|
-
}
|
|
629
|
-
// An agent's "401" is not proof the login expired — a bad API key or
|
|
630
|
-
// text typed into the sign-in screen gives one too. Starting a login
|
|
631
|
-
// then *logs the harness out* (`codex login` clears its credentials):
|
|
632
|
-
// that is how a fresh ChatGPT sign-in was wiped twice in a row
|
|
633
|
-
// (2026-09-26, Nova). Only a harness that is not logged in gets one.
|
|
634
|
-
const state = await this.checkLoginState(flow.harnessId).catch(() => 'unknown');
|
|
635
|
-
if (state !== 'logged_out') {
|
|
636
|
-
// `unknown` (status probe failed or timed out) is not proof either:
|
|
637
|
-
// starting `codex login` would revoke a login that may still work.
|
|
638
|
-
this.flows.delete(flow.harnessId);
|
|
639
|
-
this.quietUntil.set(flow.harnessId, this.now() + HARNESS_CONSTANTS.RELOGIN.NOT_EXPIRED_QUIET_MS);
|
|
640
|
-
this.logger.warn(state === 'logged_in'
|
|
641
|
-
? 'Re-login skipped: the harness is still logged in (the error was not an expired login)'
|
|
642
|
-
: 'Re-login skipped: the harness login state could not be confirmed as logged out', { harnessId: flow.harnessId, state, stuck: [...flow.stuck] });
|
|
643
|
-
return;
|
|
644
|
-
}
|
|
645
|
-
if (await this.tryStoredApiKey(flow))
|
|
646
|
-
return;
|
|
647
|
-
if (this.flows.get(flow.harnessId) !== flow)
|
|
648
|
-
return;
|
|
649
|
-
this.startBrokerSession(flow);
|
|
650
1041
|
}
|
|
651
1042
|
/**
|
|
652
1043
|
* Start (or adopt) the harness's broker login session for a flow.
|
|
@@ -664,8 +1055,8 @@ export class HarnessReloginService {
|
|
|
664
1055
|
catch (error) {
|
|
665
1056
|
const reason = error instanceof Error ? error.message : String(error);
|
|
666
1057
|
this.logger.warn('Re-login: could not start the login', { harnessId: flow.harnessId, error: reason });
|
|
667
|
-
flow.phase = '
|
|
668
|
-
void this.dm(flow, formatFailureDm(flow.harnessId, `Crewly
|
|
1058
|
+
flow.phase = 'signed_out';
|
|
1059
|
+
void this.dm(flow, formatFailureDm(flow.harnessId, `Crewly could not start the sign-in (${reason})`));
|
|
669
1060
|
return;
|
|
670
1061
|
}
|
|
671
1062
|
flow.sessionId = session.id;
|
|
@@ -733,7 +1124,12 @@ export class HarnessReloginService {
|
|
|
733
1124
|
if (flow.dm !== 'link' && linkReady) {
|
|
734
1125
|
flow.dm = 'link';
|
|
735
1126
|
this.clearScreenTimer(flow);
|
|
736
|
-
void this.dm(flow, formatLinkDm(session,
|
|
1127
|
+
void this.dm(flow, formatLinkDm(session, [], this.wordingOf(flow))).then((delivered) => {
|
|
1128
|
+
if (typeof delivered === 'object' && delivered)
|
|
1129
|
+
flow.linkChannelId = delivered.channelId;
|
|
1130
|
+
else if (delivered && flow.replyTarget)
|
|
1131
|
+
flow.linkChannelId = flow.replyTarget.channelId;
|
|
1132
|
+
});
|
|
737
1133
|
return;
|
|
738
1134
|
}
|
|
739
1135
|
if (flow.replied && session.state === 'awaiting_user' && session.message && session.message !== flow.lastRejection) {
|
|
@@ -749,8 +1145,13 @@ export class HarnessReloginService {
|
|
|
749
1145
|
*/
|
|
750
1146
|
handleFinished(session) {
|
|
751
1147
|
const flow = this.flows.get(session.harnessId);
|
|
752
|
-
if (!flow)
|
|
1148
|
+
if (!flow) {
|
|
1149
|
+
// Signed in from the dashboard / Setup with no flow here: the agents
|
|
1150
|
+
// parked on a sign-in screen still hold the dead login.
|
|
1151
|
+
if (session.state === 'succeeded')
|
|
1152
|
+
void this.adoptOutsideLogin(session.harnessId);
|
|
753
1153
|
return;
|
|
1154
|
+
}
|
|
754
1155
|
if (flow.sessionId !== session.id) {
|
|
755
1156
|
// The owner logged in another way (web / phone app) while a flow waited.
|
|
756
1157
|
if (session.state === 'succeeded') {
|
|
@@ -760,37 +1161,93 @@ export class HarnessReloginService {
|
|
|
760
1161
|
}
|
|
761
1162
|
return;
|
|
762
1163
|
}
|
|
763
|
-
if (flow.phase !== 'running')
|
|
1164
|
+
if (flow.phase !== 'running' || flow.cancelledByUs)
|
|
764
1165
|
return;
|
|
765
1166
|
this.clearScreenTimer(flow);
|
|
766
1167
|
flow.lastOutcome = session.state;
|
|
767
1168
|
if (session.state === 'succeeded') {
|
|
768
|
-
void this.
|
|
1169
|
+
void this.confirmAndSucceed(flow);
|
|
769
1170
|
return;
|
|
770
1171
|
}
|
|
771
|
-
flow.phase = '
|
|
772
|
-
|
|
773
|
-
|
|
1172
|
+
flow.phase = 'signed_out';
|
|
1173
|
+
flow.sessionId = null;
|
|
1174
|
+
flow.dm = 'none';
|
|
1175
|
+
flow.remind = flow.remind || this.isSignedOut(flow.harnessId);
|
|
774
1176
|
if (session.state === 'cancelled') {
|
|
775
1177
|
// Cancelled on the web or by a shutdown: the owner knows; stay quiet.
|
|
776
1178
|
this.logger.info('Re-login cancelled', { harnessId: flow.harnessId, sessionId: session.id });
|
|
777
1179
|
return;
|
|
778
1180
|
}
|
|
779
1181
|
this.logger.warn('Re-login did not finish', { harnessId: flow.harnessId, sessionId: session.id, state: session.state });
|
|
780
|
-
|
|
1182
|
+
const text = session.state === 'timed_out'
|
|
1183
|
+
? formatSignedOutDm(flow.harnessId, this.machineName(), this.agentNamesOn(flow), 'link_expired')
|
|
1184
|
+
: formatFailureDm(flow.harnessId, session.message);
|
|
1185
|
+
void this.dm(flow, text);
|
|
781
1186
|
}
|
|
782
1187
|
/**
|
|
783
|
-
*
|
|
1188
|
+
* A login finished outside any flow (dashboard chip, Setup, CLI): restart
|
|
1189
|
+
* only the sessions that sit at a sign-in screen and re-deliver what
|
|
1190
|
+
* waited for them. Working agents of the harness are left alone.
|
|
1191
|
+
*
|
|
1192
|
+
* @param harnessId - Harness that was signed in
|
|
1193
|
+
*/
|
|
1194
|
+
async adoptOutsideLogin(harnessId) {
|
|
1195
|
+
if (!hasBrokerLogin(harnessId))
|
|
1196
|
+
return;
|
|
1197
|
+
let live = [];
|
|
1198
|
+
try {
|
|
1199
|
+
live = this.resumer?.listSessions(harnessId) ?? [];
|
|
1200
|
+
}
|
|
1201
|
+
catch {
|
|
1202
|
+
live = [];
|
|
1203
|
+
}
|
|
1204
|
+
const waiting = live.filter((name) => this.sessionNeedsLogin?.(name) === true);
|
|
1205
|
+
const wasSignedOut = this.isSignedOut(harnessId);
|
|
1206
|
+
if (waiting.length === 0 && !wasSignedOut) {
|
|
1207
|
+
this.markSeenLoggedIn(harnessId);
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
const flow = this.newFlow(harnessId, new Set(waiting), { trigger: 'expiry', switchAccount: false, replyTarget: null }, 'running');
|
|
1211
|
+
flow.restartAll = false;
|
|
1212
|
+
this.logger.info('Signed in outside Slack; resuming the agents that waited', { harnessId, waiting });
|
|
1213
|
+
await this.succeed(flow, wasSignedOut);
|
|
1214
|
+
}
|
|
1215
|
+
/**
|
|
1216
|
+
* The broker reported success: confirm it with the probe, then finish.
|
|
1217
|
+
*
|
|
1218
|
+
* @param flow - The flow
|
|
1219
|
+
*/
|
|
1220
|
+
async confirmAndSucceed(flow) {
|
|
1221
|
+
const state = await this.probe(flow.harnessId, true);
|
|
1222
|
+
if (this.flows.get(flow.harnessId) !== flow)
|
|
1223
|
+
return;
|
|
1224
|
+
if (state === 'logged_out') {
|
|
1225
|
+
this.logger.warn('Re-login: the sign-in finished but the harness still cannot reach its API', { harnessId: flow.harnessId });
|
|
1226
|
+
flow.phase = 'signed_out';
|
|
1227
|
+
flow.sessionId = null;
|
|
1228
|
+
flow.dm = 'none';
|
|
1229
|
+
flow.remind = true;
|
|
1230
|
+
this.markSignedOut(flow.harnessId);
|
|
1231
|
+
await this.dm(flow, formatFailureDm(flow.harnessId, 'the sign-in finished, but it still does not work'));
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
await this.succeed(flow, true);
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* The login is back: restart the stuck agents, re-deliver what waited,
|
|
1238
|
+
* and tell the owner.
|
|
784
1239
|
*
|
|
785
1240
|
* @param flow - The flow
|
|
786
1241
|
* @param notify - DM the owner (false for the silent API-key path)
|
|
787
1242
|
*/
|
|
788
1243
|
async succeed(flow, notify) {
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
this.
|
|
793
|
-
|
|
1244
|
+
const { harnessId } = flow;
|
|
1245
|
+
if (this.flows.get(harnessId) === flow)
|
|
1246
|
+
this.flows.delete(harnessId);
|
|
1247
|
+
this.quietUntil.set(harnessId, this.now() + HARNESS_CONSTANTS.RELOGIN.POST_SUCCESS_QUIET_MS);
|
|
1248
|
+
this.probeCache.set(harnessId, { at: this.now(), state: 'logged_in' });
|
|
1249
|
+
this.state.update(harnessId, { signedOutSince: undefined, lastNoticeAt: undefined, noticeCount: undefined, seenLoggedInAt: this.now() });
|
|
1250
|
+
const sessions = this.sessionsToRestart(flow);
|
|
794
1251
|
let result = { resumed: [], failed: [] };
|
|
795
1252
|
if (this.resumer && sessions.length > 0) {
|
|
796
1253
|
try {
|
|
@@ -803,57 +1260,117 @@ export class HarnessReloginService {
|
|
|
803
1260
|
}
|
|
804
1261
|
for (const name of result.resumed)
|
|
805
1262
|
this.mutedScreens.add(name);
|
|
806
|
-
|
|
1263
|
+
let redelivered = 0;
|
|
1264
|
+
if (this.onLoginRestored) {
|
|
1265
|
+
try {
|
|
1266
|
+
redelivered = (await this.onLoginRestored(harnessId, result.resumed)) || 0;
|
|
1267
|
+
}
|
|
1268
|
+
catch (error) {
|
|
1269
|
+
this.logger.warn('Re-login: re-delivering waiting messages failed', { error: error instanceof Error ? error.message : String(error) });
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
this.logger.info('Re-login finished', { harnessId, resumed: result.resumed.length, failed: result.failed.length, redelivered });
|
|
807
1273
|
if (notify) {
|
|
808
|
-
const text = flow.trigger === 'owner'
|
|
1274
|
+
const text = flow.trigger === 'owner'
|
|
1275
|
+
? formatOwnerSuccessDm(harnessId, result, redelivered)
|
|
1276
|
+
: formatSuccessDm(harnessId, result, redelivered, this.machineName());
|
|
809
1277
|
await this.dm(flow, text);
|
|
810
1278
|
}
|
|
811
1279
|
}
|
|
812
1280
|
/**
|
|
813
|
-
*
|
|
814
|
-
*
|
|
815
|
-
* stuck agents; otherwise send a text reminder. No login is started here —
|
|
816
|
-
* the owner's `relogin` reply starts one while they are there to use it.
|
|
1281
|
+
* Sessions to restart after a login: the ones whose output matched plus
|
|
1282
|
+
* every live session of the harness (they all hold the dead login).
|
|
817
1283
|
*
|
|
818
|
-
* @param flow -
|
|
1284
|
+
* @param flow - The flow
|
|
1285
|
+
* @returns Session names
|
|
819
1286
|
*/
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
if (this.flows.get(flow.harnessId) !== flow || flow.phase !== 'failed')
|
|
825
|
-
return;
|
|
826
|
-
if (state === 'logged_in') {
|
|
827
|
-
this.logger.info('Re-login: the harness is logged in again; resuming the waiting agents', { harnessId: flow.harnessId });
|
|
828
|
-
await this.succeed(flow, true);
|
|
829
|
-
return;
|
|
1287
|
+
sessionsToRestart(flow) {
|
|
1288
|
+
let live = [];
|
|
1289
|
+
try {
|
|
1290
|
+
live = this.resumer?.listSessions(flow.harnessId) ?? [];
|
|
830
1291
|
}
|
|
831
|
-
|
|
1292
|
+
catch {
|
|
1293
|
+
live = [];
|
|
1294
|
+
}
|
|
1295
|
+
if (!flow.restartAll)
|
|
1296
|
+
return [...flow.stuck];
|
|
1297
|
+
return [...new Set([...flow.stuck, ...live])];
|
|
832
1298
|
}
|
|
833
1299
|
/**
|
|
834
|
-
*
|
|
1300
|
+
* Names of the agents that run on a flow's harness, for the owner: every
|
|
1301
|
+
* configured agent on it, else the stuck / live ones.
|
|
835
1302
|
*
|
|
836
|
-
* @param flow - The flow
|
|
1303
|
+
* @param flow - The flow
|
|
1304
|
+
* @returns Display names
|
|
837
1305
|
*/
|
|
838
|
-
|
|
839
|
-
this.
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
this.
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1306
|
+
agentNamesOn(flow) {
|
|
1307
|
+
const configured = this.agents.filter((agent) => agent.harnessId === flow.harnessId);
|
|
1308
|
+
if (configured.length > 0)
|
|
1309
|
+
return configured.map((agent) => agent.displayName || agent.sessionName);
|
|
1310
|
+
const byName = new Map(this.agents.map((agent) => [agent.sessionName, agent.displayName || agent.sessionName]));
|
|
1311
|
+
return this.sessionsToRestart(flow).map((name) => byName.get(name) ?? name);
|
|
1312
|
+
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Re-read the configured agents.
|
|
1315
|
+
*
|
|
1316
|
+
* @returns The agents (the last known list when the read fails)
|
|
1317
|
+
*/
|
|
1318
|
+
async refreshAgents() {
|
|
1319
|
+
if (!this.listAgents)
|
|
1320
|
+
return this.agents;
|
|
1321
|
+
try {
|
|
1322
|
+
this.agents = await this.listAgents();
|
|
1323
|
+
}
|
|
1324
|
+
catch (error) {
|
|
1325
|
+
this.logger.debug('Listing configured agents failed', { error: error instanceof Error ? error.message : String(error) });
|
|
1326
|
+
}
|
|
1327
|
+
return this.agents;
|
|
1328
|
+
}
|
|
1329
|
+
/**
|
|
1330
|
+
* Harness an agent session runs on.
|
|
1331
|
+
*
|
|
1332
|
+
* @param sessionName - Agent session
|
|
1333
|
+
* @returns Harness id, or null when unknown
|
|
1334
|
+
*/
|
|
1335
|
+
harnessOfSession(sessionName) {
|
|
1336
|
+
const configured = this.agents.find((agent) => agent.sessionName === sessionName);
|
|
1337
|
+
if (configured && isHarnessId(configured.harnessId))
|
|
1338
|
+
return configured.harnessId;
|
|
1339
|
+
for (const flow of this.flows.values())
|
|
1340
|
+
if (flow.stuck.has(sessionName))
|
|
1341
|
+
return flow.harnessId;
|
|
1342
|
+
return null;
|
|
1343
|
+
}
|
|
1344
|
+
/**
|
|
1345
|
+
* Record that a harness is signed out (first time only keeps the date).
|
|
1346
|
+
*
|
|
1347
|
+
* @param harnessId - Harness
|
|
1348
|
+
*/
|
|
1349
|
+
markSignedOut(harnessId) {
|
|
1350
|
+
if (this.state.get(harnessId).signedOutSince === undefined)
|
|
1351
|
+
this.state.update(harnessId, { signedOutSince: this.now() });
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* Remember that a harness was seen signed in (written at most hourly).
|
|
1355
|
+
*
|
|
1356
|
+
* @param harnessId - Harness
|
|
1357
|
+
*/
|
|
1358
|
+
markSeenLoggedIn(harnessId) {
|
|
1359
|
+
const record = this.state.get(harnessId);
|
|
1360
|
+
if (record.signedOutSince !== undefined)
|
|
1361
|
+
return;
|
|
1362
|
+
if (record.seenLoggedInAt !== undefined && this.now() - record.seenLoggedInAt < HARNESS_CONSTANTS.RELOGIN.PROBE_INTERVAL_MS)
|
|
1363
|
+
return;
|
|
1364
|
+
this.state.update(harnessId, { seenLoggedInAt: this.now() });
|
|
848
1365
|
}
|
|
849
1366
|
/**
|
|
850
1367
|
* DM wording for a flow.
|
|
851
1368
|
*
|
|
852
1369
|
* @param flow - The flow
|
|
853
|
-
* @returns
|
|
1370
|
+
* @returns Account-switch flag and machine name
|
|
854
1371
|
*/
|
|
855
1372
|
wordingOf(flow) {
|
|
856
|
-
return { ownerRequested: flow.trigger === 'owner', switchAccount: flow.switchAccount };
|
|
1373
|
+
return { ownerRequested: flow.trigger === 'owner', switchAccount: flow.switchAccount, machine: this.machineName() };
|
|
857
1374
|
}
|
|
858
1375
|
/**
|
|
859
1376
|
* Cancel the flow's live broker session, marking it as cancelled by the
|
|
@@ -886,56 +1403,50 @@ export class HarnessReloginService {
|
|
|
886
1403
|
return;
|
|
887
1404
|
flow.dm = 'screen';
|
|
888
1405
|
this.logger.warn('Re-login: login screen not recognised, sending it to the owner', { harnessId: flow.harnessId, sessionId: session.id });
|
|
889
|
-
await this.dm(flow, formatScreenDm(session,
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
* Agents waiting on a flow: the stuck ones, else every session of the harness.
|
|
893
|
-
*
|
|
894
|
-
* @param flow - The flow
|
|
895
|
-
* @returns Session names
|
|
896
|
-
*/
|
|
897
|
-
waitingAgents(flow) {
|
|
898
|
-
if (flow.stuck.size > 0)
|
|
899
|
-
return [...flow.stuck];
|
|
900
|
-
try {
|
|
901
|
-
return this.resumer?.listSessions(flow.harnessId) ?? [];
|
|
902
|
-
}
|
|
903
|
-
catch {
|
|
904
|
-
return [];
|
|
905
|
-
}
|
|
1406
|
+
const delivered = await this.dm(flow, formatScreenDm(session, [], this.wordingOf(flow)));
|
|
1407
|
+
if (typeof delivered === 'object' && delivered)
|
|
1408
|
+
flow.linkChannelId = delivered.channelId;
|
|
906
1409
|
}
|
|
907
1410
|
/**
|
|
908
|
-
* DM the owner
|
|
1411
|
+
* DM the owner in the flow's conversation, and remember where it landed
|
|
1412
|
+
* so later DMs (the link, the result) follow it.
|
|
909
1413
|
*
|
|
910
1414
|
* @param flow - The flow
|
|
911
1415
|
* @param text - DM text
|
|
1416
|
+
* @returns The delivery
|
|
912
1417
|
*/
|
|
913
1418
|
async dm(flow, text) {
|
|
914
1419
|
flow.lastDmAt = this.now();
|
|
915
|
-
await this.notify(text, flow.replyTarget, flow.harnessId);
|
|
1420
|
+
const delivered = await this.notify(text, flow.replyTarget, flow.harnessId);
|
|
1421
|
+
if (typeof delivered === 'object' && delivered && !flow.replyTarget)
|
|
1422
|
+
flow.replyTarget = delivered;
|
|
1423
|
+
return delivered;
|
|
916
1424
|
}
|
|
917
1425
|
/**
|
|
918
1426
|
* Send a message to the owner (logs when no notifier is wired; never logs the text).
|
|
919
1427
|
*
|
|
920
1428
|
* @param text - DM text
|
|
921
|
-
* @param target - Conversation to answer in (null =
|
|
1429
|
+
* @param target - Conversation to answer in (null = this machine's DM)
|
|
922
1430
|
* @param harnessId - Harness, for logs
|
|
1431
|
+
* @returns The delivery (false when not delivered)
|
|
923
1432
|
*/
|
|
924
1433
|
async notify(text, target, harnessId) {
|
|
925
1434
|
if (!this.notifier) {
|
|
926
1435
|
this.logger.warn('Re-login: no Slack DM path to the owner; finish the login from Setup', { harnessId });
|
|
927
|
-
return;
|
|
1436
|
+
return false;
|
|
928
1437
|
}
|
|
929
1438
|
try {
|
|
930
1439
|
const delivered = await this.notifier.sendToOwner(text, target);
|
|
931
1440
|
if (!delivered)
|
|
932
1441
|
this.logger.warn('Re-login: the owner DM was not delivered', { harnessId });
|
|
1442
|
+
return delivered;
|
|
933
1443
|
}
|
|
934
1444
|
catch (error) {
|
|
935
1445
|
this.logger.warn('Re-login: sending the owner DM failed', {
|
|
936
1446
|
harnessId,
|
|
937
1447
|
error: error instanceof Error ? error.message : String(error),
|
|
938
1448
|
});
|
|
1449
|
+
return false;
|
|
939
1450
|
}
|
|
940
1451
|
}
|
|
941
1452
|
/**
|
|
@@ -967,8 +1478,9 @@ export class HarnessReloginService {
|
|
|
967
1478
|
let instance = null;
|
|
968
1479
|
/**
|
|
969
1480
|
* The backend's re-login coordinator, bound to the backend harness service
|
|
970
|
-
* (its broker, credentials, API keys and status)
|
|
971
|
-
*
|
|
1481
|
+
* (its broker, credentials, API keys and status), the live sign-in probe and
|
|
1482
|
+
* the persisted notice state. Registers itself as the `reloginPending`
|
|
1483
|
+
* source of `GET /api/harness`.
|
|
972
1484
|
*
|
|
973
1485
|
* @returns The singleton
|
|
974
1486
|
*/
|
|
@@ -988,7 +1500,9 @@ export function getHarnessReloginService() {
|
|
|
988
1500
|
return 'unknown';
|
|
989
1501
|
return (await harness.status.getLoginInfo(def, installed.path)).loginState;
|
|
990
1502
|
},
|
|
1503
|
+
verifyLogin: createHarnessLoginProbe(),
|
|
991
1504
|
getOrcHarness: () => harness.orc.get(),
|
|
1505
|
+
state: createDefaultReloginStateStore(),
|
|
992
1506
|
logger: LoggerService.getInstance().createComponentLogger('HarnessRelogin'),
|
|
993
1507
|
});
|
|
994
1508
|
harness.setReloginPendingProvider((harnessId) => service.getPending(harnessId));
|