claude-slack-channel-bots 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/session-manager.ts +429 -157
package/package.json
CHANGED
package/src/session-manager.ts
CHANGED
|
@@ -31,6 +31,8 @@ import {
|
|
|
31
31
|
ErrNoSessionId,
|
|
32
32
|
ErrSpawnNotFound,
|
|
33
33
|
ErrSpawnNotResumable,
|
|
34
|
+
ErrTmuxSendKeys,
|
|
35
|
+
ErrTmuxSessionCreate,
|
|
34
36
|
} from 'agent-director'
|
|
35
37
|
import type { ListRow, SpawnParams } from 'agent-director'
|
|
36
38
|
import type { WebClient } from '@slack/web-api'
|
|
@@ -179,9 +181,50 @@ export function flushSpawnFailureQueue(web: WebClient): void {
|
|
|
179
181
|
// reconnectMcp — send `/mcp reconnect <server-name>` via library sendKeys
|
|
180
182
|
// ---------------------------------------------------------------------------
|
|
181
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Ensure a tmux server exists on the default socket. Injectable seam so unit
|
|
186
|
+
* tests can assert the b.rmy self-heal path without spawning real processes.
|
|
187
|
+
* Default impl runs `tmux start-server` best-effort — never throws.
|
|
188
|
+
*
|
|
189
|
+
* b.rmy: after a container restart, /tmp (and thus the tmux socket) is wiped
|
|
190
|
+
* and nothing auto-starts tmux at boot, so every startup reconnect fails with
|
|
191
|
+
* `ErrTmuxSendKeys` ("no server running"). Starting the server before a retry
|
|
192
|
+
* gives the reconnect a chance to succeed instead of failing terminally.
|
|
193
|
+
*/
|
|
194
|
+
export type TmuxServerEnsurer = () => Promise<void>
|
|
195
|
+
|
|
196
|
+
const defaultEnsureTmuxServer: TmuxServerEnsurer = async (): Promise<void> => {
|
|
197
|
+
const { spawn } = await import('child_process')
|
|
198
|
+
await new Promise<void>((resolve) => {
|
|
199
|
+
try {
|
|
200
|
+
const child = spawn('tmux', ['start-server'], { stdio: 'ignore' })
|
|
201
|
+
child.on('error', () => resolve()) // tmux missing — best-effort
|
|
202
|
+
child.on('close', () => resolve())
|
|
203
|
+
} catch {
|
|
204
|
+
resolve()
|
|
205
|
+
}
|
|
206
|
+
})
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
let _ensureTmuxServer: TmuxServerEnsurer = defaultEnsureTmuxServer
|
|
210
|
+
|
|
211
|
+
/** Test-only seam: override the tmux-server ensurer. */
|
|
212
|
+
export function _setTmuxServerEnsurer(fn: TmuxServerEnsurer): void {
|
|
213
|
+
_ensureTmuxServer = fn
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Test-only seam: restore the default tmux-server ensurer. */
|
|
217
|
+
export function _resetTmuxServerEnsurer(): void {
|
|
218
|
+
_ensureTmuxServer = defaultEnsureTmuxServer
|
|
219
|
+
}
|
|
220
|
+
|
|
182
221
|
/**
|
|
183
222
|
* Send `/mcp reconnect <MCP_SERVER_NAME>` to the spawn's pane. Library's
|
|
184
223
|
* sendKeys appends Enter automatically per its contract.
|
|
224
|
+
*
|
|
225
|
+
* b.rmy self-heal: on `ErrTmuxSendKeys` (no tmux server / session — the
|
|
226
|
+
* post-reboot field failure), ensure a tmux server exists and retry the
|
|
227
|
+
* send-keys ONCE. The retry's outcome is the returned outcome.
|
|
185
228
|
*/
|
|
186
229
|
export async function reconnectMcp(
|
|
187
230
|
channelId: string,
|
|
@@ -190,14 +233,33 @@ export async function reconnectMcp(
|
|
|
190
233
|
): Promise<boolean> {
|
|
191
234
|
const claude_instance_id = instanceIdFor(channelId, routingConfig?.routes[channelId]?.normalizedName)
|
|
192
235
|
console.error(`[slack] reconnecting MCP server "${MCP_SERVER_NAME}": channel=${channelId}`)
|
|
193
|
-
|
|
194
|
-
|
|
236
|
+
const sendReconnect = (): Promise<unknown> =>
|
|
237
|
+
withOutageDetection(channelId, undefined, (client) => client.sendKeys({
|
|
195
238
|
claude_instance_id,
|
|
196
239
|
text: `/mcp reconnect ${MCP_SERVER_NAME}`,
|
|
197
240
|
}))
|
|
241
|
+
try {
|
|
242
|
+
await sendReconnect()
|
|
198
243
|
return true
|
|
199
244
|
} catch (err) {
|
|
200
245
|
if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) return false
|
|
246
|
+
if (err instanceof ErrTmuxSendKeys) {
|
|
247
|
+
console.error(
|
|
248
|
+
`[slack] reconnectMcp: ErrTmuxSendKeys for channel=${channelId} — ensuring tmux server exists and retrying send-keys once`,
|
|
249
|
+
)
|
|
250
|
+
await _ensureTmuxServer()
|
|
251
|
+
try {
|
|
252
|
+
await sendReconnect()
|
|
253
|
+
console.error(`[slack] reconnectMcp: retry succeeded after ErrTmuxSendKeys for channel=${channelId}`)
|
|
254
|
+
return true
|
|
255
|
+
} catch (err2) {
|
|
256
|
+
if (err2 instanceof ErrSystemInstallDisappeared || err2 instanceof ErrTmuxNotAvailable) return false
|
|
257
|
+
const e2 = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('send-keys', 'UnknownError', String(err2))
|
|
258
|
+
console.error(`[slack] reconnectMcp: retry after ErrTmuxSendKeys failed for channel=${channelId}: ${e2.errName}`)
|
|
259
|
+
postSpawnFailureToChannel(channelId, e2, web)
|
|
260
|
+
return false
|
|
261
|
+
}
|
|
262
|
+
}
|
|
201
263
|
const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('send-keys', 'UnknownError', String(err))
|
|
202
264
|
console.error(`[slack] reconnectMcp: send-keys failed for channel=${channelId}: ${e.errName}`)
|
|
203
265
|
postSpawnFailureToChannel(channelId, e, web)
|
|
@@ -206,7 +268,7 @@ export async function reconnectMcp(
|
|
|
206
268
|
}
|
|
207
269
|
|
|
208
270
|
// ---------------------------------------------------------------------------
|
|
209
|
-
//
|
|
271
|
+
// approvePreSessionDialogs — auto-approve pre-SessionStart dialogs on spawn
|
|
210
272
|
// ---------------------------------------------------------------------------
|
|
211
273
|
|
|
212
274
|
/**
|
|
@@ -216,19 +278,19 @@ export async function reconnectMcp(
|
|
|
216
278
|
*/
|
|
217
279
|
export const DEV_CHANNELS_DIALOG_NEEDLE = 'I am using this for local development'
|
|
218
280
|
|
|
219
|
-
/**
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
281
|
+
/**
|
|
282
|
+
* Verified against Claude Code 2.1.120 (2026-06-02). If this stops matching,
|
|
283
|
+
* the folder-trust dialog has drifted — see b.k54 / b.uhv. Match the option
|
|
284
|
+
* label (semantic, stable) rather than the header (cosmetic, drifts).
|
|
285
|
+
*/
|
|
286
|
+
export const TRUST_DIALOG_NEEDLE = 'Yes, I trust this folder'
|
|
224
287
|
|
|
225
|
-
/**
|
|
226
|
-
export const
|
|
288
|
+
/** Default poll interval while watching for pre-session dialogs. */
|
|
289
|
+
export const DIALOG_POLL_INTERVAL_MS = 500
|
|
227
290
|
|
|
228
291
|
let _dialogPollIntervalMs = DIALOG_POLL_INTERVAL_MS
|
|
229
|
-
let _dialogPollTimeoutMs = DIALOG_POLL_TIMEOUT_MS
|
|
230
292
|
|
|
231
|
-
/** Test-only seam: override the
|
|
293
|
+
/** Test-only seam: override the dialog poll interval. */
|
|
232
294
|
export function _setDialogPollIntervalMs(ms: number): void {
|
|
233
295
|
_dialogPollIntervalMs = ms
|
|
234
296
|
}
|
|
@@ -238,182 +300,256 @@ export function _resetDialogPollIntervalMs(): void {
|
|
|
238
300
|
_dialogPollIntervalMs = DIALOG_POLL_INTERVAL_MS
|
|
239
301
|
}
|
|
240
302
|
|
|
241
|
-
/**
|
|
242
|
-
|
|
243
|
-
|
|
303
|
+
/** Hard cap: how long to wait for a fresh spawn to leave `pending` (reach a
|
|
304
|
+
* live SessionStart state) while auto-dismissing pre-session dialogs. */
|
|
305
|
+
export const DIALOG_READY_TIMEOUT_MS = 5 * 60_000
|
|
306
|
+
|
|
307
|
+
let _dialogReadyTimeoutMs = DIALOG_READY_TIMEOUT_MS
|
|
308
|
+
|
|
309
|
+
/** Test-only seam: override the ready cap. */
|
|
310
|
+
export function _setDialogReadyTimeoutMs(ms: number): void {
|
|
311
|
+
_dialogReadyTimeoutMs = ms
|
|
244
312
|
}
|
|
245
313
|
|
|
246
|
-
/** Test-only seam: restore the default
|
|
247
|
-
export function
|
|
248
|
-
|
|
314
|
+
/** Test-only seam: restore the default ready cap. */
|
|
315
|
+
export function _resetDialogReadyTimeoutMs(): void {
|
|
316
|
+
_dialogReadyTimeoutMs = DIALOG_READY_TIMEOUT_MS
|
|
249
317
|
}
|
|
250
318
|
|
|
319
|
+
// ---------------------------------------------------------------------------
|
|
320
|
+
// Raw-tmux dialog fallback (b.vub) — bypass agent-director when the row is in
|
|
321
|
+
// a non-interactive state (missing/ended) but the pane still shows the dialog.
|
|
322
|
+
// ---------------------------------------------------------------------------
|
|
323
|
+
|
|
251
324
|
/**
|
|
252
|
-
*
|
|
253
|
-
* dialog appears, send Enter to accept the pre-selected
|
|
254
|
-
* "I am using this for local development" option, then confirm the dialog
|
|
255
|
-
* has cleared. All errors caught locally — never throws to the caller.
|
|
325
|
+
* b.vub — the critical mechanism the ticket's manual mitigation used.
|
|
256
326
|
*
|
|
257
|
-
*
|
|
258
|
-
* `
|
|
327
|
+
* On the RESUME lap, `client.resume` re-creates the tmux session but the AD row
|
|
328
|
+
* stays `missing`/`ended` until SessionStart re-fires — and SessionStart can't
|
|
329
|
+
* fire because the dev-channels dialog blocks it. Crucially, agent-director's
|
|
330
|
+
* `read-pane` / `send-keys` REFUSE to touch a `missing`/`ended` row
|
|
331
|
+
* (`ErrSpawnNotInteractive`, even with allow_pending), so the approver cannot
|
|
332
|
+
* clear the dialog through AD. A raw `tmux capture-pane` + `tmux send-keys …
|
|
333
|
+
* Enter` DOES clear it (verified against real AD/tmux; this is exactly the
|
|
334
|
+
* `tmux send-keys -t <session> Enter` the operator ran by hand in the ticket),
|
|
335
|
+
* after which AD flips to `waiting`.
|
|
259
336
|
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
* fresh-spawn approval:
|
|
263
|
-
* - `dev-channels-approve-no-dialog` — needle never observed
|
|
264
|
-
* - `dev-channels-approve-still-visible` — needle persists after Enter
|
|
337
|
+
* These two seams shell out to tmux directly, keyed on the deterministic
|
|
338
|
+
* per-channel session name. Injectable so unit tests stay hermetic.
|
|
265
339
|
*/
|
|
266
|
-
export
|
|
267
|
-
|
|
268
|
-
web: WebClient | undefined,
|
|
269
|
-
isStartup: boolean,
|
|
270
|
-
normalizedName?: string,
|
|
271
|
-
): Promise<void> {
|
|
272
|
-
void web
|
|
273
|
-
const claude_instance_id = instanceIdFor(channelId, normalizedName)
|
|
274
|
-
const deadline = Date.now() + _dialogPollTimeoutMs
|
|
340
|
+
export type TmuxPaneReader = (sessionName: string) => Promise<string>
|
|
341
|
+
export type TmuxEnterSender = (sessionName: string) => Promise<void>
|
|
275
342
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
343
|
+
function defaultTmuxCapturePane(sessionName: string): Promise<string> {
|
|
344
|
+
return (async () => {
|
|
345
|
+
const { spawn } = await import('child_process')
|
|
346
|
+
return await new Promise<string>((resolve) => {
|
|
347
|
+
try {
|
|
348
|
+
const child = spawn('tmux', ['capture-pane', '-p', '-t', sessionName])
|
|
349
|
+
let out = ''
|
|
350
|
+
child.stdout?.on('data', (d: Buffer) => { out += d.toString('utf8') })
|
|
351
|
+
child.on('error', () => resolve(''))
|
|
352
|
+
child.on('close', () => resolve(out))
|
|
353
|
+
} catch {
|
|
354
|
+
resolve('')
|
|
284
355
|
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
356
|
+
})
|
|
357
|
+
})()
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function defaultTmuxSendEnter(sessionName: string): Promise<void> {
|
|
361
|
+
return (async () => {
|
|
362
|
+
const { spawn } = await import('child_process')
|
|
363
|
+
await new Promise<void>((resolve) => {
|
|
364
|
+
try {
|
|
365
|
+
const child = spawn('tmux', ['send-keys', '-t', sessionName, 'Enter'], { stdio: 'ignore' })
|
|
366
|
+
child.on('error', () => resolve())
|
|
367
|
+
child.on('close', () => resolve())
|
|
368
|
+
} catch {
|
|
369
|
+
resolve()
|
|
290
370
|
}
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
|
|
371
|
+
})
|
|
372
|
+
})()
|
|
373
|
+
}
|
|
294
374
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
console.error(`[slack] approveDevChannelsDialog: ${msg}`)
|
|
298
|
-
if (isStartup) recordStartupError('dev-channels-approve-no-dialog', msg)
|
|
299
|
-
return
|
|
300
|
-
}
|
|
375
|
+
let _tmuxCapturePane: TmuxPaneReader = defaultTmuxCapturePane
|
|
376
|
+
let _tmuxSendEnter: TmuxEnterSender = defaultTmuxSendEnter
|
|
301
377
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
console.error(`[slack] approveDevChannelsDialog: ${msg}`)
|
|
315
|
-
if (isStartup) recordStartupError('dev-channels-approve-still-visible', msg)
|
|
316
|
-
}
|
|
378
|
+
/** Test-only seam: override the raw tmux pane reader. */
|
|
379
|
+
export function _setTmuxCapturePane(fn: TmuxPaneReader): void {
|
|
380
|
+
_tmuxCapturePane = fn
|
|
381
|
+
}
|
|
382
|
+
/** Test-only seam: override the raw tmux Enter sender. */
|
|
383
|
+
export function _setTmuxSendEnter(fn: TmuxEnterSender): void {
|
|
384
|
+
_tmuxSendEnter = fn
|
|
385
|
+
}
|
|
386
|
+
/** Test-only seam: restore the default raw tmux helpers. */
|
|
387
|
+
export function _resetTmuxDialogHelpers(): void {
|
|
388
|
+
_tmuxCapturePane = defaultTmuxCapturePane
|
|
389
|
+
_tmuxSendEnter = defaultTmuxSendEnter
|
|
317
390
|
}
|
|
318
391
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
392
|
+
/** Live (post-SessionStart) states: the dialog is gone, the session is ready. */
|
|
393
|
+
const DIALOG_READY_STATES = new Set(['waiting', 'working', 'ask_user', 'check_permission'])
|
|
394
|
+
/** Terminal states: the spawn died before becoming ready. */
|
|
395
|
+
const DIALOG_DEAD_STATES = new Set(['ended', 'missing'])
|
|
396
|
+
/** Every pre-SessionStart dialog we can auto-approve (option 1 pre-selected; Enter accepts). */
|
|
397
|
+
const PRE_SESSION_DIALOG_NEEDLES = [TRUST_DIALOG_NEEDLE, DEV_CHANNELS_DIALOG_NEEDLE]
|
|
322
398
|
|
|
323
399
|
/**
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
400
|
+
* b.vub: consecutive dead-state-with-no-needle polls required before treating a
|
|
401
|
+
* spawn as genuinely dead. A freshly resumed (or freshly spawned) row is
|
|
402
|
+
* legitimately `missing`/`ended` for a brief window: after `client.resume`, the
|
|
403
|
+
* tmux pane needs a moment to render the dev-channels dialog, and AD keeps
|
|
404
|
+
* reporting the prior terminal state until SessionStart re-fires. Bailing on the
|
|
405
|
+
* FIRST such poll (as the pre-b.vub code did) races the dialog and re-hangs the
|
|
406
|
+
* resume. Requiring a short grace streak lets the dialog appear (needle → Enter,
|
|
407
|
+
* which resets the streak) while still fast-failing a truly dead spawn without
|
|
408
|
+
* waiting the full 5-minute cap. Reset to 0 on any live state or needle sighting.
|
|
327
409
|
*/
|
|
328
|
-
export const
|
|
410
|
+
export const DIALOG_DEAD_GRACE_POLLS = 20
|
|
329
411
|
|
|
330
|
-
let
|
|
331
|
-
let _trustDialogPollTimeoutMs = DIALOG_POLL_TIMEOUT_MS
|
|
332
|
-
|
|
333
|
-
/** Test-only seam: override the trust-folder poll interval. */
|
|
334
|
-
export function _setTrustDialogPollIntervalMs(ms: number): void {
|
|
335
|
-
_trustDialogPollIntervalMs = ms
|
|
336
|
-
}
|
|
412
|
+
let _dialogDeadGracePolls = DIALOG_DEAD_GRACE_POLLS
|
|
337
413
|
|
|
338
|
-
/** Test-only seam:
|
|
339
|
-
export function
|
|
340
|
-
|
|
414
|
+
/** Test-only seam: override the dead-state grace streak. */
|
|
415
|
+
export function _setDialogDeadGracePolls(n: number): void {
|
|
416
|
+
_dialogDeadGracePolls = n
|
|
341
417
|
}
|
|
342
418
|
|
|
343
|
-
/** Test-only seam:
|
|
344
|
-
export function
|
|
345
|
-
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
/** Test-only seam: restore the default trust-folder poll timeout. */
|
|
349
|
-
export function _resetTrustDialogPollTimeoutMs(): void {
|
|
350
|
-
_trustDialogPollTimeoutMs = DIALOG_POLL_TIMEOUT_MS
|
|
419
|
+
/** Test-only seam: restore the default dead-state grace streak. */
|
|
420
|
+
export function _resetDialogDeadGracePolls(): void {
|
|
421
|
+
_dialogDeadGracePolls = DIALOG_DEAD_GRACE_POLLS
|
|
351
422
|
}
|
|
352
423
|
|
|
353
424
|
/**
|
|
354
|
-
*
|
|
355
|
-
*
|
|
356
|
-
*
|
|
357
|
-
*
|
|
425
|
+
* Drive a freshly-spawned bot past its pre-SessionStart dialogs (folder-trust
|
|
426
|
+
* and/or dev-channels) by watching agent-director's state machine: while the
|
|
427
|
+
* spawn is `pending` (SessionStart not yet fired — a dialog may be blocking),
|
|
428
|
+
* poll the pane and press Enter whenever a known dialog needle is on screen.
|
|
429
|
+
* Exit the instant AD reports a live state (the dialog is gone). 5-minute hard
|
|
430
|
+
* cap; failure to reach ready is surfaced loudly — never a silent early quit.
|
|
431
|
+
*
|
|
432
|
+
* status/readPane/sendKeys are wrapped in withOutageDetection so AD-outage
|
|
433
|
+
* flags keep working (b.en2). readPane/sendKeys use allow_pending:true because
|
|
434
|
+
* the bot is `pending` here (b.98w). The needle-gate guarantees Enter is sent
|
|
435
|
+
* ONLY when a dialog is actually displayed, so we never inject a stray Enter
|
|
436
|
+
* into a live prompt. Uses the composed instance id (b.ben).
|
|
358
437
|
*
|
|
359
|
-
*
|
|
360
|
-
*
|
|
438
|
+
* Replaces the former two-function pair (trust-folder + dev-channels approvers):
|
|
439
|
+
* one loop from spawn+0 (no wasted 30s trust window) that self-heals a missed
|
|
440
|
+
* Enter (state stays pending, needle reappears, next iteration presses again)
|
|
441
|
+
* and never gives up silently at 30s.
|
|
361
442
|
*
|
|
362
|
-
*
|
|
363
|
-
*
|
|
364
|
-
*
|
|
365
|
-
*
|
|
366
|
-
*
|
|
443
|
+
* b.vub: pane-first / state-tolerant, with a RAW-TMUX fallback for dead rows.
|
|
444
|
+
* A *resumed* bot faces the same `--dangerously-load-development-channels`
|
|
445
|
+
* dialog, but its AD row is `missing`/`ended` from the prior life while it sits
|
|
446
|
+
* blocked at the dialog (SessionStart never re-fires). Two compounding facts the
|
|
447
|
+
* pre-b.vub code missed: (1) it early-returned on DIALOG_DEAD_STATES before ever
|
|
448
|
+
* reading the pane; (2) even if it had tried, agent-director's read-pane/
|
|
449
|
+
* send-keys REFUSE a `missing`/`ended` row (ErrSpawnNotInteractive), so the
|
|
450
|
+
* dialog can only be cleared by talking to tmux directly. Fix: within the
|
|
451
|
+
* pre-SessionStart window, for a dead row read+Enter via raw tmux (keyed on the
|
|
452
|
+
* deterministic session name), for a pending row via AD; press Enter whenever a
|
|
453
|
+
* needle is visible regardless of AD state; treat missing/ended as terminal only
|
|
454
|
+
* after a short no-needle grace streak (DIALOG_DEAD_GRACE_POLLS), tolerating the
|
|
455
|
+
* brief post-resume window before the dialog renders. Still needle-gated (no
|
|
456
|
+
* stray Enter into a live session) and bounded by the hard cap.
|
|
367
457
|
*/
|
|
368
|
-
export async function
|
|
458
|
+
export async function approvePreSessionDialogs(
|
|
369
459
|
channelId: string,
|
|
370
460
|
web: WebClient | undefined,
|
|
371
461
|
isStartup: boolean,
|
|
372
462
|
normalizedName?: string,
|
|
373
463
|
): Promise<void> {
|
|
374
|
-
void web
|
|
375
464
|
const claude_instance_id = instanceIdFor(channelId, normalizedName)
|
|
376
|
-
const deadline = Date.now() +
|
|
465
|
+
const deadline = Date.now() + _dialogReadyTimeoutMs
|
|
466
|
+
let deadStreak = 0
|
|
377
467
|
|
|
378
|
-
let approved = false
|
|
379
468
|
while (Date.now() < deadline) {
|
|
469
|
+
// 1) Readiness oracle.
|
|
470
|
+
let state: string
|
|
380
471
|
try {
|
|
381
|
-
const
|
|
382
|
-
|
|
383
|
-
await withOutageDetection(channelId, undefined, (client) => client.sendKeys({ claude_instance_id, text: '', allow_pending: true }))
|
|
384
|
-
approved = true
|
|
385
|
-
break
|
|
386
|
-
}
|
|
472
|
+
const r = await withOutageDetection(channelId, undefined, (client) => client.status({ claude_instance_id }))
|
|
473
|
+
state = r.state
|
|
387
474
|
} catch (err) {
|
|
388
|
-
if (err instanceof
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
475
|
+
if (err instanceof ErrSpawnNotFound) {
|
|
476
|
+
console.error(`[slack] approvePreSessionDialogs: spawn not found for channel=${channelId} — aborting`)
|
|
477
|
+
return
|
|
478
|
+
}
|
|
479
|
+
// Transient (incl. AD-outage errors already flagged by withOutageDetection) — keep polling.
|
|
480
|
+
console.error(`[slack] approvePreSessionDialogs: status error channel=${channelId}: ${String(err)}`)
|
|
481
|
+
await new Promise((r) => setTimeout(r, _dialogPollIntervalMs))
|
|
482
|
+
continue
|
|
483
|
+
}
|
|
484
|
+
if (DIALOG_READY_STATES.has(state)) return // dialog cleared, session live
|
|
485
|
+
|
|
486
|
+
// 2) Pane-first (b.vub): read the pane and dismiss any visible pre-session
|
|
487
|
+
// dialog BEFORE deciding whether a dead state is terminal.
|
|
488
|
+
//
|
|
489
|
+
// Two paths, because agent-director's read-pane/send-keys REFUSE a
|
|
490
|
+
// `missing`/`ended` row (ErrSpawnNotInteractive) even with allow_pending:
|
|
491
|
+
// - Fresh spawn (state=pending): drive via AD read-pane/send-keys.
|
|
492
|
+
// - Resumed row (state=missing/ended): AD won't touch the pane, but the
|
|
493
|
+
// dialog IS on screen and blocking SessionStart. Fall back to RAW tmux
|
|
494
|
+
// (capture-pane + send-keys Enter) keyed on the deterministic session
|
|
495
|
+
// name — exactly the `tmux send-keys -t <session> Enter` the operator
|
|
496
|
+
// ran by hand in the ticket. Once Enter lands, SessionStart fires and
|
|
497
|
+
// AD flips to a live state on the next poll.
|
|
498
|
+
let needleVisible = false
|
|
499
|
+
if (DIALOG_DEAD_STATES.has(state)) {
|
|
500
|
+
// Raw-tmux fallback (agent-director cannot interact with a dead row).
|
|
501
|
+
const sessionName = tmuxSessionNameFor(channelId, normalizedName)
|
|
502
|
+
try {
|
|
503
|
+
const pane = await _tmuxCapturePane(sessionName)
|
|
504
|
+
needleVisible = PRE_SESSION_DIALOG_NEEDLES.some((n) => pane.includes(n))
|
|
505
|
+
if (needleVisible) {
|
|
506
|
+
await _tmuxSendEnter(sessionName)
|
|
507
|
+
}
|
|
508
|
+
} catch (err) {
|
|
509
|
+
console.error(`[slack] approvePreSessionDialogs: raw-tmux fallback error channel=${channelId}: ${String(err)}`)
|
|
510
|
+
}
|
|
511
|
+
} else {
|
|
512
|
+
// Interactive (pending) — drive via agent-director.
|
|
513
|
+
try {
|
|
514
|
+
const { pane } = await withOutageDetection(channelId, undefined, (client) => client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true }))
|
|
515
|
+
needleVisible = PRE_SESSION_DIALOG_NEEDLES.some((n) => pane.includes(n))
|
|
516
|
+
if (needleVisible) {
|
|
517
|
+
await withOutageDetection(channelId, undefined, (client) => client.sendKeys({ claude_instance_id, text: '', allow_pending: true })) // Enter
|
|
518
|
+
}
|
|
519
|
+
} catch (err) {
|
|
520
|
+
console.error(`[slack] approvePreSessionDialogs: readPane/sendKeys error channel=${channelId}: ${String(err)}`)
|
|
392
521
|
}
|
|
393
522
|
}
|
|
394
|
-
await new Promise((r) => setTimeout(r, _trustDialogPollIntervalMs))
|
|
395
|
-
}
|
|
396
523
|
|
|
397
|
-
|
|
398
|
-
//
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
524
|
+
// 3) Dead-state handling (b.vub). A needle means "dialog-blocked, not dead":
|
|
525
|
+
// reset the streak and keep pressing Enter. Only treat missing/ended as
|
|
526
|
+
// terminal after it persists with NO needle for DIALOG_DEAD_GRACE_POLLS
|
|
527
|
+
// consecutive polls — this tolerates the brief post-spawn/post-resume window
|
|
528
|
+
// where the row is still terminal and the pane hasn't rendered the dialog
|
|
529
|
+
// yet, without racing the dialog and re-hanging the resume.
|
|
530
|
+
if (needleVisible) {
|
|
531
|
+
deadStreak = 0
|
|
532
|
+
} else if (DIALOG_DEAD_STATES.has(state)) {
|
|
533
|
+
deadStreak += 1
|
|
534
|
+
if (deadStreak >= _dialogDeadGracePolls) {
|
|
535
|
+
const msg = `spawn reached ${state} before clearing dev-channels dialog for channel=${channelId} (no needle for ${deadStreak} polls)`
|
|
536
|
+
console.error(`[slack] approvePreSessionDialogs: ${msg}`)
|
|
537
|
+
if (isStartup) recordStartupError('dev-channels-approve-spawn-died', msg)
|
|
538
|
+
return
|
|
539
|
+
}
|
|
540
|
+
} else {
|
|
541
|
+
// pending (or any other non-dead, non-ready state) — reset the streak.
|
|
542
|
+
deadStreak = 0
|
|
410
543
|
}
|
|
544
|
+
|
|
545
|
+
await new Promise((r) => setTimeout(r, _dialogPollIntervalMs))
|
|
411
546
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
547
|
+
|
|
548
|
+
// 3) Hard cap hit — genuine failure, surfaced loudly (no silent give-up).
|
|
549
|
+
const msg = `spawn never reached a live state within ${_dialogReadyTimeoutMs}ms for channel=${channelId} — dialog unrecognized or session hung (dev-needle='${DEV_CHANNELS_DIALOG_NEEDLE}')`
|
|
550
|
+
console.error(`[slack] approvePreSessionDialogs: ${msg}`)
|
|
551
|
+
if (isStartup) recordStartupError('dev-channels-approve-not-ready', msg)
|
|
552
|
+
postSpawnFailureToChannel(channelId, new AgentDirectorError('status', 'DialogApprovalTimeout', msg), web, isStartup)
|
|
417
553
|
}
|
|
418
554
|
|
|
419
555
|
// ---------------------------------------------------------------------------
|
|
@@ -530,6 +666,78 @@ async function tryKill(channelId: string, normalizedName: string | undefined): P
|
|
|
530
666
|
}
|
|
531
667
|
}
|
|
532
668
|
|
|
669
|
+
// ---------------------------------------------------------------------------
|
|
670
|
+
// Orphan-tmux self-heal (b.vub)
|
|
671
|
+
// ---------------------------------------------------------------------------
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Kill a tmux session by its exact name. Injectable seam so unit tests can
|
|
675
|
+
* assert the self-heal path without spawning real processes. Default impl
|
|
676
|
+
* runs `tmux kill-session -t <name>` best-effort.
|
|
677
|
+
*
|
|
678
|
+
* b.vub: the field failure is an orphan tmux session that survives the AD row
|
|
679
|
+
* going `missing` — the AD `client.kill` verb does NOT reap it (observed across
|
|
680
|
+
* dozens of restart cycles). Killing the session directly by its deterministic,
|
|
681
|
+
* per-channel name (`tmuxSessionNameFor`) is the only reliable reap, and the
|
|
682
|
+
* name can only ever belong to this channel's spawn, so it is safe.
|
|
683
|
+
*/
|
|
684
|
+
export type TmuxSessionKiller = (sessionName: string) => Promise<void>
|
|
685
|
+
|
|
686
|
+
let _killTmuxSession: TmuxSessionKiller = async (sessionName: string): Promise<void> => {
|
|
687
|
+
const { spawn } = await import('child_process')
|
|
688
|
+
await new Promise<void>((resolve) => {
|
|
689
|
+
try {
|
|
690
|
+
const child = spawn('tmux', ['kill-session', '-t', sessionName], { stdio: 'ignore' })
|
|
691
|
+
child.on('error', () => resolve()) // tmux missing / session absent — best-effort
|
|
692
|
+
child.on('close', () => resolve())
|
|
693
|
+
} catch {
|
|
694
|
+
resolve()
|
|
695
|
+
}
|
|
696
|
+
})
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/** Test-only seam: override the tmux-session killer. */
|
|
700
|
+
export function _setTmuxSessionKiller(fn: TmuxSessionKiller): void {
|
|
701
|
+
_killTmuxSession = fn
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/** Test-only seam: restore the default tmux-session killer. */
|
|
705
|
+
export function _resetTmuxSessionKiller(): void {
|
|
706
|
+
_killTmuxSession = async (sessionName: string): Promise<void> => {
|
|
707
|
+
const { spawn } = await import('child_process')
|
|
708
|
+
await new Promise<void>((resolve) => {
|
|
709
|
+
try {
|
|
710
|
+
const child = spawn('tmux', ['kill-session', '-t', sessionName], { stdio: 'ignore' })
|
|
711
|
+
child.on('error', () => resolve())
|
|
712
|
+
child.on('close', () => resolve())
|
|
713
|
+
} catch {
|
|
714
|
+
resolve()
|
|
715
|
+
}
|
|
716
|
+
})
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/**
|
|
721
|
+
* Self-heal an `ErrTmuxSessionCreate` collision (b.vub): the deterministic tmux
|
|
722
|
+
* session name is still held by an orphaned session while the AD row is
|
|
723
|
+
* terminal/gone, so a fresh spawn/resume cannot create the session. Kill the
|
|
724
|
+
* orphan by name, then retry `client.spawn` ONCE. Returns the spawn result on
|
|
725
|
+
* success, or rethrows the retry's error (caller surfaces it).
|
|
726
|
+
*/
|
|
727
|
+
async function selfHealTmuxCollisionAndRespawn(
|
|
728
|
+
channelId: string,
|
|
729
|
+
route: { cwd: string },
|
|
730
|
+
params: SpawnParams,
|
|
731
|
+
normalizedName: string | undefined,
|
|
732
|
+
): Promise<{ claude_instance_id: string }> {
|
|
733
|
+
const sessionName = tmuxSessionNameFor(channelId, normalizedName)
|
|
734
|
+
console.error(
|
|
735
|
+
`[slack] spawnForRoute: ErrTmuxSessionCreate for channel=${channelId} — killing orphan tmux session "${sessionName}" and retrying spawn once`,
|
|
736
|
+
)
|
|
737
|
+
await _killTmuxSession(sessionName)
|
|
738
|
+
return withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
739
|
+
}
|
|
740
|
+
|
|
533
741
|
/** Delete the spawn row; surface failures. Returns whether the delete succeeded. */
|
|
534
742
|
async function tryDelete(
|
|
535
743
|
channelId: string,
|
|
@@ -585,13 +793,36 @@ export async function spawnForRoute(
|
|
|
585
793
|
try {
|
|
586
794
|
const r = await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
587
795
|
console.error(`[slack] spawnForRoute: spawned channel=${channelId} instanceId=${r.claude_instance_id}`)
|
|
588
|
-
await
|
|
589
|
-
await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
|
|
796
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
590
797
|
return { channelId, action: 'spawned' }
|
|
591
798
|
} catch (err) {
|
|
592
799
|
if (err instanceof ErrInstanceIdCollision) {
|
|
593
800
|
// Collision → fall through to get-then-act
|
|
594
801
|
console.error(`[slack] spawnForRoute: ErrInstanceIdCollision for channel=${channelId} — fetching current state`)
|
|
802
|
+
} else if (err instanceof ErrTmuxSessionCreate) {
|
|
803
|
+
// b.vub: fresh spawn collided on the deterministic tmux session name held
|
|
804
|
+
// by an orphan session (no instance-id collision → no AD row to resolve).
|
|
805
|
+
// Self-heal: kill the orphan by name, retry spawn once.
|
|
806
|
+
try {
|
|
807
|
+
const r = await selfHealTmuxCollisionAndRespawn(channelId, route, params, normalizedName)
|
|
808
|
+
console.error(`[slack] spawnForRoute: self-heal spawn succeeded after ErrTmuxSessionCreate for channel=${channelId} instanceId=${r.claude_instance_id}`)
|
|
809
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
810
|
+
return { channelId, action: 'spawned' }
|
|
811
|
+
} catch (err2) {
|
|
812
|
+
if (
|
|
813
|
+
err2 instanceof ErrSystemInstallDisappeared ||
|
|
814
|
+
err2 instanceof ErrTmuxNotAvailable ||
|
|
815
|
+
err2 instanceof ErrCwdNotFound ||
|
|
816
|
+
err2 instanceof ErrCwdNotADirectory
|
|
817
|
+
) {
|
|
818
|
+
return { channelId, action: 'failed' }
|
|
819
|
+
}
|
|
820
|
+
const e = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('spawn', 'UnknownError', String(err2))
|
|
821
|
+
console.error(`[slack] spawnForRoute: self-heal spawn after ErrTmuxSessionCreate failed for channel=${channelId}: ${e.errName}`)
|
|
822
|
+
if (isStartup) recordStartupError('spawn-failed', `self-heal spawn after ErrTmuxSessionCreate failed for channel=${channelId}: ${e.errName}`, e)
|
|
823
|
+
postSpawnFailureToChannel(channelId, e, web, isStartup)
|
|
824
|
+
return { channelId, action: 'failed' }
|
|
825
|
+
}
|
|
595
826
|
} else if (
|
|
596
827
|
err instanceof ErrSystemInstallDisappeared ||
|
|
597
828
|
err instanceof ErrTmuxNotAvailable ||
|
|
@@ -620,8 +851,7 @@ export async function spawnForRoute(
|
|
|
620
851
|
try {
|
|
621
852
|
const r = await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
622
853
|
console.error(`[slack] spawnForRoute: retry-spawn succeeded for channel=${channelId} instanceId=${r.claude_instance_id}`)
|
|
623
|
-
await
|
|
624
|
-
await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
|
|
854
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
625
855
|
return { channelId, action: 'spawned' }
|
|
626
856
|
} catch (err2) {
|
|
627
857
|
if (
|
|
@@ -658,8 +888,7 @@ export async function spawnForRoute(
|
|
|
658
888
|
try {
|
|
659
889
|
await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
660
890
|
console.error(`[slack] spawnForRoute: fresh-spawned (after kill+delete) for channel=${channelId}`)
|
|
661
|
-
await
|
|
662
|
-
await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
|
|
891
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
663
892
|
return { channelId, action: 'spawned' }
|
|
664
893
|
} catch (err) {
|
|
665
894
|
if (
|
|
@@ -683,16 +912,45 @@ export async function spawnForRoute(
|
|
|
683
912
|
try {
|
|
684
913
|
await withSpawnDetection(channelId, route.cwd, (client) => client.resume({ claude_instance_id: instanceIdFor(channelId, normalizedName) }))
|
|
685
914
|
console.error(`[slack] spawnForRoute: resumed channel=${channelId}`)
|
|
915
|
+
// b.vub: a resumed bot faces the same --dangerously-load-development-channels
|
|
916
|
+
// dialog. Its AD row is still `missing`/`ended` while blocked at the dialog
|
|
917
|
+
// (SessionStart hasn't re-fired), so the pane-first approver drives it past.
|
|
918
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
686
919
|
return { channelId, action: 'resumed' }
|
|
687
920
|
} catch (err) {
|
|
921
|
+
if (err instanceof ErrTmuxSessionCreate) {
|
|
922
|
+
// b.vub: the deterministic tmux session name is still held by an orphan
|
|
923
|
+
// session while the AD row is terminal — resume cannot re-create it.
|
|
924
|
+
// This is the observed field failure (resume throws ErrTmuxSessionCreate
|
|
925
|
+
// every ~2 min). Self-heal: kill the orphan by name, retry spawn once.
|
|
926
|
+
try {
|
|
927
|
+
const r = await selfHealTmuxCollisionAndRespawn(channelId, route, params, normalizedName)
|
|
928
|
+
console.error(`[slack] spawnForRoute: self-heal spawn succeeded after ErrTmuxSessionCreate for channel=${channelId} instanceId=${r.claude_instance_id}`)
|
|
929
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
930
|
+
return { channelId, action: 'spawned' }
|
|
931
|
+
} catch (err2) {
|
|
932
|
+
if (
|
|
933
|
+
err2 instanceof ErrSystemInstallDisappeared ||
|
|
934
|
+
err2 instanceof ErrTmuxNotAvailable ||
|
|
935
|
+
err2 instanceof ErrCwdNotFound ||
|
|
936
|
+
err2 instanceof ErrCwdNotADirectory
|
|
937
|
+
) {
|
|
938
|
+
return { channelId, action: 'failed' }
|
|
939
|
+
}
|
|
940
|
+
const e = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('spawn', 'UnknownError', String(err2))
|
|
941
|
+
console.error(`[slack] spawnForRoute: self-heal spawn after ErrTmuxSessionCreate failed for channel=${channelId}: ${e.errName}`)
|
|
942
|
+
if (isStartup) recordStartupError('spawn-failed', `self-heal spawn after ErrTmuxSessionCreate failed for channel=${channelId}: ${e.errName}`, e)
|
|
943
|
+
postSpawnFailureToChannel(channelId, e, web, isStartup)
|
|
944
|
+
return { channelId, action: 'failed' }
|
|
945
|
+
}
|
|
946
|
+
}
|
|
688
947
|
if (err instanceof ErrNoSessionId || err instanceof ErrJsonlMissing) {
|
|
689
948
|
console.error(`[slack] spawnForRoute: ${err.errName} on resume for channel=${channelId} — delete+fresh`)
|
|
690
949
|
if (!(await tryDelete(channelId, normalizedName, web, isStartup))) return { channelId, action: 'failed' }
|
|
691
950
|
try {
|
|
692
951
|
await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
693
952
|
console.error(`[slack] spawnForRoute: fresh-spawned (after delete) for channel=${channelId}`)
|
|
694
|
-
await
|
|
695
|
-
await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
|
|
953
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
696
954
|
return { channelId, action: 'spawned' }
|
|
697
955
|
} catch (err2) {
|
|
698
956
|
if (
|
|
@@ -717,8 +975,7 @@ export async function spawnForRoute(
|
|
|
717
975
|
if (!(await tryDelete(channelId, normalizedName, web, isStartup))) return { channelId, action: 'failed' }
|
|
718
976
|
try {
|
|
719
977
|
await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
|
|
720
|
-
await
|
|
721
|
-
await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
|
|
978
|
+
await approvePreSessionDialogs(channelId, web, isStartup, normalizedName)
|
|
722
979
|
return { channelId, action: 'spawned' }
|
|
723
980
|
} catch (err2) {
|
|
724
981
|
if (
|
|
@@ -751,12 +1008,27 @@ export async function spawnForRoute(
|
|
|
751
1008
|
}
|
|
752
1009
|
|
|
753
1010
|
if (state === 'waiting') {
|
|
754
|
-
|
|
1011
|
+
// b.rmy: propagate the reconnect outcome — a failed reconnect must count
|
|
1012
|
+
// as `failed` so startupSessionManager's ok/failed totals reflect reality
|
|
1013
|
+
// (previously this reported 'reconnected' unconditionally, masking a
|
|
1014
|
+
// total post-reboot outage as "0 failed").
|
|
1015
|
+
if (!(await reconnectMcp(channelId, web, routingConfig))) {
|
|
1016
|
+
console.error(`[slack] spawnForRoute: reconnect failed for channel=${channelId}`)
|
|
1017
|
+
if (isStartup) recordStartupError('spawn-failed', `reconnect failed for channel=${channelId} (state=waiting)`)
|
|
1018
|
+
return { channelId, action: 'failed' }
|
|
1019
|
+
}
|
|
755
1020
|
return { channelId, action: 'reconnected' }
|
|
756
1021
|
}
|
|
757
1022
|
|
|
758
1023
|
if (state === 'working') {
|
|
759
|
-
|
|
1024
|
+
// b.rmy: same outcome propagation as the `waiting` branch. Note
|
|
1025
|
+
// waitForWaitingAndReconnect returns true on timeout/terminal transitions
|
|
1026
|
+
// by design (long turns aren't errors) — only real failures reach here.
|
|
1027
|
+
if (!(await waitForWaitingAndReconnect(channelId, routingConfig, web))) {
|
|
1028
|
+
console.error(`[slack] spawnForRoute: reconnect failed for channel=${channelId}`)
|
|
1029
|
+
if (isStartup) recordStartupError('spawn-failed', `reconnect failed for channel=${channelId} (state=working)`)
|
|
1030
|
+
return { channelId, action: 'failed' }
|
|
1031
|
+
}
|
|
760
1032
|
return { channelId, action: 'reconnected' }
|
|
761
1033
|
}
|
|
762
1034
|
|