@agfpd/iapeer 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/index.ts +13 -80
- package/src/cli/listTui.ts +4 -14
- package/src/launch/cmdlog.ts +7 -110
- package/src/launch/codexHooksTrust.ts +0 -3
- package/src/launch/index.ts +14 -314
- package/src/launch/launchdRun.ts +9 -52
- package/src/launch/ptyHost.ts +0 -12
- package/src/launch/readyGateModel.ts +9 -80
- package/src/lifecycle/index.ts +25 -101
- package/src/registry/index.ts +9 -7
- package/src/supervisor/daemon.ts +1 -1
- package/src/supervisor/paths.ts +1 -10
- package/src/{shadow → supervisor}/render.ts +10 -10
- package/src/transport/index.ts +41 -400
- package/src/update/index.ts +0 -9
- package/src/launch/canary.ts +0 -239
- package/src/shadow/gate.ts +0 -396
- package/src/shadow/index.ts +0 -174
- package/src/shadow/install.ts +0 -256
package/src/launch/index.ts
CHANGED
|
@@ -15,13 +15,9 @@
|
|
|
15
15
|
|
|
16
16
|
import { existsSync, mkdirSync, statSync } from 'fs'
|
|
17
17
|
import { dirname } from 'path'
|
|
18
|
-
import { spawnSync } from 'child_process'
|
|
19
18
|
import { type Runtime } from '../core/constants.ts'
|
|
20
19
|
import { buildLaunchInvocation } from './invocation.ts'
|
|
21
|
-
import {
|
|
22
|
-
import { hardenCmdLogDir, prepareCmdLogDir } from './cmdlog.ts'
|
|
23
|
-
import { logReadyGateCrossCheck, readyGatePtyFlipEnabled, readyGateViewport } from './readyGateModel.ts'
|
|
24
|
-
import { deliverHosted, hostSessionAlive, ptyHostEnabled, startPtyHost, waitHostReady } from './ptyHost.ts'
|
|
20
|
+
import { deliverHosted, hostSessionAlive, startPtyHost, waitHostReady } from './ptyHost.ts'
|
|
25
21
|
import { appendLifecycleEvent } from '../lifecycle/eventlog.ts'
|
|
26
22
|
import { claudeAdapter } from './adapters/claude.ts'
|
|
27
23
|
import { codexAdapter } from './adapters/codex.ts'
|
|
@@ -95,23 +91,6 @@ export function getAdapter(runtime: Runtime): RuntimeAdapter {
|
|
|
95
91
|
throw new Error(`no launch adapter for runtime "${runtime}"`)
|
|
96
92
|
}
|
|
97
93
|
|
|
98
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
99
|
-
// tmux helpers (direct spawnSync; no lifecycle dependency)
|
|
100
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
101
|
-
|
|
102
|
-
function tmux(sock: string, ...args: string[]): { ok: boolean; out: string; err: string } {
|
|
103
|
-
const r = spawnSync('tmux', ['-S', sock, ...args], { encoding: 'utf8' })
|
|
104
|
-
return { ok: r.status === 0, out: r.stdout ?? '', err: r.stderr ?? '' }
|
|
105
|
-
}
|
|
106
|
-
function sessionAlive(sock: string, identity: string): boolean {
|
|
107
|
-
return tmux(sock, 'has-session', '-t', identity).ok
|
|
108
|
-
}
|
|
109
|
-
function capturePane(sock: string, identity: string): string {
|
|
110
|
-
return tmux(sock, 'capture-pane', '-t', identity, '-p').out
|
|
111
|
-
}
|
|
112
|
-
function shellQuote(value: string): string {
|
|
113
|
-
return `'${String(value).replace(/'/g, `'\\''`)}'`
|
|
114
|
-
}
|
|
115
94
|
function sleep(ms: number): Promise<void> {
|
|
116
95
|
return new Promise(r => setTimeout(r, ms))
|
|
117
96
|
}
|
|
@@ -120,101 +99,21 @@ function fail(identity: string, reason: string): LaunchResult {
|
|
|
120
99
|
return { status: 'FAILED', identity, process_address: identity, reason }
|
|
121
100
|
}
|
|
122
101
|
|
|
123
|
-
/**
|
|
124
|
-
* The last two non-empty lines of a pane capture, flattened for embedding in a
|
|
125
|
-
* fail reason: quotes → ', whitespace runs collapsed, joined with ' ⏎ ', clipped
|
|
126
|
-
* to 160 chars. '' for an empty/blank capture (caller omits the clause). Exported
|
|
127
|
-
* for unit tests — the boot loop embeds this in `never-became-ready` so the
|
|
128
|
-
* delivery.log err field shows what the TUI was actually displaying.
|
|
129
|
-
*/
|
|
130
|
-
export function paneTail(pane: string): string {
|
|
131
|
-
const lines = pane
|
|
132
|
-
.split(/\r?\n/)
|
|
133
|
-
.map(l => l.replace(/\s+/g, ' ').trim())
|
|
134
|
-
.filter(Boolean)
|
|
135
|
-
if (lines.length === 0) return ''
|
|
136
|
-
return lines.slice(-2).join(' ⏎ ').replace(/"/g, "'").slice(0, 160)
|
|
137
|
-
}
|
|
138
102
|
function ready(identity: string): LaunchResult {
|
|
139
103
|
return { status: 'READY', identity, process_address: identity }
|
|
140
104
|
}
|
|
141
105
|
|
|
142
106
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
143
|
-
// Exit-cause observability —
|
|
144
|
-
//
|
|
145
|
-
// (
|
|
146
|
-
//
|
|
147
|
-
//
|
|
148
|
-
// pane's leader process exits (with `remain-on-exit on` retaining the dead pane so
|
|
149
|
-
// `#{pane_dead_status}`/`#{pane_dead_signal}` are populated), logs one logfmt line,
|
|
150
|
-
// then kill-sessions the now-dead pane so the daemon's `has-session` death
|
|
151
|
-
// detection (and the always-on KeepAlive block-watch) stay intact.
|
|
152
|
-
//
|
|
153
|
-
// Scope — verified live on tmux 3.6a (3 death modes + the daemon-reap path):
|
|
154
|
-
// • graceful exit → `dead_status=<code> dead_signal=` (code, no signal)
|
|
155
|
-
// • SIGTERM/SIGKILL/crash to the PROCESS → `dead_status= dead_signal=<name>`
|
|
156
|
-
// • daemon-initiated `kill-session` (idle-reap / self-TTL / stop) does NOT fire
|
|
157
|
-
// pane-died → NO line here (those are already in lifecycle.log — no double-log).
|
|
158
|
-
// SERVER-LEVEL GAP: SIGKILL to the tmux SERVER process itself runs no hook (the
|
|
159
|
-
// event loop is gone). Closed one level up by the server-death canary
|
|
160
|
-
// (canary.ts — `ev=server-exit` + forensics snapshot); the daemon's post-factum
|
|
161
|
-
// reaped-gone death-class remains the detection backstop.
|
|
107
|
+
// Exit-cause observability — the supervisor records WHY a hosted session's child
|
|
108
|
+
// died (dead_status/dead_signal) into <exitLogDir>/exits.log at the moment of death
|
|
109
|
+
// (a death-EVENT, host=supervisor), passed in via startPtyHost's exitLogPath. The
|
|
110
|
+
// daemon's supervise tick learns of the death post-factum (reaped-gone) and reads
|
|
111
|
+
// the cause from that log.
|
|
162
112
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
163
113
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
/**
|
|
169
|
-
* Build the tmux `pane-died` hook command string (the value of `set-hook -t <id>
|
|
170
|
-
* pane-died <value>`). On the pane leader's death it appends ONE logfmt line —
|
|
171
|
-
* `ts=<ISO> ev=session-exit identity=<id> dead_status=#{…} dead_signal=#{…}`
|
|
172
|
-
* — to `exitLogFile`, then runs a tmux-NATIVE `kill-session` (no shell `tmux`, so
|
|
173
|
-
* it needs no PATH — launchd gives always-on servers a minimal one). Pure (no I/O)
|
|
174
|
-
* so the exact string is unit-testable. Quoting: the `run-shell` arg is wrapped in
|
|
175
|
-
* tmux SINGLE quotes (literal at the tmux layer, still `#{}`-format-expanded) with
|
|
176
|
-
* sh DOUBLE quotes inside — the two levels never collide; `\n`/`$(…)` pass through
|
|
177
|
-
* tmux untouched to sh. `identity`/`exitLogFile` are assumed free of single quotes
|
|
178
|
-
* (runtime-personality identities and the ~/.iapeer/logs path always are). */
|
|
179
|
-
export function exitCauseHook(identity: string, exitLogFile: string): string {
|
|
180
|
-
const line =
|
|
181
|
-
`ts=%s ev=session-exit identity=${identity} ` +
|
|
182
|
-
`dead_status=#{pane_dead_status} dead_signal=#{pane_dead_signal}\\n`
|
|
183
|
-
const log =
|
|
184
|
-
`printf "${line}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${exitLogFile}"`
|
|
185
|
-
// Silence the server-death canary BEFORE kill-session: with a single-session
|
|
186
|
-
// server the kill empties the server and exit-empty takes it down — without
|
|
187
|
-
// explicit silence the canary would append a second, muddier `ev=server-exit`
|
|
188
|
-
// record for a death this very hook just captured (the session-exit line is
|
|
189
|
-
// the richer, authoritative one: it has the exit code/signal). v2 contract:
|
|
190
|
-
// deliberate = channel signal (tmux-NATIVE wait-for -S) + DISMISS the sh
|
|
191
|
-
// recorder (abs-path /usr/bin/pkill — survives the minimal launchd PATH;
|
|
192
|
-
// `\$` keeps the regex anchor literal through the sh double-quote layer; the
|
|
193
|
-
// `[i]` class keeps the pattern from matching its OWN occurrence in this
|
|
194
|
-
// hook-sh's cmdline — the pgrep self-match classic).
|
|
195
|
-
const dismiss = `/usr/bin/pkill -f "[i]${canaryChannel(identity).slice(1)}([^a-z0-9-]|\\$)"`
|
|
196
|
-
return `run-shell '${log} ; ${dismiss}' ; wait-for -S "${canaryChannel(identity)}" ; kill-session -t "${identity}"`
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
/** Install the exit-cause observability on a freshly-created session: ensure the
|
|
200
|
-
* exit-log dir exists, turn `remain-on-exit` on (so pane-died can read the dead
|
|
201
|
-
* pane's status/signal) and register the hook. Best-effort — a tmux/FS hiccup
|
|
202
|
-
* here must never fail the launch (observability is never load-bearing). No-op
|
|
203
|
-
* when `exitLogDir` is falsy (a partial/test cfg): `remain-on-exit` stays OFF so
|
|
204
|
-
* behavior is byte-identical to before (and no dead pane can linger un-reaped). */
|
|
205
|
-
function installExitHook(sock: string, identity: string, exitLogDir: string | undefined): void {
|
|
206
|
-
if (!exitLogDir) return
|
|
207
|
-
try {
|
|
208
|
-
mkdirSync(exitLogDir, { recursive: true, mode: 0o700 })
|
|
209
|
-
// remain-on-exit must be ON before the process can die, else pane-died won't
|
|
210
|
-
// retain the dead pane and the status/signal are lost. Set it (and the hook)
|
|
211
|
-
// immediately after new-session — the only un-coverable window is the few ms
|
|
212
|
-
// before this runs, irrelevant for a runtime that takes seconds to initialize.
|
|
213
|
-
tmux(sock, 'set-option', '-t', identity, 'remain-on-exit', 'on')
|
|
214
|
-
tmux(sock, 'set-hook', '-t', identity, 'pane-died', exitCauseHook(identity, exitLogPath(exitLogDir)))
|
|
215
|
-
} catch {
|
|
216
|
-
/* observability is best-effort — never block a wake on a hook-install hiccup */
|
|
217
|
-
}
|
|
114
|
+
/** The exit-cause log file (sibling to lifecycle.log) the supervisor appends death-EVENTs to. */
|
|
115
|
+
export function exitLogPath(exitLogDir: string): string {
|
|
116
|
+
return `${exitLogDir}/exits.log`
|
|
218
117
|
}
|
|
219
118
|
|
|
220
119
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -264,7 +163,7 @@ export const launch: LaunchFn = async (
|
|
|
264
163
|
// skipped; no transcript ready-gate — newestActivityMtime=null). @xterm stays out of THIS process
|
|
265
164
|
// (ptyHost: @xterm-free start + deliver + sessionAlive; readiness's @xterm import is dynamic and only
|
|
266
165
|
// loads here, default-on — and the router branch never reaches it).
|
|
267
|
-
if (
|
|
166
|
+
if (adapter.kind === 'tui' || adapter.kind === 'router') {
|
|
268
167
|
const invocation = buildLaunchInvocation(spec, adapter, cfg)
|
|
269
168
|
mkdirSync(cfg.logDir, { recursive: true, mode: 0o700 }) // the supervisor writes the pane-log here
|
|
270
169
|
const paneLogPath = `${cfg.logDir}/${identity}.log`
|
|
@@ -329,208 +228,9 @@ export const launch: LaunchFn = async (
|
|
|
329
228
|
}
|
|
330
229
|
return fail(identity, 'model-did-not-process-task (hosted: no activity advance after delivery)')
|
|
331
230
|
}
|
|
332
|
-
//
|
|
333
|
-
appendLifecycleEvent(cfg.exitLogDir, { ev: 'pty-host-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
// (1) Pre-clean any stale tmux server on this socket, then launch detached.
|
|
337
|
-
// Silence the server-death canary EXPLICITLY first — this teardown is
|
|
338
|
-
// deliberate: signal the channel (client exits 0) AND dismiss the sh
|
|
339
|
-
// recorder (TERM → trap → silent; canary v2 no longer reads a signaled
|
|
340
|
-
// client alone as deliberate — an external killer sweeping server+client
|
|
341
|
-
// was exactly the death-#4 silence). Only then sweep the server processes
|
|
342
|
-
// (the pkill also matches a leftover canary client — harmless, both
|
|
343
|
-
// canary processes are already dismissed).
|
|
344
|
-
signalCanaryClean(sock, identity)
|
|
345
|
-
dismissCanary(identity)
|
|
346
|
-
spawnSync('pkill', ['-f', `tmux -S ${sock} `], { stdio: 'ignore' })
|
|
347
|
-
tmux(sock, 'kill-server')
|
|
348
|
-
|
|
349
|
-
// (2) tmux new-session -d with the adapter-built, shell-quoted argv. The per-peer
|
|
350
|
-
// per-runtime launch.env (zone Хранение / Рантайм-адаптеры) contributes
|
|
351
|
-
// PEER_START_ARGS (APPENDED after the adapter's base flags — extraArgs is last
|
|
352
|
-
// in every buildArgv) and extra child env. spec.extraArgs (an explicit caller
|
|
353
|
-
// override) comes first, then launch.env's; both land after the base flags.
|
|
354
|
-
// argv + child env are composed by buildLaunchInvocation (invocation.ts) — the SINGLE source shared
|
|
355
|
-
// with the supervisor serving path (cutover Block 2) so a served session is byte-identical to this
|
|
356
|
-
// tmux-launched one. The tmux path shell-quotes+joins the argv into the new-session string; the
|
|
357
|
-
// supervisor path passes the same argv array to Bun.spawn. (PEER_START_ARGS append, codex bearer
|
|
358
|
-
// stub, identity ABI, PATH — all live there now, byte-for-byte unchanged.)
|
|
359
|
-
const { argv: runtimeArgv, env: childEnv } = buildLaunchInvocation(spec, adapter, cfg)
|
|
360
|
-
const runtimeCmd = runtimeArgv.map(shellQuote).join(' ')
|
|
361
|
-
// (2.4) tmux command-log (killer-hunt socket-command layer, cmdlog.ts): WARM
|
|
362
|
-
// agent sessions start their server with `-v` so every client command —
|
|
363
|
-
// incl. a killer's `kill-server`/`kill-session`, invisible to signal
|
|
364
|
-
// forensics — lands in tmux-server-<pid>.log with client attribution
|
|
365
|
-
// (`message: client-<pid> command: …`). The starting client runs FROM the
|
|
366
|
-
// per-identity 0700 cmdlog dir (that is where tmux drops the logs; the
|
|
367
|
-
// pane's cwd is still `-c cwd`). Gated like the other observability layers
|
|
368
|
-
// on cfg.exitLogDir; skipped for always-on infra (rare relaunch = no
|
|
369
|
-
// rotation point, and not the dying class). Best-effort: a failed dir
|
|
370
|
-
// prepare just launches without -v.
|
|
371
|
-
const cmdLogDir = !cfg.alwaysOn && cfg.exitLogDir ? prepareCmdLogDir(cfg.exitLogDir, identity) : undefined
|
|
372
|
-
const start = spawnSync(
|
|
373
|
-
'tmux',
|
|
374
|
-
[
|
|
375
|
-
'-S', sock,
|
|
376
|
-
...(cmdLogDir ? ['-v'] : []),
|
|
377
|
-
'new-session', '-d', '-s', identity, '-x', '220', '-y', '50', '-c', cwd, runtimeCmd,
|
|
378
|
-
],
|
|
379
|
-
{ env: childEnv as Record<string, string>, encoding: 'utf8', ...(cmdLogDir ? { cwd: cmdLogDir } : {}) },
|
|
380
|
-
)
|
|
381
|
-
if (start.status !== 0) {
|
|
382
|
-
return fail(identity, `tmux new-session failed: ${(start.stderr ?? '').trim() || 'exit ' + start.status}`)
|
|
383
|
-
}
|
|
384
|
-
// The server/client logs exist once new-session returns — clamp them to 0600.
|
|
385
|
-
if (cmdLogDir) hardenCmdLogDir(cmdLogDir)
|
|
386
|
-
|
|
387
|
-
// (2.5) Exit-cause observability: a `pane-died` hook that records WHY this
|
|
388
|
-
// session's process dies (status/signal) at the moment of death into
|
|
389
|
-
// <exitLogDir>/exits.log, then kill-sessions the dead pane (so the
|
|
390
|
-
// daemon's has-session death detection + always-on KeepAlive stay intact).
|
|
391
|
-
// Installed ASAP — before pipe-pane — so even a runtime that dies during
|
|
392
|
-
// boot leaves a cause. No-op without cfg.exitLogDir (remain-on-exit off).
|
|
393
|
-
installExitHook(sock, identity, cfg.exitLogDir)
|
|
394
|
-
|
|
395
|
-
// (2.6) Server-death canary (canary.ts): the SERVER-level catcher pane-died
|
|
396
|
-
// structurally cannot be — a detached wait-for client OUTSIDE the server
|
|
397
|
-
// that records `ev=server-exit` + a forensics snapshot when the whole
|
|
398
|
-
// server dies dirty (SIGKILL/OOM class). Same gate as the exit hook;
|
|
399
|
-
// best-effort by construction (every failure → a state, never a throw).
|
|
400
|
-
// The ensure-state is logged (origin=launch) so the arming trail is
|
|
401
|
-
// complete in lifecycle.log: 'spawned' is the newborn-server norm here,
|
|
402
|
-
// 'failed' a newborn server starting its life UNWATCHED (death-#4
|
|
403
|
-
// postmortem hinged on this very question being unanswerable).
|
|
404
|
-
const canaryState = ensureServerCanary({ identity, sock, exitLogDir: cfg.exitLogDir, env })
|
|
405
|
-
if (canaryState !== 'skipped') {
|
|
406
|
-
appendLifecycleEvent(cfg.exitLogDir, { ev: 'canary', identity, state: canaryState, origin: 'launch' }, { env })
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
// (3) pipe-pane the session output to the per-identity log.
|
|
410
|
-
mkdirSync(cfg.logDir, { recursive: true, mode: 0o700 })
|
|
411
|
-
const paneLog = `${cfg.logDir}/${identity}.log`
|
|
412
|
-
tmux(sock, 'pipe-pane', '-t', identity, `cat >> ${shellQuote(paneLog)}`)
|
|
413
|
-
|
|
414
|
-
// (4) [REMOVED 0.2.55] Session self-TTL. Was a tmux server-side
|
|
415
|
-
// `run-shell -b -d <maxAgeSecs>` → `kill-session`, intended to bound a zombie
|
|
416
|
-
// even if no supervise sweep runs. It was a BLIND wall-clock kill armed ONCE
|
|
417
|
-
// at birth and never re-armed, so it guillotined HEALTHY long-lived sessions
|
|
418
|
-
// at exactly birth+maxAge (4 h) — the recurrent "mystery deaths".
|
|
419
|
-
// By construction any session that survives to maxAge is active: idle ones are
|
|
420
|
-
// already idle-reaped at idleSecs (1 h) by the activity-aware supervisor — so
|
|
421
|
-
// the blind kill could ONLY ever destroy live work (its sole "useful" case,
|
|
422
|
-
// daemon dead the whole interval, leaves an inert orphan nothing can deliver
|
|
423
|
-
// to, self-healed by idle-reap on daemon restart). The activity-aware idle-reap
|
|
424
|
-
// (superviseTick) is now the SOLE lifecycle bound. (alwaysOn infra never armed it.)
|
|
425
|
-
|
|
426
|
-
// (5) Router runtime (telegram): no TUI input surface — there is no boot dialog
|
|
427
|
-
// and no ready marker to gate on. The process is up; return READY.
|
|
428
|
-
if (adapter.kind === 'router') {
|
|
429
|
-
return ready(identity)
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
// (6) BOOT phase (tui) — baseline the activity proxy, answer startup dialogs,
|
|
433
|
-
// wait for the input surface, then deliver firstMessage (load-buffer +
|
|
434
|
-
// bracketed paste-buffer + Enter — the SAME byte-path as warm delivery).
|
|
435
|
-
// An EMPTY firstMessage is a BARE bring-up (folder-launch with no seed, or an
|
|
436
|
-
// attach-resume that carries no message): reach the input surface and return
|
|
437
|
-
// READY — there is no message to deliver and nothing to ready-gate on (the
|
|
438
|
-
// operator drives the session). A non-empty message takes the wake path
|
|
439
|
-
// (deliver + ready-gate on a model turn).
|
|
440
|
-
const hasMessage = firstMessage.trim().length > 0
|
|
441
|
-
const baselineMtime = adapter.newestActivityMtime(cwd) ?? 0
|
|
442
|
-
const bootIters = Math.max(1, Math.ceil(cfg.bootDeadlineSecs / 2))
|
|
443
|
-
let delivered = false
|
|
444
|
-
let lastPane = '' // the final capture — surfaced in the never-became-ready reason
|
|
445
|
-
for (let i = 0; i < bootIters && !delivered; i++) {
|
|
446
|
-
await sleep(2000)
|
|
447
|
-
if (!sessionAlive(sock, identity)) {
|
|
448
|
-
return fail(identity, 'tmux session vanished during boot')
|
|
449
|
-
}
|
|
450
|
-
const pane = capturePane(sock, identity)
|
|
451
|
-
if (!pane) continue
|
|
452
|
-
lastPane = pane
|
|
453
|
-
const dialogKeys = adapter.bootDialogKeys(pane)
|
|
454
|
-
if (dialogKeys) {
|
|
455
|
-
tmux(sock, 'send-keys', '-t', identity, ...dialogKeys)
|
|
456
|
-
continue
|
|
457
|
-
}
|
|
458
|
-
// Ф1 READY-GATE SOURCE FLIP (per-peer flag, DARK by default). When the peer's flip flag is set,
|
|
459
|
-
// run the SAME isInputReady predicate on the pty-MODEL viewport (validated 0-divergence vs
|
|
460
|
-
// capture-pane by the burn-in gate, incl. the codex resume-with-history splash-off-screen center)
|
|
461
|
-
// instead of the capture-pane scrape, with an UNCONDITIONAL capture-pane fallback. This is the
|
|
462
|
-
// ONLY changed input — dialog detection (above) and delivery + delivery-confirm (below + transport)
|
|
463
|
-
// are untouched. Flag OFF → readyView === pane (byte-identical to today).
|
|
464
|
-
let readyView = pane
|
|
465
|
-
if (readyGatePtyFlipEnabled(cfg.logDir, identity)) {
|
|
466
|
-
const geo = tmux(sock, 'display', '-p', '-t', identity, '#{pane_width}x#{pane_height}')
|
|
467
|
-
const [c, r] = geo.ok ? geo.out.trim().split('x').map(Number) : [0, 0]
|
|
468
|
-
readyView = await readyGateViewport({ logDir: cfg.logDir, identity, fallback: pane, cols: c ?? 0, rows: r ?? 0 })
|
|
469
|
-
// SERVING-path cross-check (post-flip soak gate): the model verdict is AUTHORITATIVE (readyView);
|
|
470
|
-
// record what capture-pane WOULD have said, to prove 0 model-vs-capture divergence live in the
|
|
471
|
-
// serving path. Observability only — never affects the ready decision.
|
|
472
|
-
logReadyGateCrossCheck(cfg.exitLogDir, {
|
|
473
|
-
identity,
|
|
474
|
-
modelReady: adapter.isInputReady(readyView),
|
|
475
|
-
captureReady: adapter.isInputReady(pane),
|
|
476
|
-
geom: `${c ?? 0}x${r ?? 0}`,
|
|
477
|
-
nowISO: new Date().toISOString(),
|
|
478
|
-
})
|
|
479
|
-
}
|
|
480
|
-
if (adapter.isInputReady(readyView)) {
|
|
481
|
-
if (!hasMessage) return ready(identity) // bare bring-up — session up, no message
|
|
482
|
-
// Boot delivery = load-buffer → paste-buffer -p → Enter (Ф-#8b hardening):
|
|
483
|
-
// the SAME mechanism warm delivery uses (transport.deliverViaTmux), replacing
|
|
484
|
-
// the old `send-keys -l`. send-keys retypes the message as literal keystrokes —
|
|
485
|
-
// a multi-KB envelope replays key-by-key through the pty input buffer, and the
|
|
486
|
-
// two paths could diverge (a message that survives warm delivery could be
|
|
487
|
-
// mangled at cold-wake). load-buffer hands the TUI the whole envelope as ONE
|
|
488
|
-
// bracketed paste, byte-identical to the warm path. A load/paste hiccup →
|
|
489
|
-
// retry on the next boot iteration (previously the send-keys result was
|
|
490
|
-
// ignored and a failed inject was declared delivered, failing the wake later
|
|
491
|
-
// with the wrong reason at the ready-gate).
|
|
492
|
-
const bufferName = `iapeer-boot-${process.pid}-${Date.now()}`
|
|
493
|
-
const load = spawnSync(
|
|
494
|
-
'tmux',
|
|
495
|
-
['-S', sock, 'load-buffer', '-b', bufferName, '-'],
|
|
496
|
-
{ input: firstMessage, encoding: 'utf8' },
|
|
497
|
-
)
|
|
498
|
-
if (load.status !== 0) continue
|
|
499
|
-
const paste = tmux(sock, 'paste-buffer', '-p', '-b', bufferName, '-t', identity)
|
|
500
|
-
if (!paste.ok) {
|
|
501
|
-
tmux(sock, 'delete-buffer', '-b', bufferName)
|
|
502
|
-
continue
|
|
503
|
-
}
|
|
504
|
-
await sleep(300)
|
|
505
|
-
tmux(sock, 'send-keys', '-t', identity, 'Enter')
|
|
506
|
-
tmux(sock, 'delete-buffer', '-b', bufferName)
|
|
507
|
-
delivered = true
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
if (!delivered) {
|
|
511
|
-
// Surface WHAT the pane showed when the gate gave up — a codex-resume
|
|
512
|
-
// incident (splash scrolled off, composer ready, predicate false for 240 s) was
|
|
513
|
-
// undiagnosable from the bare reason alone. Last two non-empty lines, quotes
|
|
514
|
-
// flattened (the reason lands inside a logfmt-quoted err field), clipped.
|
|
515
|
-
const tail = paneTail(lastPane)
|
|
516
|
-
return fail(
|
|
517
|
-
identity,
|
|
518
|
-
`never-became-ready (stuck at a startup prompt or boot hang)${tail ? `; pane tail: ${tail}` : ''}`,
|
|
519
|
-
)
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
// (7) READY-GATE — the activity proxy must strictly advance past baseline
|
|
523
|
-
// (the model picked up and processed the first message).
|
|
524
|
-
const readyDeadline = Date.now() + cfg.readyGateSecs * 1000
|
|
525
|
-
while (Date.now() < readyDeadline) {
|
|
526
|
-
await sleep(2000)
|
|
527
|
-
if (!sessionAlive(sock, identity)) {
|
|
528
|
-
return fail(identity, 'tmux session vanished during ready-gate')
|
|
529
|
-
}
|
|
530
|
-
const mt = adapter.newestActivityMtime(cwd) ?? 0
|
|
531
|
-
if (mt > baselineMtime) {
|
|
532
|
-
return ready(identity)
|
|
533
|
-
}
|
|
231
|
+
// pty-only: no tmux fallback — a failed host bringup fails the launch (loud, not silent).
|
|
232
|
+
appendLifecycleEvent(cfg.exitLogDir, { ev: 'pty-host-failed', identity, reason: host.error }, { env })
|
|
233
|
+
return fail(identity, `pty host bringup failed: ${host.error ?? 'unknown error'}`)
|
|
534
234
|
}
|
|
535
|
-
return fail(identity,
|
|
235
|
+
return fail(identity, `unsupported adapter kind "${adapter.kind}"`)
|
|
536
236
|
}
|
package/src/launch/launchdRun.ts
CHANGED
|
@@ -16,7 +16,6 @@
|
|
|
16
16
|
// with WorkingDirectory=<peer cwd> and EnvironmentVariables PEER_* set by launchd,
|
|
17
17
|
// so process.cwd() IS the peer cwd.
|
|
18
18
|
|
|
19
|
-
import { spawnSync } from 'child_process'
|
|
20
19
|
import { join } from 'path'
|
|
21
20
|
import { INFRA_RUNTIME_BIN_ENV, isInfraRuntime, resolveSockDir } from '../core/constants.ts'
|
|
22
21
|
import { buildProcessAddress, buildSocketPath } from '../core/socket.ts'
|
|
@@ -24,7 +23,6 @@ import { peerLogsDir, pluginLogsDir } from '../storage/index.ts'
|
|
|
24
23
|
import { readPeerProfile } from '../identity/index.ts'
|
|
25
24
|
import { getAdapter, launch } from './index.ts'
|
|
26
25
|
import { hostSessionAlive, killPtyHost } from './ptyHost.ts'
|
|
27
|
-
import { dismissCanary, signalCanaryClean } from './canary.ts'
|
|
28
26
|
import type { LaunchConfig, LaunchSpec } from './types.ts'
|
|
29
27
|
|
|
30
28
|
/** Block-watch poll cadence — seconds, deliberately NOT a tight loop (the session
|
|
@@ -37,36 +35,6 @@ const WATCH_INTERVAL_MS = 5000
|
|
|
37
35
|
* this turns that into a NON-zero diagnostic exit instead of a clean-looking run. */
|
|
38
36
|
const BOOT_RECHECK_MS = 2000
|
|
39
37
|
|
|
40
|
-
function sessionAlive(sock: string, identity: string): boolean {
|
|
41
|
-
return spawnSync('tmux', ['-S', sock, 'has-session', '-t', identity], { stdio: 'ignore' }).status === 0
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Tear down the always-on session WITH its tmux server (when this session was the
|
|
46
|
-
* last one) — the signal-exit counterpart of lifecycle.killSession, local to avoid
|
|
47
|
-
* a launch ⇆ lifecycle import. Canary-signaled first: this is a DELIBERATE stop.
|
|
48
|
-
*
|
|
49
|
-
* Fixes the defect where bootout did not kill the poller: `launchctl bootout`
|
|
50
|
-
* TERMs THIS watcher process, but the detached tmux server — and the runtime
|
|
51
|
-
* poller inside it — survived holding STALE in-memory state (e.g. a notifier
|
|
52
|
-
* trigger's old target after a same-id replace), so a plain bootout+bootstrap was
|
|
53
|
-
* NOT a real restart. With the teardown, the session
|
|
54
|
-
* dies with its watcher: bootout = full stop, bootstrap = fresh bring-up that
|
|
55
|
-
* re-reads durable state. `iapeer stop` (bootout + killSession) is unchanged —
|
|
56
|
-
* both paths now converge on the same end state.
|
|
57
|
-
*/
|
|
58
|
-
export function teardownAlwaysOnSession(sock: string, identity: string): void {
|
|
59
|
-
// Explicit canary silence (v2): channel signal (client exits 0) + dismiss the
|
|
60
|
-
// sh recorder — a signaled client alone is no longer read as deliberate.
|
|
61
|
-
signalCanaryClean(sock, identity)
|
|
62
|
-
dismissCanary(identity)
|
|
63
|
-
spawnSync('tmux', ['-S', sock, 'kill-session', '-t', identity], { stdio: 'ignore' })
|
|
64
|
-
const ls = spawnSync('tmux', ['-S', sock, 'list-sessions', '-F', '#{session_name}'], { encoding: 'utf8' })
|
|
65
|
-
if (!(ls.stdout ?? '').trim()) {
|
|
66
|
-
spawnSync('tmux', ['-S', sock, 'kill-server'], { stdio: 'ignore' })
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
38
|
/**
|
|
71
39
|
* Build the always-on LaunchSpec for an infra peer, reading intelligence from the
|
|
72
40
|
* local peer-profile.json. launchd sets WorkingDirectory = peer cwd, so that file
|
|
@@ -116,13 +84,10 @@ export async function runAlwaysOn(personality: string, runtime: string, cwd: str
|
|
|
116
84
|
const sock = buildSocketPath(runtime, personality, sockDir)
|
|
117
85
|
const adapter = getAdapter(runtime)
|
|
118
86
|
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
|
|
123
|
-
// death BOTH go false → block-watch exits → run-infra exits → launchd KeepAlive respawns (the SOLE
|
|
124
|
-
// respawn owner; the supervisor never self-respawns — H4 held).
|
|
125
|
-
const isAlive = (): boolean => hostSessionAlive(identity) || sessionAlive(sock, identity)
|
|
87
|
+
// pty-only liveness: a router is supervisor-HOSTED. Probe by RUNTIME STATE (not the `.pty-host`
|
|
88
|
+
// marker). On child death it goes false → block-watch exits → run-infra exits → launchd KeepAlive
|
|
89
|
+
// respawns (the SOLE respawn owner; the supervisor never self-respawns — H4 held).
|
|
90
|
+
const isAlive = (): boolean => hostSessionAlive(identity)
|
|
126
91
|
|
|
127
92
|
const cfg: LaunchConfig = {
|
|
128
93
|
claudeBin: env.CLAUDE_BIN ?? 'claude',
|
|
@@ -194,19 +159,11 @@ export async function runAlwaysOn(personality: string, runtime: string, cwd: str
|
|
|
194
159
|
})
|
|
195
160
|
interrupt = null
|
|
196
161
|
}
|
|
197
|
-
// Signal-initiated
|
|
198
|
-
//
|
|
199
|
-
// holding stale in-memory state
|
|
200
|
-
//
|
|
201
|
-
|
|
202
|
-
// Signal-initiated stop (bootout / shutdown / kickstart -k) tears the session down WITH this watcher.
|
|
203
|
-
// A hosted router → kill the supervisor daemon (killPtyHost: SIGTERM → it SIGKILLs the child + cleans
|
|
204
|
-
// up); a tmux router → the tmux teardown. Pick by the live backing. A NATURAL death (stop=false) skips
|
|
205
|
-
// this: nothing to tear down, exit 0 → KeepAlive respawns a fresh bring-up.
|
|
206
|
-
if (stop) {
|
|
207
|
-
if (hostSessionAlive(identity)) killPtyHost(identity)
|
|
208
|
-
else teardownAlwaysOnSession(sock, identity)
|
|
209
|
-
}
|
|
162
|
+
// Signal-initiated stop (bootout / shutdown / kickstart -k) tears the session down WITH this watcher —
|
|
163
|
+
// kill the supervisor daemon (killPtyHost: SIGTERM → it SIGKILLs the child + cleans up sock/pid/serve)
|
|
164
|
+
// so the detached child does not outlive bootout holding stale in-memory state. A NATURAL death
|
|
165
|
+
// (stop=false) skips this: nothing to tear down, exit 0 → KeepAlive respawns a fresh bring-up.
|
|
166
|
+
if (stop) killPtyHost(identity)
|
|
210
167
|
return 0
|
|
211
168
|
}
|
|
212
169
|
|
package/src/launch/ptyHost.ts
CHANGED
|
@@ -26,18 +26,6 @@ import type { RuntimeAdapter } from './types.ts'
|
|
|
26
26
|
export const HOST_COLS = 220
|
|
27
27
|
export const HOST_ROWS = 50
|
|
28
28
|
|
|
29
|
-
/** Per-peer pty-hosting decision. pty-supervisor hosting is the DEFAULT — a fresh peer is hosted under
|
|
30
|
-
* the supervisor (not `tmux new-session`) WITHOUT any marker (pty out of the box). A peer opts OUT
|
|
31
|
-
* to the legacy tmux path with a `<logDir>/<identity>.no-pty-host` marker (the dev/debug escape, valid
|
|
32
|
-
* only while tmux still exists). Read in TWO places, both of which want "is this peer pty-hosted": the
|
|
33
|
-
* spawn-gate (launch — host vs tmux) and the shadow observer (skip pty-hosted peers, they have no tmux
|
|
34
|
-
* to compare; an opt-OUT peer is tmux → compared). Legacy `.pty-host` opt-in markers are now REDUNDANT
|
|
35
|
-
* (pty is the default) and harmless — ignored here, so the 29 migrated peers keep hosting unchanged.
|
|
36
|
-
* The golden spawn-fallback (startPtyHost fail → tmux, in launch) is SEPARATE and unaffected: a
|
|
37
|
-
* pty-default peer whose host bring-up fails still falls through to tmux — never worse on spawn. */
|
|
38
|
-
export function ptyHostEnabled(logDir: string, identity: string): boolean {
|
|
39
|
-
return !existsSync(`${logDir}/${identity}.no-pty-host`)
|
|
40
|
-
}
|
|
41
29
|
|
|
42
30
|
/** The supervisor's run dir (session sockets/pids/serve-specs). Env-isolatable via IAPEER_ROOT. */
|
|
43
31
|
export function hostRunDir(): string {
|
|
@@ -1,86 +1,15 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// production predicate (adapter.isInputReady) and an UNCONDITIONAL capture-pane fallback.
|
|
8
|
-
//
|
|
9
|
-
// Why this is low-radius: it is a viewport-SOURCE swap, NOT a predicate change.
|
|
10
|
-
// The model verdict was validated 0-divergence vs capture-pane on the readiness predicate by the
|
|
11
|
-
// burn-in gate (incl. the codex resume-with-history splash-off-screen center cell) before any peer
|
|
12
|
-
// is flipped. It touches ONLY the isInputReady INPUT — dialog detection upstream and the delivery +
|
|
13
|
-
// delivery-confirm downstream (submitIntoTui's submit-landed + deliverViaTmux's transcript-mtime
|
|
14
|
-
// advance, the "real model advancement" final-arbiter) are untouched, so the canonical
|
|
15
|
-
// harmless-false-glyph property is preserved by construction.
|
|
16
|
-
//
|
|
17
|
-
// FLAG: a per-peer marker `<logDir>/<identity>.pty-readygate`. Absent → capture-pane (byte-identical
|
|
18
|
-
// to today). Present → the model viewport. Per-peer so the first flip is a single canary with the
|
|
19
|
-
// whole fleet still on tmux; dynamic (touch/rm, no daemon restart). @xterm is dynamic-imported ONLY
|
|
20
|
-
// when the flag is set, so the launch hot path stays @xterm-free with the flip off.
|
|
1
|
+
// Pane-log MODEL leaf — render a supervisor pane-log (raw child pty bytes) through @xterm/headless to
|
|
2
|
+
// the readiness viewport + composer-occupancy verdict. The pty-hosted ready-gate (ptyHost.waitHostReady)
|
|
3
|
+
// and the warm-deliver busy-composer guard (transport) both read the session state from this model —
|
|
4
|
+
// a hosted session has no capture-pane, so the pane-log model is the ONLY readiness/occupancy source.
|
|
5
|
+
// @xterm is dynamic-imported (loaded only when a model is actually built), so callers that never build
|
|
6
|
+
// one stay @xterm-free.
|
|
21
7
|
|
|
22
|
-
import {
|
|
23
|
-
import { join } from 'node:path'
|
|
8
|
+
import { closeSync, existsSync, fstatSync, openSync, readSync } from 'node:fs'
|
|
24
9
|
import type { Terminal } from '@xterm/headless' // TYPE-ONLY (erased) — the runtime @xterm load is the dynamic import below
|
|
25
10
|
|
|
26
11
|
const SEED_BYTES = 4 * 1024 * 1024
|
|
27
12
|
|
|
28
|
-
/**
|
|
29
|
-
* SERVING-path cross-check log (the post-flip soak gate). When a peer is flipped, the MODEL verdict
|
|
30
|
-
* is AUTHORITATIVE in the cold-wake ready-gate; this records, per evaluation, what capture-pane WOULD
|
|
31
|
-
* have said — so a soak can prove 0 model-vs-capture divergence IN THE SERVING PATH (the read-only
|
|
32
|
-
* burn-in measures continuously but never drove the actual ready decision). Append-only jsonl under
|
|
33
|
-
* the event-log dir. Best-effort: a log hiccup never affects the ready decision. No-op without a dir.
|
|
34
|
-
*/
|
|
35
|
-
export function logReadyGateCrossCheck(
|
|
36
|
-
eventLogDir: string | undefined,
|
|
37
|
-
rec: { identity: string; modelReady: boolean; captureReady: boolean; geom: string; nowISO: string },
|
|
38
|
-
): void {
|
|
39
|
-
if (!eventLogDir) return
|
|
40
|
-
try {
|
|
41
|
-
appendFileSync(
|
|
42
|
-
join(eventLogDir, 'readygate-flip.jsonl'),
|
|
43
|
-
JSON.stringify({
|
|
44
|
-
ts: rec.nowISO,
|
|
45
|
-
identity: rec.identity,
|
|
46
|
-
modelReady: rec.modelReady,
|
|
47
|
-
captureReady: rec.captureReady,
|
|
48
|
-
diverged: rec.modelReady !== rec.captureReady,
|
|
49
|
-
geom: rec.geom,
|
|
50
|
-
}) + '\n',
|
|
51
|
-
)
|
|
52
|
-
} catch {
|
|
53
|
-
/* observability only — never block the ready-gate */
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** Per-peer ready-gate flip flag: the marker `<logDir>/<identity>.pty-readygate`. */
|
|
58
|
-
export function readyGatePtyFlipEnabled(logDir: string, identity: string): boolean {
|
|
59
|
-
return existsSync(`${logDir}/${identity}.pty-readygate`)
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export interface ReadyGateViewportOpts {
|
|
63
|
-
logDir: string
|
|
64
|
-
identity: string
|
|
65
|
-
/** The capture-pane scrape — returned verbatim when the flag is off OR any model build fails. */
|
|
66
|
-
fallback: string
|
|
67
|
-
cols: number
|
|
68
|
-
rows: number
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Resolve the ready-gate viewport. Flag OFF → `fallback` (capture-pane) verbatim — zero work, the
|
|
73
|
-
* caller never even reaches here unless it pre-checked the flag, but this stays safe regardless.
|
|
74
|
-
* Flag ON → the pty-model viewport as plain text. ANY failure (unreadable/short log, @xterm hiccup)
|
|
75
|
-
* → `fallback`: the flip can never make readiness WORSE than capture-pane today (tmux is the floor).
|
|
76
|
-
*/
|
|
77
|
-
export async function readyGateViewport(opts: ReadyGateViewportOpts): Promise<string> {
|
|
78
|
-
if (!readyGatePtyFlipEnabled(opts.logDir, opts.identity)) return opts.fallback
|
|
79
|
-
if (!(opts.cols > 0 && opts.rows > 0)) return opts.fallback
|
|
80
|
-
const view = await paneLogViewport(`${opts.logDir}/${opts.identity}.log`, opts.cols, opts.rows)
|
|
81
|
-
return view ?? opts.fallback // tmux-fallback is unconditional — never worse than today
|
|
82
|
-
}
|
|
83
|
-
|
|
84
13
|
/**
|
|
85
14
|
* Read a pane-log's tail into a headless @xterm and render the viewport as plain text — the SAME
|
|
86
15
|
* 0-divergence-validated model the ready-gate flip uses, shared with the spawn-flip hosted path (a
|
|
@@ -124,7 +53,7 @@ async function loadPaneLogModel(log: string, cols: number, rows: number, startBy
|
|
|
124
53
|
export async function paneLogViewport(log: string, cols: number, rows: number, startByte = 0): Promise<string | null> {
|
|
125
54
|
const term = await loadPaneLogModel(log, cols, rows, startByte)
|
|
126
55
|
if (!term) return null
|
|
127
|
-
const { modelToPlainText } = await import('../
|
|
56
|
+
const { modelToPlainText } = await import('../supervisor/render.ts')
|
|
128
57
|
return modelToPlainText(term, cols, rows)
|
|
129
58
|
}
|
|
130
59
|
|
|
@@ -152,6 +81,6 @@ export async function paneLogViewport(log: string, cols: number, rows: number, s
|
|
|
152
81
|
export async function paneLogComposerOccupied(log: string, cols: number, rows: number, runtime: string): Promise<boolean> {
|
|
153
82
|
const term = await loadPaneLogModel(log, cols, rows)
|
|
154
83
|
if (!term) return false // unreadable pane-log → no observed human composer text → deliver, never stall (see JSDoc)
|
|
155
|
-
const { composerOccupancyFromModel } = await import('../
|
|
84
|
+
const { composerOccupancyFromModel } = await import('../supervisor/render.ts')
|
|
156
85
|
return composerOccupancyFromModel(term, cols, rows, runtime)
|
|
157
86
|
}
|