@agfpd/iapeer 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/index.ts +13 -80
- package/src/cli/listTui.ts +4 -14
- package/src/launch/cmdlog.ts +7 -110
- package/src/launch/codexHooksTrust.ts +0 -3
- package/src/launch/index.ts +14 -314
- package/src/launch/launchdRun.ts +9 -52
- package/src/launch/ptyHost.ts +0 -12
- package/src/launch/readyGateModel.ts +9 -80
- package/src/lifecycle/index.ts +25 -101
- package/src/registry/index.ts +9 -7
- package/src/supervisor/daemon.ts +1 -1
- package/src/supervisor/paths.ts +1 -10
- package/src/{shadow → supervisor}/render.ts +10 -10
- package/src/transport/index.ts +41 -400
- package/src/update/index.ts +0 -9
- package/src/launch/canary.ts +0 -239
- package/src/shadow/gate.ts +0 -396
- package/src/shadow/index.ts +0 -174
- package/src/shadow/install.ts +0 -256
package/src/launch/canary.ts
DELETED
|
@@ -1,239 +0,0 @@
|
|
|
1
|
-
// Server-death canary — the SERVER-level catcher for the exit-cause gap the
|
|
2
|
-
// pane-died hook structurally cannot close (контракт: pane-died needs a living
|
|
3
|
-
// tmux event loop; SIGKILL/OOM to the tmux SERVER runs no hook). Raised from the
|
|
4
|
-
// deferred backlog after the SECOND live case of the class (both: a healthy
|
|
5
|
-
// session with active Bash subprocessing, the whole server gone silently, zero
|
|
6
|
-
// system traces).
|
|
7
|
-
//
|
|
8
|
-
// Mechanism: one tiny detached `sh` per LIVE tmux server, holding a blocking
|
|
9
|
-
// client `tmux -S <sock> wait-for iap-canary-<identity>` — a process OUTSIDE the
|
|
10
|
-
// dying server, connected via its socket, so the server's death (any cause,
|
|
11
|
-
// including SIGKILL) is observed the moment the connection drops. Protocol (v2 —
|
|
12
|
-
// a live death proved v1's exit-code trust wrong twice over: the killer
|
|
13
|
-
// took the SERVER and the canary CLIENT together, AND a TERMed tmux client
|
|
14
|
-
// returns rc=0 — indistinguishable from a clean channel signal — so the rc-based
|
|
15
|
-
// `0 || ≥128 → silent` guard silenced a real death):
|
|
16
|
-
// • DELIBERATE silence is an explicit act, never an exit-code inference:
|
|
17
|
-
// every deliberate teardown (idle-reap / stop / pre-clean / pane-died hook /
|
|
18
|
-
// bootout teardown) signals the channel (`wait-for -S`) AND dismisses the
|
|
19
|
-
// sh recorder (`dismissCanary` → TERM → trap → silent exit). POSIX trap
|
|
20
|
-
// semantics make this race-free: the trap runs before any recording.
|
|
21
|
-
// • When wait-for returns — ANY code — the script sleeps 2 s (a concurrent
|
|
22
|
-
// dismissal TERM wins here), then probes the ORIGINAL SERVER PID captured at
|
|
23
|
-
// arm time: still alive → exit silently (a lost canary is re-armed and logged
|
|
24
|
-
// by the supervise retrofit within a tick); gone with nobody having dismissed
|
|
25
|
-
// us → the death is real and unclaimed → ONE logfmt line `ev=server-exit`
|
|
26
|
-
// into exits.log (the per-peer death-cause home, next to pane-died's
|
|
27
|
-
// `ev=session-exit`) + a forensics snapshot (vm_stat / swap / top-RSS ps /
|
|
28
|
-
// fresh DiagnosticReports) captured within seconds — the evidence the 60 s
|
|
29
|
-
// supervise tick can never recover (zero system traces was the recurring
|
|
30
|
-
// investigation outcome). PID identity matters: a wake/attach can replace the
|
|
31
|
-
// tmux server on the same socket before the 2 s grace elapses; probing only
|
|
32
|
-
// the socket would mistake the successor for the dead original and lose the
|
|
33
|
-
// canary record (a live canary-gap).
|
|
34
|
-
// The raw wait_rc still ATTRIBUTES the death (cause=server-vanished /
|
|
35
|
-
// signaled-server-gone / client-killed-server-gone).
|
|
36
|
-
// Residual blind spot (structural): a killer that SIGKILLs the sh recorder
|
|
37
|
-
// itself leaves no in-process way to record. With v2 the ABSENCE of a record on
|
|
38
|
-
// a server-dead reap narrows the diagnosis to exactly that shape; the canary
|
|
39
|
-
// ensure-state lines in lifecycle.log (origin=launch/retrofit) evidence the
|
|
40
|
-
// churn post-hoc.
|
|
41
|
-
//
|
|
42
|
-
// The canary is pure observability: it never wakes, reaps, restarts or otherwise
|
|
43
|
-
// manages anything (H4-compatible by construction), it fires at most once, and
|
|
44
|
-
// every failure to spawn/record is swallowed (never load-bearing). The daemon's
|
|
45
|
-
// reaped-gone death-class (classifyGoneSession) remains the detection backstop
|
|
46
|
-
// when no canary was running.
|
|
47
|
-
|
|
48
|
-
import { spawn, spawnSync } from 'child_process'
|
|
49
|
-
import { mkdirSync } from 'fs'
|
|
50
|
-
import { join } from 'path'
|
|
51
|
-
import { resolveExecutable } from './launchd.ts'
|
|
52
|
-
import { cmdLogDirFor } from './cmdlog.ts'
|
|
53
|
-
|
|
54
|
-
/** The exit-cause log file inside `exitLogDir` (sibling to lifecycle.log).
|
|
55
|
-
* Lives here (not index.ts) so canary ⇆ launch never import-cycle; index.ts
|
|
56
|
-
* re-exports it, keeping the public import surface unchanged. */
|
|
57
|
-
export function exitLogPath(exitLogDir: string): string {
|
|
58
|
-
return `${exitLogDir}/exits.log`
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/** The per-identity tmux wait-for channel the clean-teardown paths signal. */
|
|
62
|
-
export function canaryChannel(identity: string): string {
|
|
63
|
-
return `iap-canary-${identity}`
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/** Forensics snapshots live next to exits.log: `<exitLogDir>/server-deaths/`. */
|
|
67
|
-
export function serverDeathsDir(exitLogDir: string): string {
|
|
68
|
-
return join(exitLogDir, 'server-deaths')
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export interface CanaryScriptOptions {
|
|
72
|
-
identity: string
|
|
73
|
-
sock: string
|
|
74
|
-
/** Absolute tmux path — the detached canary may outlive any rich PATH. */
|
|
75
|
-
tmuxBin: string
|
|
76
|
-
exitLogFile: string
|
|
77
|
-
forensicsDir: string
|
|
78
|
-
/** The identity's tmux-cmdlog dir (<exitLogDir>/tmux-cmdlog/<identity>). When set,
|
|
79
|
-
* the death forensics tail the dead server's `tmux-server-<SPID>.log` — the ONLY
|
|
80
|
-
* witness of a client-COMMAND kill (`kill-server`/`kill-session`), which leaves zero
|
|
81
|
-
* signals (so signal forensics see nothing) and is otherwise wiped by the respawn's
|
|
82
|
-
* prepareCmdLogDir before anyone reads it. Best-effort: absent log → grep no-ops. */
|
|
83
|
-
cmdLogDir?: string
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* Build the canary shell script (PURE — unit-testable). Quoting: identity is a
|
|
88
|
-
* runtime-personality (`[a-z0-9-]`), sock/log paths are ~/.iapeer-style (no
|
|
89
|
-
* single quotes) — same assumption as exitCauseHook. All forensic tools are
|
|
90
|
-
* /usr/bin- or /bin-resident, so the script survives launchd's minimal PATH;
|
|
91
|
-
* only tmux needs the baked absolute path.
|
|
92
|
-
*/
|
|
93
|
-
export function canaryScript(o: CanaryScriptOptions): string {
|
|
94
|
-
const ch = canaryChannel(o.identity)
|
|
95
|
-
return [
|
|
96
|
-
// Server PID captured while alive — the postmortem grep key for system logs.
|
|
97
|
-
`SPID="$('${o.tmuxBin}' -S '${o.sock}' display-message -p '#{pid}' 2>/dev/null)"`,
|
|
98
|
-
// A signal to the SH WRAPPER is deliberate dismissal (dismissCanary) or host
|
|
99
|
-
// shutdown → silent. POSIX: the trap runs after the foreground command
|
|
100
|
-
// completes — so a TERM delivered during wait-for/sleep always exits us
|
|
101
|
-
// BEFORE any recording below (the race-free deliberate-silence guarantee).
|
|
102
|
-
`trap 'exit 0' HUP INT TERM`,
|
|
103
|
-
`'${o.tmuxBin}' -S '${o.sock}' wait-for '${ch}'`,
|
|
104
|
-
`rc=$?`,
|
|
105
|
-
// NO wait-for exit code is trusted as "deliberate" by itself — PROVEN live
|
|
106
|
-
// (a death postmortem): a TERM to the tmux CLIENT returns rc=0, identical
|
|
107
|
-
// to a clean channel signal, so an external killer sweeping server+client
|
|
108
|
-
// rides the clean-looking code straight past any rc-based guard (v1's
|
|
109
|
-
// `rc=0 || rc>=128 → silent` was exactly that hole). v2 contract instead:
|
|
110
|
-
// • deliberate teardowns DISMISS this sh (TERM → trap above) — the sleep
|
|
111
|
-
// below gives a concurrently-delivered dismissal time to win;
|
|
112
|
-
// • then the ORIGINAL SERVER PID's liveness, not the exit code, decides: alive →
|
|
113
|
-
// nothing to record (a lost canary is re-armed and logged by the
|
|
114
|
-
// supervise retrofit within a tick); original server dead and nobody
|
|
115
|
-
// dismissed us → the death is real and unclaimed → record it. IMPORTANT:
|
|
116
|
-
// check the ORIGINAL SPID, not merely "some server answers on the same
|
|
117
|
-
// socket". A wake/attach can replace the server inside this 2 s window;
|
|
118
|
-
// socket-liveness would then silently swallow the predecessor's death.
|
|
119
|
-
`sleep 2`,
|
|
120
|
-
`if [ -n "$SPID" ] && kill -0 "$SPID" 2>/dev/null; then exit 0; fi`,
|
|
121
|
-
`if [ -z "$SPID" ] && '${o.tmuxBin}' -S '${o.sock}' has-session 2>/dev/null; then exit 0; fi`,
|
|
122
|
-
// The exit code still ATTRIBUTES the recorded death (raw wait_rc is kept):
|
|
123
|
-
// rc=0 → signaled-server-gone (channel signal or client-TERM, server died)
|
|
124
|
-
// rc≥128 → client-killed-server-gone (client took a non-TERM kill)
|
|
125
|
-
// else → server-vanished (connection drop — SIGKILL/OOM class)
|
|
126
|
-
`cause=server-vanished`,
|
|
127
|
-
`if [ "$rc" -eq 0 ]; then cause=signaled-server-gone; fi`,
|
|
128
|
-
`if [ "$rc" -ge 128 ]; then cause=client-killed-server-gone; fi`,
|
|
129
|
-
// The server is gone under us — record, within seconds of the death.
|
|
130
|
-
`ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"`,
|
|
131
|
-
`f='${o.forensicsDir}/${o.identity}'-"$(date +%s)".txt`,
|
|
132
|
-
`{`,
|
|
133
|
-
` echo "server-death identity=${o.identity} ts=$ts server_pid=$SPID wait_rc=$rc cause=$cause"`,
|
|
134
|
-
` echo '--- vm_stat'; vm_stat`,
|
|
135
|
-
` echo '--- swapusage'; /usr/sbin/sysctl -n vm.swapusage`,
|
|
136
|
-
` echo '--- ps-top-rss'; ps axo pid,ppid,rss,etime,command | sort -rn -k3 | head -25`,
|
|
137
|
-
` echo '--- diagnosticreports-user'; ls -t "$HOME/Library/Logs/DiagnosticReports" 2>/dev/null | head`,
|
|
138
|
-
` echo '--- diagnosticreports-system'; ls -t /Library/Logs/DiagnosticReports 2>/dev/null | head`,
|
|
139
|
-
// The death-class discriminator signal-forensics structurally cannot see: a tmux
|
|
140
|
-
// server killed by a CLIENT COMMAND (`kill-server`/`kill-session`) leaves zero
|
|
141
|
-
// signals. Its only witness is the dead server's own -v cmdlog, which the respawn
|
|
142
|
-
// wipes seconds later — captured HERE (canary fires ~+2 s, respawn ~+7 s) so the
|
|
143
|
-
// killer's `client-<pid> command: …` survives into the forensics.
|
|
144
|
-
...(o.cmdLogDir
|
|
145
|
-
? [
|
|
146
|
-
` echo '--- cmdlog-tail (last client commands; a kill-server/kill-session here IS the killer)'`,
|
|
147
|
-
` tail -c 131072 '${o.cmdLogDir}/tmux-server-'"$SPID"'.log' 2>/dev/null | grep -aE 'command:' | tail -30`,
|
|
148
|
-
]
|
|
149
|
-
: []),
|
|
150
|
-
`} > "$f" 2>&1`,
|
|
151
|
-
`printf 'ts=%s ev=server-exit identity=${o.identity} server_pid=%s wait_rc=%s cause=%s forensics=%s\\n' "$ts" "$SPID" "$rc" "$cause" "$f" >> '${o.exitLogFile}'`,
|
|
152
|
-
].join('\n')
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export type CanaryEnsureState = 'spawned' | 'already' | 'skipped' | 'failed'
|
|
156
|
-
|
|
157
|
-
export interface EnsureCanaryOptions {
|
|
158
|
-
identity: string
|
|
159
|
-
sock: string
|
|
160
|
-
/** Same gate as installExitHook: falsy → observability off, no canary. */
|
|
161
|
-
exitLogDir?: string
|
|
162
|
-
env?: NodeJS.ProcessEnv
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/**
|
|
166
|
-
* Ensure ONE canary is watching this identity's tmux server. Idempotent via a
|
|
167
|
-
* pgrep on the wait-for channel (anchored so an identity can never match another
|
|
168
|
-
* identity's prefix). Called from launch (newborn server) and from the supervise
|
|
169
|
-
* tick's alive-branch (retrofits canaries onto a fleet launched by older code —
|
|
170
|
-
* coverage within one tick of a deploy, no session restarts). Best-effort:
|
|
171
|
-
* every failure returns a state, never throws.
|
|
172
|
-
*/
|
|
173
|
-
export function ensureServerCanary(o: EnsureCanaryOptions): CanaryEnsureState {
|
|
174
|
-
if (!o.exitLogDir) return 'skipped'
|
|
175
|
-
const env = o.env ?? process.env
|
|
176
|
-
try {
|
|
177
|
-
const probe = spawnSync('pgrep', ['-f', `wait-for.*${canaryChannel(o.identity)}([^a-z0-9-]|$)`], {
|
|
178
|
-
stdio: 'ignore',
|
|
179
|
-
env: env as Record<string, string>,
|
|
180
|
-
})
|
|
181
|
-
if (probe.status === 0) return 'already'
|
|
182
|
-
const tmuxBin = resolveExecutable('tmux', env)
|
|
183
|
-
if (!tmuxBin) return 'failed'
|
|
184
|
-
const forensicsDir = serverDeathsDir(o.exitLogDir)
|
|
185
|
-
mkdirSync(forensicsDir, { recursive: true, mode: 0o700 })
|
|
186
|
-
mkdirSync(o.exitLogDir, { recursive: true, mode: 0o700 })
|
|
187
|
-
const script = canaryScript({
|
|
188
|
-
identity: o.identity,
|
|
189
|
-
sock: o.sock,
|
|
190
|
-
tmuxBin,
|
|
191
|
-
exitLogFile: exitLogPath(o.exitLogDir),
|
|
192
|
-
forensicsDir,
|
|
193
|
-
cmdLogDir: cmdLogDirFor(o.exitLogDir, o.identity),
|
|
194
|
-
})
|
|
195
|
-
const child = spawn('/bin/sh', ['-c', script], { detached: true, stdio: 'ignore', env })
|
|
196
|
-
child.unref()
|
|
197
|
-
return 'spawned'
|
|
198
|
-
} catch {
|
|
199
|
-
return 'failed' // observability is best-effort — never block a launch/tick
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/**
|
|
204
|
-
* Signal the canary that the upcoming server teardown is DELIBERATE (idle-reap /
|
|
205
|
-
* stop / launch pre-clean / empty-server kill) so it exits silently instead of
|
|
206
|
-
* recording a false server-death. Best-effort: no server / no canary on the
|
|
207
|
-
* channel → harmless no-op.
|
|
208
|
-
*/
|
|
209
|
-
export function signalCanaryClean(sock: string, identity: string): void {
|
|
210
|
-
try {
|
|
211
|
-
spawnSync('tmux', ['-S', sock, 'wait-for', '-S', canaryChannel(identity)], { stdio: 'ignore' })
|
|
212
|
-
} catch {
|
|
213
|
-
/* best-effort */
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/** The pgrep/pkill ERE matching BOTH canary processes of ONE identity — the sh
|
|
218
|
-
* wrapper (its -c script quotes the channel: `…'iap-canary-<id>'…`) and the
|
|
219
|
-
* tmux client (argv ends with the bare channel). Anchored so an identity can
|
|
220
|
-
* never match another identity's prefix (claude-alice ≠ claude-alice-bob). */
|
|
221
|
-
export function canaryProcessPattern(identity: string): string {
|
|
222
|
-
return `${canaryChannel(identity)}([^a-z0-9-]|$)`
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/**
|
|
226
|
-
* Dismiss this identity's canary BEFORE a deliberate server teardown: TERM the
|
|
227
|
-
* sh wrapper (trap → silent exit, race-free — the trap always runs before the
|
|
228
|
-
* v2 recording branch) and the tmux client. The explicit counterpart of the
|
|
229
|
-
* channel signal: with v2 a signaled CLIENT alone is no longer read as
|
|
230
|
-
* deliberate, so every deliberate path must dismiss the RECORDER (the sh).
|
|
231
|
-
* Best-effort: no canary running → harmless no-op (pkill exits 1).
|
|
232
|
-
*/
|
|
233
|
-
export function dismissCanary(identity: string): void {
|
|
234
|
-
try {
|
|
235
|
-
spawnSync('pkill', ['-f', canaryProcessPattern(identity)], { stdio: 'ignore' })
|
|
236
|
-
} catch {
|
|
237
|
-
/* best-effort */
|
|
238
|
-
}
|
|
239
|
-
}
|
package/src/shadow/gate.ts
DELETED
|
@@ -1,396 +0,0 @@
|
|
|
1
|
-
// Gate-grade instrumentation for the tmux→pty fidelity burn-in (cutover Block 1).
|
|
2
|
-
//
|
|
3
|
-
// PURE classifiers + one stateful tracker, with NO tmux and NO @xterm import — the observer
|
|
4
|
-
// loop feeds it booleans, so all classification is hermetically unit-testable. The synthetic
|
|
5
|
-
// acceptance: PROVE the instrument classifies known-behind vs known-ahead and records coverage
|
|
6
|
-
// correctly BEFORE we trust any pty verdict for a flip.
|
|
7
|
-
//
|
|
8
|
-
// It operationalizes the adjudicated gate criterion (NOT a flat sample count):
|
|
9
|
-
// • DIRECTION — pty-BEHIND (pty=False,tmux=True: a stale frame while live tmux moved on) is
|
|
10
|
-
// the reattach-stale class → ZERO tolerance, always hard. pty-AHEAD (pty=True,tmux=False)
|
|
11
|
-
// is benign ONLY when it (a) opened in the boot window, (b) self-heals within the converge
|
|
12
|
-
// budget, and (c) is not steady-state. Anything else → hard. For READY-GATE a non-converging
|
|
13
|
-
// ahead is a FALSE-ready (early-delivery → msg-loss class), so it is hard, never benign.
|
|
14
|
-
// • COVERAGE — which matrix cells the burn-in actually traversed {runtime × {idle,
|
|
15
|
-
// model-advancing, composer-busy}} + births (fresh vs wake-resume-with-history) + restarts +
|
|
16
|
-
// history-bearing samples + fleet width. "Clean" counts only over traversed cells (idle
|
|
17
|
-
// alone is a false green).
|
|
18
|
-
|
|
19
|
-
export type Verdict = 'occupancy' | 'readyGate' | 'liveness'
|
|
20
|
-
export const VERDICTS: Verdict[] = ['occupancy', 'readyGate', 'liveness']
|
|
21
|
-
|
|
22
|
-
export type Direction = 'agree' | 'pty-behind' | 'pty-ahead'
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Direction of a (pty, tmux) verdict pair. tmux is AUTHORITATIVE; `true` is the
|
|
26
|
-
* active/ready/alive pole. pty UNDER-reporting it (pty=false, tmux=true) is BEHIND — the
|
|
27
|
-
* stale-frame class (zero tolerance). pty OVER-reporting (pty=true, tmux=false) is AHEAD.
|
|
28
|
-
*/
|
|
29
|
-
export function classifyDirection(ptyVal: boolean, tmuxVal: boolean): Direction {
|
|
30
|
-
if (ptyVal === tmuxVal) return 'agree'
|
|
31
|
-
return tmuxVal ? 'pty-behind' : 'pty-ahead'
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export type SampleState = 'idle' | 'model-advancing' | 'composer-busy'
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Per-sample fleet state for the coverage matrix. composer-busy when tmux (authoritative) sees
|
|
38
|
-
* human occupancy; else model-advancing when output churned this tick (the model is actively
|
|
39
|
-
* rendering — the NON-trivial fidelity regime, not at rest); else idle. The split is what keeps
|
|
40
|
-
* an idle burn-in from reading as a false green.
|
|
41
|
-
*/
|
|
42
|
-
export function classifySampleState(occTmux: boolean, bytesFedThisTick: number): SampleState {
|
|
43
|
-
if (occTmux) return 'composer-busy'
|
|
44
|
-
if (bytesFedThisTick > 0) return 'model-advancing'
|
|
45
|
-
return 'idle'
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface VerdictSample {
|
|
49
|
-
/** null when the tmux side was unavailable this tick (capture failed) — skipped, not scored. */
|
|
50
|
-
pty: boolean
|
|
51
|
-
tmux: boolean | null
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export interface PeerSample {
|
|
55
|
-
peer: string
|
|
56
|
-
runtime: string
|
|
57
|
-
occupancy: VerdictSample
|
|
58
|
-
readyGate: VerdictSample
|
|
59
|
-
liveness: VerdictSample
|
|
60
|
-
/** Bytes appended to this peer's pane-log since the previous tick (drives state coverage). */
|
|
61
|
-
bytesFed: number
|
|
62
|
-
/** The model already carries scrollback (transcript taller than the pane) — a history-bearing
|
|
63
|
-
* peer; at first sight this means it woke WITH resume-history (vs a fresh empty birth). */
|
|
64
|
-
hasHistory: boolean
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export interface GateConfig {
|
|
68
|
-
/** Consecutive tmux-ready ticks after which a peer is considered booted (out of the boot
|
|
69
|
-
* window). Default 3. */
|
|
70
|
-
settleTicks?: number
|
|
71
|
-
/** Max ticks an ahead-transient may stay open and still count as a benign self-heal.
|
|
72
|
-
* Default 3. */
|
|
73
|
-
convergeBudgetTicks?: number
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export interface HardViolation {
|
|
77
|
-
peer: string
|
|
78
|
-
runtime: string
|
|
79
|
-
verdict: Verdict
|
|
80
|
-
tick: number
|
|
81
|
-
kind: 'behind' | 'ahead-steady' | 'ahead-nonconverged'
|
|
82
|
-
detail?: string
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
interface OpenAhead {
|
|
86
|
-
startTick: number
|
|
87
|
-
bootWindow: boolean
|
|
88
|
-
flaggedNonConverged: boolean
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
interface PeerState {
|
|
92
|
-
runtime: string
|
|
93
|
-
firstTick: number
|
|
94
|
-
readyStreak: number
|
|
95
|
-
bootDone: boolean
|
|
96
|
-
/** History present at first sight ⇒ this peer woke WITH resume-history (vs a fresh empty birth)
|
|
97
|
-
* — the population the center cell (codex×wake-resume×history) is measured on. */
|
|
98
|
-
bornWithHistory: boolean
|
|
99
|
-
/** This peer reappeared after we saw it die (it was in goneOnce) — a gone→back resume/restart,
|
|
100
|
-
* NOT a warm-continuous peer. The center cell's warm-exclusion discriminator. */
|
|
101
|
-
wasGoneReenroll: boolean
|
|
102
|
-
/** The center cell is evaluated exactly ONCE per life (first ready sample) — dedup guard. */
|
|
103
|
-
centerEvaluated: boolean
|
|
104
|
-
/** Previous tmux ready value, to annotate whether the not-ready→ready window was observed. */
|
|
105
|
-
prevReadyTmux: boolean | null
|
|
106
|
-
open: Record<Verdict, OpenAhead | null>
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
interface VerdictTally {
|
|
110
|
-
samples: number
|
|
111
|
-
agree: number
|
|
112
|
-
behind: number
|
|
113
|
-
aheadBenign: number
|
|
114
|
-
aheadHard: number
|
|
115
|
-
aheadLatencies: number[]
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export class GateTracker {
|
|
119
|
-
private readonly settleTicks: number
|
|
120
|
-
private readonly budget: number
|
|
121
|
-
private peers = new Map<string, PeerState>()
|
|
122
|
-
private goneOnce = new Set<string>()
|
|
123
|
-
private tally: Record<Verdict, VerdictTally>
|
|
124
|
-
private hard: HardViolation[] = []
|
|
125
|
-
// coverage
|
|
126
|
-
private fleetWidthMax = 0
|
|
127
|
-
private stateCells = new Map<string, number>() // `${runtime}/${state}` → count
|
|
128
|
-
private births = new Map<string, number>() // `${runtime}/${fresh|wake-resume}` → count
|
|
129
|
-
private restarts = new Map<string, number>() // runtime → count
|
|
130
|
-
private historySamples = new Map<string, number>() // runtime → count
|
|
131
|
-
// CENTER cell (robust, first-sight): a gone→back-with-history peer whose readiness the MODEL
|
|
132
|
-
// matches tmux on at its first ready sample — the codex splash-off-screen case the ready-gate flip
|
|
133
|
-
// exists for. A model MISS there lands in readyGate.behind (dirty). PRIMARY eligibility counter.
|
|
134
|
-
private readyResumeCaught = new Map<string, number>() // runtime → count
|
|
135
|
-
// Diagnostic ONLY (not a threshold path): the subset of center hits where the not-ready→ready
|
|
136
|
-
// window was actually observed (prev===false) — "we caught the window", a stronger evidence note.
|
|
137
|
-
private readyResumeCaughtViaTransition = new Map<string, number>() // runtime → count
|
|
138
|
-
private runtimesSeen = new Set<string>()
|
|
139
|
-
|
|
140
|
-
constructor(cfg: GateConfig = {}) {
|
|
141
|
-
this.settleTicks = cfg.settleTicks ?? 3
|
|
142
|
-
this.budget = cfg.convergeBudgetTicks ?? 3
|
|
143
|
-
const mk = (): VerdictTally => ({ samples: 0, agree: 0, behind: 0, aheadBenign: 0, aheadHard: 0, aheadLatencies: [] })
|
|
144
|
-
this.tally = { occupancy: mk(), readyGate: mk(), liveness: mk() }
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
/** Once per tick, before the per-peer samples — records the concurrent fleet width. */
|
|
148
|
-
tickStart(nPeers: number): void {
|
|
149
|
-
if (nPeers > this.fleetWidthMax) this.fleetWidthMax = nPeers
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
/** The observer dropped a peer (session gone) — a later reappearance is a restart. */
|
|
153
|
-
peerGone(addr: string): void {
|
|
154
|
-
if (this.peers.delete(addr)) this.goneOnce.add(addr)
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
private inc(map: Map<string, number>, key: string): void {
|
|
158
|
-
map.set(key, (map.get(key) ?? 0) + 1)
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/** Record one peer's sample at `tick`. Updates boot state, classifies every verdict's
|
|
162
|
-
* direction, manages ahead-transients, and accumulates the coverage matrix. */
|
|
163
|
-
sample(tick: number, s: PeerSample): void {
|
|
164
|
-
this.runtimesSeen.add(s.runtime)
|
|
165
|
-
let ps = this.peers.get(s.peer)
|
|
166
|
-
if (!ps) {
|
|
167
|
-
// First sight this lifetime → a birth. A peer we saw DIE before (in goneOnce) reappearing is
|
|
168
|
-
// a gone→back cycle (a resume/restart, NOT a warm-continuous peer) — the discriminator the
|
|
169
|
-
// robust center cell keys on.
|
|
170
|
-
const birthType = s.hasHistory ? 'wake-resume' : 'fresh'
|
|
171
|
-
this.inc(this.births, `${s.runtime}/${birthType}`)
|
|
172
|
-
const reenrolled = this.goneOnce.has(s.peer)
|
|
173
|
-
if (reenrolled) {
|
|
174
|
-
this.inc(this.restarts, s.runtime)
|
|
175
|
-
this.goneOnce.delete(s.peer)
|
|
176
|
-
}
|
|
177
|
-
ps = {
|
|
178
|
-
runtime: s.runtime,
|
|
179
|
-
firstTick: tick,
|
|
180
|
-
readyStreak: 0,
|
|
181
|
-
bootDone: false,
|
|
182
|
-
bornWithHistory: s.hasHistory,
|
|
183
|
-
wasGoneReenroll: reenrolled,
|
|
184
|
-
centerEvaluated: false,
|
|
185
|
-
prevReadyTmux: null,
|
|
186
|
-
open: { occupancy: null, readyGate: null, liveness: null },
|
|
187
|
-
}
|
|
188
|
-
this.peers.set(s.peer, ps)
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
// Boot-window: a peer is booted once tmux-ready has held true for settleTicks in a row.
|
|
192
|
-
if (s.readyGate.tmux === true) {
|
|
193
|
-
ps.readyStreak++
|
|
194
|
-
if (ps.readyStreak >= this.settleTicks) ps.bootDone = true
|
|
195
|
-
} else if (s.readyGate.tmux === false) {
|
|
196
|
-
ps.readyStreak = 0
|
|
197
|
-
}
|
|
198
|
-
const bootWindow = !ps.bootDone
|
|
199
|
-
|
|
200
|
-
// CENTER cell — cadence-ROBUST (first-sight, the SAME invariant births/restarts use), so a fast
|
|
201
|
-
// resume can't slip through a transition-polling gap. Counted ONCE per life (dedup: one resume =
|
|
202
|
-
// one center++) at the FIRST ready sample of a peer that (a) is a gone→back re-enroll (resume/
|
|
203
|
-
// restart, NOT warm-continuous — the warm-exclusion discriminator: a warm peer never entered
|
|
204
|
-
// goneOnce; also stricter than a transition, which a fresh peer's first boot would trip) and
|
|
205
|
-
// (b) re-enrolled WITH history (a resume-with-history). Parity there (rgPty===rgTmux===true) on a
|
|
206
|
-
// splash-off-screen frame validates the source swap; a model MISS is a pty-behind (the splash-blind
|
|
207
|
-
// bug) → readyGate.behind → dirty → ineligible. The observed not-ready→ready transition is kept as a
|
|
208
|
-
// DIAGNOSTIC annotation only ("we caught the window" — stronger evidence), NOT a second path to threshold.
|
|
209
|
-
if (s.readyGate.tmux !== null) {
|
|
210
|
-
if (ps.wasGoneReenroll && ps.bornWithHistory && !ps.centerEvaluated && s.readyGate.tmux === true) {
|
|
211
|
-
ps.centerEvaluated = true // evaluate exactly once, at the first ready sample
|
|
212
|
-
if (s.readyGate.pty === true) {
|
|
213
|
-
this.inc(this.readyResumeCaught, s.runtime)
|
|
214
|
-
if (ps.prevReadyTmux === false) this.inc(this.readyResumeCaughtViaTransition, s.runtime) // diagnostic: window caught
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
ps.prevReadyTmux = s.readyGate.tmux
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// Coverage.
|
|
221
|
-
const state = classifySampleState(s.occupancy.tmux === true, s.bytesFed)
|
|
222
|
-
this.inc(this.stateCells, `${s.runtime}/${state}`)
|
|
223
|
-
if (s.hasHistory) this.inc(this.historySamples, s.runtime)
|
|
224
|
-
|
|
225
|
-
// Per-verdict direction + transient bookkeeping.
|
|
226
|
-
this.scoreVerdict('occupancy', tick, ps, bootWindow, s.runtime, s.peer, s.occupancy)
|
|
227
|
-
this.scoreVerdict('readyGate', tick, ps, bootWindow, s.runtime, s.peer, s.readyGate)
|
|
228
|
-
this.scoreVerdict('liveness', tick, ps, bootWindow, s.runtime, s.peer, s.liveness)
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
private scoreVerdict(
|
|
232
|
-
verdict: Verdict,
|
|
233
|
-
tick: number,
|
|
234
|
-
ps: PeerState,
|
|
235
|
-
bootWindow: boolean,
|
|
236
|
-
runtime: string,
|
|
237
|
-
peer: string,
|
|
238
|
-
vs: VerdictSample,
|
|
239
|
-
): void {
|
|
240
|
-
if (vs.tmux === null) return // tmux side unavailable → not scored
|
|
241
|
-
const t = this.tally[verdict]
|
|
242
|
-
t.samples++
|
|
243
|
-
const dir = classifyDirection(vs.pty, vs.tmux)
|
|
244
|
-
|
|
245
|
-
// An open ahead-transient that exceeds the converge budget is a hard violation the moment
|
|
246
|
-
// it overruns (logged once), regardless of how it later resolves.
|
|
247
|
-
const open = ps.open[verdict]
|
|
248
|
-
if (open && !open.flaggedNonConverged && tick - open.startTick > this.budget) {
|
|
249
|
-
open.flaggedNonConverged = true
|
|
250
|
-
t.aheadHard++
|
|
251
|
-
this.hard.push({ peer, runtime, verdict, tick, kind: 'ahead-nonconverged', detail: `ahead open ${tick - open.startTick} ticks > budget ${this.budget}` })
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
if (dir === 'agree') {
|
|
255
|
-
t.agree++
|
|
256
|
-
if (open) {
|
|
257
|
-
// transient closed → measure self-heal latency; benign iff it opened in the boot
|
|
258
|
-
// window AND healed within budget (and was not already flagged non-converged).
|
|
259
|
-
const latency = tick - open.startTick
|
|
260
|
-
t.aheadLatencies.push(latency)
|
|
261
|
-
if (!open.flaggedNonConverged) {
|
|
262
|
-
if (open.bootWindow && latency <= this.budget) t.aheadBenign++
|
|
263
|
-
else {
|
|
264
|
-
t.aheadHard++
|
|
265
|
-
this.hard.push({ peer, runtime, verdict, tick, kind: open.bootWindow ? 'ahead-nonconverged' : 'ahead-steady', detail: `latency ${latency}, boot=${open.bootWindow}` })
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
ps.open[verdict] = null
|
|
269
|
-
}
|
|
270
|
-
return
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
if (dir === 'pty-behind') {
|
|
274
|
-
// The stale-frame class — always hard, zero tolerance. A behind also closes any open
|
|
275
|
-
// ahead-transient as an abnormal resolution (not a benign heal).
|
|
276
|
-
t.behind++
|
|
277
|
-
this.hard.push({ peer, runtime, verdict, tick, kind: 'behind' })
|
|
278
|
-
ps.open[verdict] = null
|
|
279
|
-
return
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
// pty-ahead: open a transient if none is open. Steady-state ahead is hard immediately
|
|
283
|
-
// (it cannot be a boot self-heal); boot-window ahead waits to see if it converges.
|
|
284
|
-
if (!open) {
|
|
285
|
-
if (!bootWindow) {
|
|
286
|
-
t.aheadHard++
|
|
287
|
-
this.hard.push({ peer, runtime, verdict, tick, kind: 'ahead-steady' })
|
|
288
|
-
// still open it so we can measure when it clears, but it is already counted hard
|
|
289
|
-
ps.open[verdict] = { startTick: tick, bootWindow: false, flaggedNonConverged: true }
|
|
290
|
-
} else {
|
|
291
|
-
ps.open[verdict] = { startTick: tick, bootWindow: true, flaggedNonConverged: false }
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
/** True iff no hard divergence has been recorded for `verdict` (the per-verdict gate bar). */
|
|
297
|
-
verdictGateClean(verdict: Verdict): boolean {
|
|
298
|
-
const t = this.tally[verdict]
|
|
299
|
-
return t.behind === 0 && t.aheadHard === 0
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
summary(): GateSummary {
|
|
303
|
-
const latStats = (xs: number[]): { count: number; max: number; mean: number } =>
|
|
304
|
-
xs.length ? { count: xs.length, max: Math.max(...xs), mean: +(xs.reduce((a, b) => a + b, 0) / xs.length).toFixed(2) } : { count: 0, max: 0, mean: 0 }
|
|
305
|
-
const verdicts = {} as Record<Verdict, VerdictGateStat>
|
|
306
|
-
for (const v of VERDICTS) {
|
|
307
|
-
const t = this.tally[v]
|
|
308
|
-
verdicts[v] = {
|
|
309
|
-
samples: t.samples,
|
|
310
|
-
agree: t.agree,
|
|
311
|
-
behind: t.behind,
|
|
312
|
-
aheadBenign: t.aheadBenign,
|
|
313
|
-
aheadHard: t.aheadHard,
|
|
314
|
-
selfHeal: latStats(t.aheadLatencies),
|
|
315
|
-
gateClean: this.verdictGateClean(v),
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
return {
|
|
319
|
-
verdicts,
|
|
320
|
-
hardViolations: this.hard.slice(),
|
|
321
|
-
coverage: {
|
|
322
|
-
fleetWidthMax: this.fleetWidthMax,
|
|
323
|
-
runtimes: [...this.runtimesSeen].sort(),
|
|
324
|
-
byRuntimeState: Object.fromEntries([...this.stateCells].sort()),
|
|
325
|
-
births: Object.fromEntries([...this.births].sort()),
|
|
326
|
-
restarts: Object.fromEntries([...this.restarts].sort()),
|
|
327
|
-
historySamples: Object.fromEntries([...this.historySamples].sort()),
|
|
328
|
-
readyResumeCaught: Object.fromEntries([...this.readyResumeCaught].sort()),
|
|
329
|
-
readyResumeCaughtViaTransition: Object.fromEntries([...this.readyResumeCaughtViaTransition].sort()),
|
|
330
|
-
},
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
/** Ready-gate FLIP eligibility — the verdict-scoped gate, operationalized so the burn-in
|
|
336
|
-
* SELF-REPORTS instead of being eyeballed. NECESSARY input, NOT an auto-trigger: the flip stays
|
|
337
|
-
* an explicit human decision (the representativeness of the filled cells — e.g. the splash truly
|
|
338
|
-
* scrolled off-screen — is verified separately on top of this). Eligible iff:
|
|
339
|
-
* • ready-gate is HARD-clean (behind=0 ∧ aheadHard=0), AND
|
|
340
|
-
* • the CENTER cell is filled: codex woke WITH history and the model caught its readiness, AND
|
|
341
|
-
* • a codex AND a claude birth were observed, AND
|
|
342
|
-
* • a restart was observed.
|
|
343
|
-
* composer-busy is ORTHOGONAL to readiness (a typing human already passed readiness) → excluded;
|
|
344
|
-
* it gates the LATER occupancy flip. On !eligible `reasons` say WHICH cell is empty/dirty —
|
|
345
|
-
* actionable, never a black-box "not yet". PURE → its own logic is unit-testable. */
|
|
346
|
-
export function evaluateReadyGateFlipEligibility(s: GateSummary): FlipEligibility {
|
|
347
|
-
const reasons: string[] = []
|
|
348
|
-
const rg = s.verdicts.readyGate
|
|
349
|
-
if (rg.behind > 0 || rg.aheadHard > 0) reasons.push(`readyGate dirty: behind=${rg.behind} aheadHard=${rg.aheadHard}`)
|
|
350
|
-
const center = s.coverage.readyResumeCaught['codex'] ?? 0
|
|
351
|
-
if (center < 1) reasons.push(`center codex×wake-resume×history: ${center} readiness caught (need ≥1, representatively splash-off-screen)`)
|
|
352
|
-
const hasBirth = (rt: string): boolean => Object.keys(s.coverage.births).some(k => k.startsWith(`${rt}/`))
|
|
353
|
-
if (!hasBirth('codex')) reasons.push('no codex birth observed')
|
|
354
|
-
if (!hasBirth('claude')) reasons.push('no claude birth observed')
|
|
355
|
-
const restarts = Object.values(s.coverage.restarts).reduce((a, b) => a + b, 0)
|
|
356
|
-
if (restarts < 1) reasons.push('no restart observed')
|
|
357
|
-
return { verdict: 'readyGate', eligible: reasons.length === 0, reasons }
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
export interface VerdictGateStat {
|
|
361
|
-
samples: number
|
|
362
|
-
agree: number
|
|
363
|
-
behind: number
|
|
364
|
-
aheadBenign: number
|
|
365
|
-
aheadHard: number
|
|
366
|
-
selfHeal: { count: number; max: number; mean: number }
|
|
367
|
-
gateClean: boolean
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
export interface GateCoverage {
|
|
371
|
-
fleetWidthMax: number
|
|
372
|
-
runtimes: string[]
|
|
373
|
-
byRuntimeState: Record<string, number>
|
|
374
|
-
births: Record<string, number>
|
|
375
|
-
restarts: Record<string, number>
|
|
376
|
-
historySamples: Record<string, number>
|
|
377
|
-
/** runtime → center-cell count: gone→back-with-history peers whose first-ready model verdict
|
|
378
|
-
* matched tmux (robust, first-sight). The PRIMARY ready-gate-flip eligibility signal. */
|
|
379
|
-
readyResumeCaught: Record<string, number>
|
|
380
|
-
/** runtime → DIAGNOSTIC subset of readyResumeCaught where the not-ready→ready window was observed
|
|
381
|
-
* (transition caught). Stronger evidence; NOT a separate threshold path. */
|
|
382
|
-
readyResumeCaughtViaTransition: Record<string, number>
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
export interface GateSummary {
|
|
386
|
-
verdicts: Record<Verdict, VerdictGateStat>
|
|
387
|
-
hardViolations: HardViolation[]
|
|
388
|
-
coverage: GateCoverage
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
export interface FlipEligibility {
|
|
392
|
-
verdict: 'readyGate'
|
|
393
|
-
eligible: boolean
|
|
394
|
-
/** On !eligible: one actionable line per unmet requirement (which cell is empty/dirty). */
|
|
395
|
-
reasons: string[]
|
|
396
|
-
}
|