@agfpd/iapeer 0.2.21 → 0.2.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agfpd/iapeer",
-  "version": "0.2.21",
+  "version": "0.2.22",
   "description": "Foundation core for the iapeer multi-agent ecosystem: identity, registry, storage, codec.",
   "type": "module",
   "bin": {

package/src/launch/canary.test.ts

@@ -0,0 +1,173 @@
+// Server-death canary — pure script-builder shape + live-tmux behavior:
+// dirty server death (SIGKILL) → ev=server-exit line + forensics snapshot;
+// clean teardown (signal / killSession) → silent, no record. Live tests follow
+// the suite's test.if(tmuxAvailable) sandbox-socket pattern (lifecycle.test.ts).
+
+import { afterAll, describe, expect, test } from 'bun:test'
+import { spawnSync } from 'child_process'
+import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'fs'
+import { tmpdir } from 'os'
+import { join } from 'path'
+import {
+  canaryChannel,
+  canaryScript,
+  ensureServerCanary,
+  exitLogPath,
+  serverDeathsDir,
+  signalCanaryClean,
+} from './canary.ts'
+import { killSession } from '../lifecycle/index.ts'
+
+const tmuxAvailable = spawnSync('tmux', ['-V'], { stdio: 'ignore' }).status === 0
+
+const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
+
+/** Poll until `cond` is true or `ms` elapsed. */
+async function waitFor(cond: () => boolean, ms: number): Promise<boolean> {
+  const deadline = Date.now() + ms
+  while (Date.now() < deadline) {
+    if (cond()) return true
+    await sleep(100)
+  }
+  return cond()
+}
+
+function canaryRunning(identity: string): boolean {
+  return (
+    spawnSync('pgrep', ['-f', `wait-for.*${canaryChannel(identity)}([^a-z0-9-]|$)`], { stdio: 'ignore' }).status === 0
+  )
+}
+
+function serverPid(sock: string): number {
+  const r = spawnSync('tmux', ['-S', sock, 'display-message', '-p', '#{pid}'], { encoding: 'utf8' })
+  return Number((r.stdout ?? '').trim())
+}
+
+describe('canary script (pure)', () => {
+  test('channel is per-identity', () => {
+    expect(canaryChannel('claude-bob')).toBe('iap-canary-claude-bob')
+  })
+
+  test('exitLogPath → exits.log inside the dir', () => {
+    expect(exitLogPath('/r/logs/iapeer')).toBe('/r/logs/iapeer/exits.log')
+  })
+
+  test('script carries the protocol: wait-for channel, silent-exit guards, record line, forensics', () => {
+    const s = canaryScript({
+      identity: 'claude-bob',
+      sock: '/tmp/x.sock',
+      tmuxBin: '/opt/homebrew/bin/tmux',
+      exitLogFile: '/r/logs/iapeer/exits.log',
+      forensicsDir: '/r/logs/iapeer/server-deaths',
+    })
+    expect(s).toContain(`wait-for 'iap-canary-claude-bob'`) // the blocking client
+    expect(s).toContain(`trap 'exit 0' HUP INT TERM`) // signaled canary = silent
+    expect(s).toContain(`[ "$rc" -eq 0 ] || [ "$rc" -ge 128 ]`) // clean/signaled guards
+    expect(s).toContain('ev=server-exit identity=claude-bob') // the exits.log record
+    expect(s).toContain(`>> '/r/logs/iapeer/exits.log'`)
+    expect(s).toContain('/r/logs/iapeer/server-deaths/claude-bob') // forensics file
+    expect(s).toContain('vm_stat') // memory evidence (OOM hypothesis)
+    expect(s).toContain(`'/opt/homebrew/bin/tmux' -S '/tmp/x.sock'`) // abs tmux, quoted
+  })
+
+  test('ensureServerCanary without exitLogDir → skipped (observability off)', () => {
+    expect(ensureServerCanary({ identity: 'claude-none', sock: '/tmp/none.sock' })).toBe('skipped')
+  })
+})
+
+describe.if(tmuxAvailable)('canary live (sandbox tmux servers)', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'iap-canary-'))
+  const socks: string[] = []
+
+  function bringUp(identity: string): { sock: string; logDir: string } {
+    const sock = join(dir, `${identity}.sock`)
+    socks.push(sock)
+    const r = spawnSync('tmux', ['-S', sock, 'new-session', '-d', '-s', identity, 'sleep', '120'], {
+      stdio: 'ignore',
+    })
+    expect(r.status).toBe(0)
+    return { sock, logDir: join(dir, `logs-${identity}`) }
+  }
+
+  afterAll(() => {
+    for (const sock of socks) {
+      // teardown is DELIBERATE → signal each canary before killing its server
+      for (const id of ['claude-canadirty', 'claude-canaclean', 'claude-canakill']) {
+        signalCanaryClean(sock, id)
+      }
+      spawnSync('tmux', ['-S', sock, 'kill-server'], { stdio: 'ignore' })
+    }
+    rmSync(dir, { recursive: true, force: true })
+  })
+
+  test(
+    'dirty server death (SIGKILL) → ev=server-exit + forensics snapshot',
+    async () => {
+      const identity = 'claude-canadirty'
+      const { sock, logDir } = bringUp(identity)
+      const pid = serverPid(sock)
+      expect(pid).toBeGreaterThan(0)
+
+      expect(ensureServerCanary({ identity, sock, exitLogDir: logDir })).toBe('spawned')
+      // idempotency: a second ensure must NOT double-spawn
+      expect(await waitFor(() => canaryRunning(identity), 3000)).toBe(true)
+      expect(ensureServerCanary({ identity, sock, exitLogDir: logDir })).toBe('already')
+      await sleep(500) // let the wait-for client register on the channel
+
+      process.kill(pid, 'SIGKILL') // the silent server-killer class
+      const log = exitLogPath(logDir)
+      expect(await waitFor(() => existsSync(log) && readFileSync(log, 'utf8').includes('ev=server-exit'), 8000)).toBe(
+        true,
+      )
+      const line = readFileSync(log, 'utf8')
+      expect(line).toContain(`ev=server-exit identity=${identity}`)
+      expect(line).toContain(`server_pid=${pid}`)
+      const snaps = readdirSync(serverDeathsDir(logDir))
+      expect(snaps.length).toBe(1)
+      const snap = readFileSync(join(serverDeathsDir(logDir), snaps[0]!), 'utf8')
+      expect(snap).toContain(`server-death identity=${identity}`)
+      // canary is one-shot: after firing it must be gone
+      expect(await waitFor(() => !canaryRunning(identity), 3000)).toBe(true)
+    },
+    20000,
+  )
+
+  test(
+    'clean teardown (signal, then kill-server) → silent, no record',
+    async () => {
+      const identity = 'claude-canaclean'
+      const { sock, logDir } = bringUp(identity)
+      expect(ensureServerCanary({ identity, sock, exitLogDir: logDir })).toBe('spawned')
+      expect(await waitFor(() => canaryRunning(identity), 3000)).toBe(true)
+      await sleep(500) // let the wait-for client register before signaling
+
+      signalCanaryClean(sock, identity)
+      expect(await waitFor(() => !canaryRunning(identity), 3000)).toBe(true) // exited silently
+      spawnSync('tmux', ['-S', sock, 'kill-server'], { stdio: 'ignore' })
+      await sleep(300)
+      expect(existsSync(exitLogPath(logDir)) && readFileSync(exitLogPath(logDir), 'utf8').includes('ev=server-exit')).toBe(
+        false,
+      )
+    },
+    20000,
+  )
+
+  test(
+    'killSession (lifecycle clean reap) signals the canary before kill-server → no record',
+    async () => {
+      const identity = 'claude-canakill'
+      const { sock, logDir } = bringUp(identity)
+      expect(ensureServerCanary({ identity, sock, exitLogDir: logDir })).toBe('spawned')
+      expect(await waitFor(() => canaryRunning(identity), 3000)).toBe(true)
+      await sleep(500)
+
+      killSession(sock, identity) // last session → kills the SERVER, signal-first
+      expect(await waitFor(() => !canaryRunning(identity), 3000)).toBe(true)
+      await sleep(300)
+      expect(existsSync(exitLogPath(logDir)) && readFileSync(exitLogPath(logDir), 'utf8').includes('ev=server-exit')).toBe(
+        false,
+      )
+    },
+    20000,
+  )
+})

package/src/launch/canary.ts

@@ -0,0 +1,152 @@
+// Server-death canary — the SERVER-level catcher for the exit-cause gap the
+// pane-died hook structurally cannot close (контракт: pane-died needs a living
+// tmux event loop; SIGKILL/OOM to the tmux SERVER runs no hook). Raised from the
+// deferred backlog after the SECOND live case of the class (iapeer-memory 10.06
+// 01:43, boris 10.06 11:41 — both: a healthy session with active Bash
+// subprocessing, the whole server gone silently, zero system traces).
+//
+// Mechanism: one tiny detached `sh` per LIVE tmux server, holding a blocking
+// client `tmux -S <sock> wait-for iap-canary-<identity>` — a process OUTSIDE the
+// dying server, connected via its socket, so the server's death (any cause,
+// including SIGKILL) is observed the moment the connection drops. Protocol:
+//   • clean teardown (idle-reap / stop / pre-clean / killServerIfEmpty) SIGNALS
+//     the channel first (`wait-for -S`) → the canary exits 0, silently;
+//   • the canary itself signaled (TERM/HUP/INT, e.g. launch pre-clean pkill) →
+//     rc ≥ 128 / trap → exit silently (someone manages it deliberately);
+//   • anything else (wait-for returns an error = the server vanished under us)
+//     → ONE logfmt line `ev=server-exit` into exits.log (the per-peer death-cause
+//     home, next to pane-died's `ev=session-exit`) + a forensics snapshot file
+//     (vm_stat / swap / top-RSS ps / fresh DiagnosticReports) captured WITHIN
+//     SECONDS of the death — the evidence the 60 s supervise tick can never
+//     recover ("системных следов ноль" was the recurring investigation outcome).
+//
+// The canary is pure observability: it never wakes, reaps, restarts or otherwise
+// manages anything (H4-compatible by construction), it fires at most once, and
+// every failure to spawn/record is swallowed (never load-bearing). The daemon's
+// reaped-gone death-class (classifyGoneSession) remains the detection backstop
+// when no canary was running.
+
+import { spawn, spawnSync } from 'child_process'
+import { mkdirSync } from 'fs'
+import { join } from 'path'
+import { resolveExecutable } from './launchd.ts'
+
+/** The exit-cause log file inside `exitLogDir` (sibling to lifecycle.log).
+ *  Lives here (not index.ts) so canary ⇆ launch never import-cycle; index.ts
+ *  re-exports it, keeping the public import surface unchanged. */
+export function exitLogPath(exitLogDir: string): string {
+  return `${exitLogDir}/exits.log`
+}
+
+/** The per-identity tmux wait-for channel the clean-teardown paths signal. */
+export function canaryChannel(identity: string): string {
+  return `iap-canary-${identity}`
+}
+
+/** Forensics snapshots live next to exits.log: `<exitLogDir>/server-deaths/`. */
+export function serverDeathsDir(exitLogDir: string): string {
+  return join(exitLogDir, 'server-deaths')
+}
+
+export interface CanaryScriptOptions {
+  identity: string
+  sock: string
+  /** Absolute tmux path — the detached canary may outlive any rich PATH. */
+  tmuxBin: string
+  exitLogFile: string
+  forensicsDir: string
+}
+
+/**
+ * Build the canary shell script (PURE — unit-testable). Quoting: identity is a
+ * runtime-personality (`[a-z0-9-]`), sock/log paths are ~/.iapeer-style (no
+ * single quotes) — same assumption as exitCauseHook. All forensic tools are
+ * /usr/bin- or /bin-resident, so the script survives launchd's minimal PATH;
+ * only tmux needs the baked absolute path.
+ */
+export function canaryScript(o: CanaryScriptOptions): string {
+  const ch = canaryChannel(o.identity)
+  return [
+    // Server PID captured while alive — the postmortem grep key for system logs.
+    `SPID="$('${o.tmuxBin}' -S '${o.sock}' display-message -p '#{pid}' 2>/dev/null)"`,
+    // A signal to the CANARY is deliberate management (pre-clean pkill) → silent.
+    `trap 'exit 0' HUP INT TERM`,
+    `'${o.tmuxBin}' -S '${o.sock}' wait-for '${ch}'`,
+    `rc=$?`,
+    // rc=0 → clean-teardown signal; rc≥128 → the tmux client itself was signaled.
+    `if [ "$rc" -eq 0 ] || [ "$rc" -ge 128 ]; then exit 0; fi`,
+    // The server vanished under us — record, within seconds of the death.
+    `ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"`,
+    `f='${o.forensicsDir}/${o.identity}'-"$(date +%s)".txt`,
+    `{`,
+    `  echo "server-death identity=${o.identity} ts=$ts server_pid=$SPID wait_rc=$rc"`,
+    `  echo '--- vm_stat'; vm_stat`,
+    `  echo '--- swapusage'; sysctl -n vm.swapusage`,
+    `  echo '--- ps-top-rss'; ps axo pid,ppid,rss,etime,command | sort -rn -k3 | head -25`,
+    `  echo '--- diagnosticreports-user'; ls -t "$HOME/Library/Logs/DiagnosticReports" 2>/dev/null | head`,
+    `  echo '--- diagnosticreports-system'; ls -t /Library/Logs/DiagnosticReports 2>/dev/null | head`,
+    `} > "$f" 2>&1`,
+    `printf 'ts=%s ev=server-exit identity=${o.identity} server_pid=%s wait_rc=%s forensics=%s\\n' "$ts" "$SPID" "$rc" "$f" >> '${o.exitLogFile}'`,
+  ].join('\n')
+}
+
+export type CanaryEnsureState = 'spawned' | 'already' | 'skipped' | 'failed'
+
+export interface EnsureCanaryOptions {
+  identity: string
+  sock: string
+  /** Same gate as installExitHook: falsy → observability off, no canary. */
+  exitLogDir?: string
+  env?: NodeJS.ProcessEnv
+}
+
+/**
+ * Ensure ONE canary is watching this identity's tmux server. Idempotent via a
+ * pgrep on the wait-for channel (anchored so an identity can never match another
+ * identity's prefix). Called from launch (newborn server) and from the supervise
+ * tick's alive-branch (retrofits canaries onto a fleet launched by older code —
+ * coverage within one tick of a deploy, no session restarts). Best-effort:
+ * every failure returns a state, never throws.
+ */
+export function ensureServerCanary(o: EnsureCanaryOptions): CanaryEnsureState {
+  if (!o.exitLogDir) return 'skipped'
+  const env = o.env ?? process.env
+  try {
+    const probe = spawnSync('pgrep', ['-f', `wait-for.*${canaryChannel(o.identity)}([^a-z0-9-]|$)`], {
+      stdio: 'ignore',
+      env: env as Record<string, string>,
+    })
+    if (probe.status === 0) return 'already'
+    const tmuxBin = resolveExecutable('tmux', env)
+    if (!tmuxBin) return 'failed'
+    const forensicsDir = serverDeathsDir(o.exitLogDir)
+    mkdirSync(forensicsDir, { recursive: true, mode: 0o700 })
+    mkdirSync(o.exitLogDir, { recursive: true, mode: 0o700 })
+    const script = canaryScript({
+      identity: o.identity,
+      sock: o.sock,
+      tmuxBin,
+      exitLogFile: exitLogPath(o.exitLogDir),
+      forensicsDir,
+    })
+    const child = spawn('/bin/sh', ['-c', script], { detached: true, stdio: 'ignore', env })
+    child.unref()
+    return 'spawned'
+  } catch {
+    return 'failed' // observability is best-effort — never block a launch/tick
+  }
+}
+
+/**
+ * Signal the canary that the upcoming server teardown is DELIBERATE (idle-reap /
+ * stop / launch pre-clean / empty-server kill) so it exits silently instead of
+ * recording a false server-death. Best-effort: no server / no canary on the
+ * channel → harmless no-op.
+ */
+export function signalCanaryClean(sock: string, identity: string): void {
+  try {
+    spawnSync('tmux', ['-S', sock, 'wait-for', '-S', canaryChannel(identity)], { stdio: 'ignore' })
+  } catch {
+    /* best-effort */
+  }
+}

package/src/launch/index.ts

@@ -19,6 +19,7 @@ import { dirname } from 'path'
 import { spawnSync } from 'child_process'
 import { CODEX_BEARER_ENV_VAR, CODEX_DUMMY_BEARER, type Runtime } from '../core/constants.ts'
 import { readLaunchEnv } from '../storage/index.ts'
+import { ensureServerCanary, exitLogPath, signalCanaryClean } from './canary.ts'
 import { claudeAdapter } from './adapters/claude.ts'
 import { codexAdapter } from './adapters/codex.ts'
 import { telegramAdapter } from './adapters/telegram.ts'
@@ -127,14 +128,15 @@ function ready(identity: string): LaunchResult {
 //   • SIGTERM/SIGKILL/crash to the PROCESS → `dead_status= dead_signal=<name>`
 //   • daemon-initiated `kill-session` (idle-reap / self-TTL / stop) does NOT fire
 //     pane-died → NO line here (those are already in lifecycle.log — no double-log).
-// IRREDUCIBLE GAP: SIGKILL to the tmux SERVER process itself runs no hook (the
-//   event loop is gone); only the daemon's post-factum reaped-gone catches that.
+// SERVER-LEVEL GAP: SIGKILL to the tmux SERVER process itself runs no hook (the
+//   event loop is gone). Closed one level up by the server-death canary
+//   (canary.ts — `ev=server-exit` + forensics snapshot); the daemon's post-factum
+//   reaped-gone death-class remains the detection backstop.
 // ─────────────────────────────────────────────────────────────────────────────
 
-/** The exit-cause log file inside `exitLogDir` (sibling to lifecycle.log). */
-export function exitLogPath(exitLogDir: string): string {
-  return `${exitLogDir}/exits.log`
-}
+// The exit-cause log path lives in canary.ts (no import cycle); re-exported here
+// so the public surface (`launch/index.ts → exitLogPath`) is unchanged.
+export { exitLogPath } from './canary.ts'
 
 /**
  * Build the tmux `pane-died` hook command string (the value of `set-hook -t <id>
@@ -210,6 +212,11 @@ export const launch: LaunchFn = async (
   mkdirSync(dirname(sock), { recursive: true })
 
   // (1) Pre-clean any stale tmux server on this socket, then launch detached.
+  //     Signal the server-death canary FIRST: this teardown is deliberate, the
+  //     canary must exit silently, not record a false server-death (the pkill
+  //     below also matches the canary's cmdline — by design, it sweeps a stale
+  //     canary along with the stale server; a signaled canary is already gone).
+  signalCanaryClean(sock, identity)
   spawnSync('pkill', ['-f', `tmux -S ${sock} `], { stdio: 'ignore' })
   tmux(sock, 'kill-server')
 
@@ -256,6 +263,13 @@ export const launch: LaunchFn = async (
   //       boot leaves a cause. No-op without cfg.exitLogDir (remain-on-exit off).
   installExitHook(sock, identity, cfg.exitLogDir)
 
+  // (2.6) Server-death canary (canary.ts): the SERVER-level catcher pane-died
+  //       structurally cannot be — a detached wait-for client OUTSIDE the server
+  //       that records `ev=server-exit` + a forensics snapshot when the whole
+  //       server dies dirty (SIGKILL/OOM class). Same gate as the exit hook;
+  //       best-effort by construction (every failure → a state, never a throw).
+  ensureServerCanary({ identity, sock, exitLogDir: cfg.exitLogDir, env })
+
   // (3) pipe-pane the session output to the per-identity log.
   mkdirSync(cfg.logDir, { recursive: true, mode: 0o700 })
   const paneLog = `${cfg.logDir}/${identity}.log`

package/src/lifecycle/index.ts

@@ -46,6 +46,7 @@ import {
   type LaunchSpec,
 } from '../launch/index.ts'
 import { composeSystemPrompt, gatherPromptInput } from '../launch/composeSystemPrompt.ts'
+import { ensureServerCanary, signalCanaryClean } from '../launch/canary.ts'
 import { appendLifecycleEvent, superviseLogVerbose } from './eventlog.ts'
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -594,7 +595,8 @@ function sessionAlive(sock: string, identity: string): boolean {
  *    exits.log should carry the cause.
  *  - `server-dead` — the server itself is gone: the socket file is missing, or it
  *    exists but nothing serves it (stale socket — the SIGKILL/OOM class). pane-died
- *    could never fire, so the lifecycle.log line is the only durable trace. */
+ *    could never fire; the server-death canary (launch/canary.ts) is the recorder
+ *    for this class (`ev=server-exit` + forensics), this line is the backstop. */
 export function classifyGoneSession(sock: string): { death: 'server-dead' | 'session-gone'; reason: string } {
   if (!existsSync(sock)) {
     return { death: 'server-dead', reason: 'tmux server gone (socket file missing)' }
@@ -603,7 +605,7 @@ export function classifyGoneSession(sock: string): { death: 'server-dead' | 'ses
   // proves the server is alive and merely lost this session.
   return tmux(sock, 'list-sessions').ok
     ? { death: 'session-gone', reason: 'session gone, tmux server alive (exit cause should be in exits.log)' }
-    : { death: 'server-dead', reason: 'tmux server dead — stale socket (SIGKILL/OOM class; exits.log has no entry)' }
+    : { death: 'server-dead', reason: 'tmux server dead — stale socket (SIGKILL/OOM class; check exits.log ev=server-exit canary record)' }
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -958,6 +960,9 @@ export function killSession(sock: string, identity: string): void {
   tmux(sock, 'kill-session', '-t', identity)
   const sessions = tmux(sock, 'list-sessions', '-F', '#{session_name}').out
   if (!sessions.trim()) {
+    // Deliberate server teardown → signal the server-death canary FIRST so it
+    // exits silently instead of recording a false `ev=server-exit` (canary.ts).
+    signalCanaryClean(sock, identity)
     tmux(sock, 'kill-server')
     try {
       rmSync(sock, { force: true })
@@ -1108,6 +1113,11 @@ export function superviseTick(cfg: LifecycleConfig, deps: SuperviseDeps = {}): S
       out.push({ identity: s.identity, action: 'reaped-idle', reason: `idle ${ageSecs}s` })
       trace({ identity: s.identity, action: 'reaped-idle', age: `${ageSecs}s`, outcome: 'resume-eligible' })
     } else {
+      // Server-death canary retrofit (canary.ts): ensure a canary is watching
+      // every ALIVE daemon-owned server — covers a fleet launched by older code
+      // within one tick of a deploy, no session restarts. Idempotent (pgrep on
+      // the per-identity wait-for channel), best-effort, pure observability.
+      ensureServerCanary({ identity: s.identity, sock, exitLogDir: cfg.eventLogDir, env })
       out.push({ identity: s.identity, action: 'alive' })
       if (verbose) trace({ identity: s.identity, action: 'alive', age: `${ageSecs}s` })
     }