@agfpd/iapeer 0.4.97 → 0.4.98

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -48,7 +48,7 @@ The full fleet state. The peer rows are produced by the **same in-process functi
48
48
  "default_runtime": "claude",
49
49
  "cwd": "/Users/me/.iapeer/peers/boris",
50
50
  "runtimes": [
51
- { "runtime": "claude", "status": "live", "attached": true },
51
+ { "runtime": "claude", "status": "live", "attached": true, "activity": "working" },
52
52
  { "runtime": "codex", "status": "asleep" }
53
53
  ],
54
54
  "last_active_runtime": "claude",
@@ -78,6 +78,7 @@ Field notes:
78
78
 
79
79
  - `runtimes[].status` — `live` (session up) · `asleep` (wakeable on demand) · `stopped` (operator stop flag; the daemon won't wake it). Same glyph semantics as the CLI: `●` / `○` / `✕`.
80
80
  - `runtimes[].attached` — present (`true`) only when a **human operator** is attached to that live hosted session.
81
+ - `runtimes[].activity` — **turn activity**: is this runtime *in a turn* right now. See «Activity» below. Additive; absent on a pre-activity daemon, on every non-`live` runtime, and whenever the daemon has no evidence — clients MUST treat absence as **no claim**, never as `idle`.
81
82
  - `attached` — any runtime attached (peer-level convenience).
82
83
  - `launchd_managed` — launchd owns this peer's lifecycle (H4): the daemon never wakes/reaps it, and wake/stop/start commands are guarded accordingly.
83
84
  - `wake_policy` — `warm` (default: wake-on-message, idle-reap, resume) or `ephemeral` (stateless worker: serial task queue, die-after-reply).
@@ -90,6 +91,65 @@ Field notes:
90
91
  - `approvals` (top-level, docs/17-approval) — the pending human-approval queue, the SAME items `GET /fleet/v1/approvals` returns, carried in the snapshot so a client that already re-fetches `/snapshot` on every event renders the queue with no extra call. **ADDITIVE + OMITTED when the queue is empty** — a client MUST treat its absence as an empty queue (the same rule as `approval_mode`). Each item: `id` (broker id, the target of `POST /fleet/v1/approvals/<id>/(approve|deny)`), `personality` + `runtime` (whose action), `kind` (`tool`|`plan`|`question`|`circuit-breaker`), `tool` (the tool / breaker name), `summary` (a one-line badge string), `content` (the FULL verbatim action — command / diff / plan; criterion #7), `createdMs` / `expiresMs`. The broker's item is a superset — unknown extra fields (`title`, `approvers`) MUST be ignored (obligation 2).
91
92
  - `notices` (top-level, docs/19-notices) — live OWNER-facing notices: things the daemon noticed that the peer itself could not report, the v1 case being a structural API error that left it MUTE (`kind: peer-mute`). The SAME items `GET /fleet/v1/notices` returns. **ADDITIVE + OMITTED when the board is empty** — absence ⇒ empty board (the same rule as `approvals`). **READ-ONLY**: a notice is one-way, there is nothing to approve or deny. Each item: `id`, `personality` + `runtime` (who went mute), `kind` (`peer-mute`; treat unknown kinds as renderable), `errorType` (the RUNTIME's own value — `rate_limit`, `overloaded`, `rate_limit_reached`, … — free-form, do not switch exhaustively), `model?`, `resetsAtMs?`, `summary`, `content` (verbatim for claude), `sessionId?`, `createdMs` / `lastMs` / `expiresMs`, `count` (detections folded in — render as `×N`). **`model` and `resetsAtMs` are ABSENT when the runtime did not state them** — absence means *the runtime did not say*, NOT *there is no limit*; a client MUST render unknown and MUST NOT substitute the 5h/7d reset, which belongs to a different limit (docs/19 §3).
92
93
 
94
+ ## Activity — is this runtime in a turn?
95
+
96
+ `runtimes[].activity` answers the operator's actual question, which `status` never did: a peer
97
+ idling with an unfinished job and a peer grinding through a turn are BOTH `live`.
98
+
99
+ **Activity is not liveness, and the two never merge.** `status` keeps its exact prior meaning.
100
+ `activity` is an additive sibling on the same per-runtime object.
101
+
102
+ | value | meaning |
103
+ |---|---|
104
+ | `working` | a turn is in flight on this runtime |
105
+ | `idle` | live, but not in a turn |
106
+ | `unknown` | live; the turn was seen to start, never seen to finish, and the evidence has gone stale |
107
+ | *absent* | **no claim** — see below |
108
+
109
+ `activity` is **ABSENT** (never `idle`) when any of these hold, and a client MUST NOT infer
110
+ idleness from absence:
111
+
112
+ - the daemon predates this field (obligation 1);
113
+ - `status !== 'live'` — an asleep/stopped session has no turn to be in;
114
+ - the daemon has no evidence for that live runtime yet.
115
+
116
+ ### Where the value comes from
117
+
118
+ The **runtime's own turn marker**, read from its session file. Never CPU, never a pane-log
119
+ mtime, never a reconstruction:
120
+
121
+ - **codex** — the rollout's `event_msg` payloads `task_started` → `working`, `task_complete` →
122
+ `idle`; last one wins. Codex emits `task_complete` for a turn that ends on an error too, so
123
+ the terminal-error path needs no separate marker.
124
+ - **claude** — the transcript's assistant `stop_reason`. **Only `tool_use` means the turn
125
+ continues**; every other terminal reason (`end_turn`, `stop_sequence`, `max_tokens`, a future
126
+ value, `null`) means it ended. The polarity is deliberately asymmetric: an unrecognised reason
127
+ must never invent work. A genuine `user` line also reads `working` (a prompt to answer or a
128
+ tool_result to consume), but compaction summaries (`isCompactSummary`), sub-agent sidechains
129
+ (`isSidechain`) and injected meta lines (`isMeta`) are excluded — each is a measured
130
+ false-`working` vector, the worst being a `/compact` run while idle leaving a phantom
131
+ `working` forever.
132
+
133
+ ### Bounds a client can rely on
134
+
135
+ - **Lag** — the daemon re-reads changed session files every **~3 s**, so an `idle`↔`working`
136
+ flip appears within roughly one interval plus fs latency. A turn shorter than the interval may
137
+ never be observed at all; activity is a *state* to render, not an event ledger to count.
138
+ - **Stale bound** — a `working` claim whose evidence is older than **10 min** decays to
139
+ `unknown`. `idle` NEVER decays: a session that ended its turn stays idle until something pokes
140
+ it, and age proves nothing against that.
141
+ - **Restart** — the board is in-memory and re-seeds from the session files (a 48 h look-back) on
142
+ daemon start, so a restart cannot preserve a phantom `working`; a stale one decays by the rule
143
+ above, and a dead session drops out via `status` first.
144
+
145
+ ### Events
146
+
147
+ `turns.log` → the SSE stream, `src: "turns"`, `ev`: **`turn-started`** / **`turn-ended`**, with
148
+ `personality`, `runtime`, `from` (the previous state, absent on first observation), `at` +
149
+ `at_iso` (the MARKER's own timestamp, not the sweep's). Emitted only on a real state **change** —
150
+ two lines per turn per runtime, not one per sweep. Treat the snapshot as the source of truth and
151
+ these as a push hint (obligation: the stream is at-least-once).
152
+
93
153
  ## GET /fleet/v1/peers/&lt;personality&gt;
94
154
 
95
155
  One peer's card: the same object as the snapshot row, plus its recent history.
@@ -122,7 +182,7 @@ GET /fleet/v1/events?replay=50
122
182
  - A comment `: connected` marks the transition from replay to live; `: hb` comments every ~15 s keep the connection alive.
123
183
  - Latency: live events surface within ~0.5 s (log-tail poll).
124
184
 
125
- Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently).
185
+ Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently), `turn-started` / `turn-ended` (from `turns.log`, `src: turns` — a runtime entered/left a turn; see «Activity». One line per real state CHANGE, never one per sweep).
126
186
 
127
187
  What the headline states look like on the stream: a peer **waking** is `ev=wake`; a peer **parked to sleep** is `ev=supervise action=reaped-idle`; an operator **stop/start** is `ev=stopped` / `ev=started`; a **death** is `ev=session-exit` (immediately, from the supervisor) followed by the daemon's classification on its next pass.
128
188
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agfpd/iapeer",
3
- "version": "0.4.97",
3
+ "version": "0.4.98",
4
4
  "description": "Run Claude Code and Codex CLI as local background agents that message each other, wake on demand, and share one memory.",
5
5
  "license": "Apache-2.0",
6
6
  "author": "agfpd",
@@ -64,6 +64,8 @@ import { approvalsLogPath } from './approvalslog.ts'
64
64
  import { ApprovalBroker } from './approvals.ts'
65
65
  import { noticesLogPath } from './noticeslog.ts'
66
66
  import { NoticeBoard } from './notices.ts'
67
+ import { turnsLogPath } from './turnslog.ts'
68
+ import { ActivityBoard, type TurnActivity } from './turnwatch.ts'
67
69
  import { callTool, type CallToolDeps } from './index.ts'
68
70
 
69
71
  /** The API generation baked into every snapshot (additive evolution within v1 —
@@ -79,6 +81,18 @@ export interface FleetRuntimeStatus {
79
81
  status: 'live' | 'asleep' | 'stopped'
80
82
  /** Present (true) iff a HUMAN operator is attached to this live hosted session. */
81
83
  attached?: boolean
84
+ /** Whether this runtime is IN A TURN right now — a SEPARATE concept from `status`, which
85
+ * stays pure liveness (docs/15 §Activity). Read from the runtime's own turn marker
86
+ * (codex `task_started`/`task_complete`; claude assistant `stop_reason`), never from CPU or
87
+ * an mtime proxy.
88
+ *
89
+ * ABSENT unless `status === 'live'`: an asleep/stopped session has no turn to be in, and a
90
+ * fabricated `idle` would read as a claim we have not earned. Also absent for a live session
91
+ * the daemon has no evidence for yet.
92
+ *
93
+ * `unknown` = live, saw the turn start, never saw it finish, and the evidence has gone
94
+ * stale (TURN_STALE_MS) — an honest "no claim", never a phantom `working`. */
95
+ activity?: TurnActivity
82
96
  }
83
97
 
84
98
  export interface FleetPeer {
@@ -208,12 +222,25 @@ export function buildFleetSnapshot(
208
222
  nowMs: number = Date.now(),
209
223
  approvals: FleetApproval[] = [],
210
224
  notices: FleetNotice[] = [],
225
+ /** Per-runtime turn activity (docs/15 §Activity). Injected — the snapshot never scans
226
+ * session files itself; the turn-watch owns that and keeps an in-memory board, so building
227
+ * a snapshot stays O(peers). Absent lookup (library/test daemons) → the field is simply
228
+ * never emitted, which is exactly its "no claim" semantics. */
229
+ activityOf?: (personality: string, runtime: string) => TurnActivity | undefined,
211
230
  ): FleetSnapshot {
212
231
  const peers: FleetPeer[] = ops.listPeers({ env }).map(r => {
213
232
  const runtimes: FleetRuntimeStatus[] = r.runtimes.map(s => {
214
233
  const attached =
215
234
  s.status === 'live' && hasAttachedSupervisorClient(buildProcessAddress(s.runtime, r.personality), env)
216
- return { runtime: s.runtime, status: s.status, ...(attached ? { attached: true } : {}) }
235
+ // Live-only, by design: see FleetRuntimeStatus.activity on why a dead session gets no
236
+ // claim rather than a fabricated `idle`.
237
+ const activity = s.status === 'live' ? activityOf?.(r.personality, s.runtime) : undefined
238
+ return {
239
+ runtime: s.runtime,
240
+ status: s.status,
241
+ ...(attached ? { attached: true } : {}),
242
+ ...(activity ? { activity } : {}),
243
+ }
217
244
  })
218
245
  let ephemeral = false
219
246
  try {
@@ -296,6 +323,10 @@ export function fleetEventFiles(cfg: LifecycleConfig): Array<{ path: string; src
296
323
  // path — a face subscribed to /fleet/v1/events receives owner notices with no
297
324
  // further wiring, exactly as it already receives approvals.
298
325
  { path: noticesLogPath(cfg.eventLogDir), src: 'notices' },
326
+ // Turn-activity events (docs/15 §Activity): turn-started / turn-ended. Same trick — the
327
+ // log IS the push path. Written on a real state CHANGE only, so this is two lines per turn
328
+ // per runtime, not one per sweep.
329
+ { path: turnsLogPath(cfg.eventLogDir), src: 'turns' },
299
330
  ]
300
331
  }
301
332
 
@@ -517,6 +548,11 @@ export interface FleetHandlerOptions {
517
548
  * mute-watch raises into — that shared instance IS the wiring: the watch writes, this
518
549
  * handler reads. Default: a fresh one (a handler with no watch simply stays empty). */
519
550
  board?: NoticeBoard
551
+ /** The turn-activity board (docs/15 §Activity). Same sharing contract as `board`: the
552
+ * composition point passes the SAME instance the turn-watch folds observations into.
553
+ * Default: a fresh one — a handler with no watch reports no activity at all, which is the
554
+ * correct "no claim" rather than a fabricated idle. */
555
+ activity?: ActivityBoard
520
556
  }
521
557
 
522
558
  export type FleetHandler = (
@@ -542,6 +578,11 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
542
578
  // The notice board — one per daemon; the mute-watch RAISES into it and the /notices reads
543
579
  // below serve it. Same shape as the broker above: one in-memory store, many interfaces.
544
580
  const board = opts.board ?? new NoticeBoard({ logDir: cfg.eventLogDir, env })
581
+ // The turn-activity board — same one-store-many-interfaces shape. The turn-watch folds
582
+ // observations in; the snapshot below reads it. Reading is O(1): no file touched per request.
583
+ const activity = opts.activity ?? new ActivityBoard()
584
+ const activityOf = (personality: string, runtime: string): TurnActivity | undefined =>
585
+ activity.get(personality, runtime, Date.now())
545
586
  let opsPromise: Promise<FleetOps> | undefined
546
587
  const getOps = (): Promise<FleetOps> => {
547
588
  if (opts.ops) return Promise.resolve(opts.ops)
@@ -564,7 +605,7 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
564
605
  return sendJson(
565
606
  res,
566
607
  200,
567
- buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list()),
608
+ buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list(), activityOf),
568
609
  )
569
610
  }
570
611
  if (path === '/fleet/v1/events') {
@@ -576,7 +617,17 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
576
617
  if (peerMatch) {
577
618
  if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/peers/<peer>' })
578
619
  const personality = decodeURIComponent(peerMatch[1]!)
579
- const snapshot = buildFleetSnapshot(env, cfg, await getOps(), startedAtMs)
620
+ // activityOf passed here too: peer-detail must not disagree with the list it came from.
621
+ const snapshot = buildFleetSnapshot(
622
+ env,
623
+ cfg,
624
+ await getOps(),
625
+ startedAtMs,
626
+ undefined,
627
+ undefined,
628
+ undefined,
629
+ activityOf,
630
+ )
580
631
  const peer = snapshot.peers.find(p => p.personality === personality)
581
632
  if (!peer) return sendJson(res, 404, { error: `unknown peer "${personality}"` })
582
633
  const events = readRecentEvents(files, 50, personality)
@@ -70,6 +70,7 @@ import { buildFleetHandler } from './fleet.ts'
70
70
  import { NoticeBoard } from './notices.ts'
71
71
  import { startMuteWatch } from './mutewatch.ts'
72
72
  import { startGoalWatch } from './goalwatch.ts'
73
+ import { ActivityBoard, startTurnWatch } from './turnwatch.ts'
73
74
  import { defaultDaemonSocketPath, startDaemon, type DaemonHandle } from './index.ts'
74
75
 
75
76
  /** Default TCP loopback port for the always-on router. Real http MCP clients
@@ -107,6 +108,12 @@ export const DEFAULT_MUTEWATCH_INTERVAL_MS = 20_000
107
108
  * DB, and the threads join runs ONLY when a stalled row is actually present. */
108
109
  export const DEFAULT_GOALWATCH_INTERVAL_MS = 20_000
109
110
 
111
+ /** Turn-watch cadence (docs/15 §Activity) — how often the daemon re-reads the turn markers of
112
+ * session files that CHANGED. Much faster than the notice watches because the fact is
113
+ * short-lived: a turn can be shorter than 20 s, so a 20 s sweep would report `idle` straight
114
+ * through one. This bounds the snapshot's idle↔working lag to roughly one interval. */
115
+ export const DEFAULT_TURNWATCH_INTERVAL_MS = 3_000
116
+
110
117
  // This module's own path — the launchd plist runs `bun <this>` as the daemon.
111
118
  const DAEMON_MAIN_PATH = fileURLToPath(import.meta.url)
112
119
 
@@ -339,6 +346,8 @@ export interface ConfiguredDaemonOptions {
339
346
  muteWatchIntervalMs?: number
340
347
  /** Goal-watch sweep cadence (default DEFAULT_GOALWATCH_INTERVAL_MS). */
341
348
  goalWatchIntervalMs?: number
349
+ /** Turn-watch sweep cadence (default DEFAULT_TURNWATCH_INTERVAL_MS). */
350
+ turnWatchIntervalMs?: number
342
351
  /** Write the router.json discovery file (default true for production). */
343
352
  discovery?: boolean
344
353
  rootDir?: string
@@ -392,6 +401,22 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
392
401
  } catch { /* best-effort */ }
393
402
  },
394
403
  })
404
+ // Turn-activity board + watch (docs/15 §Activity). Constructed HERE for the same reason as
405
+ // the notice board: the watch WRITES it and the fleet handler READS it, and that sharing is
406
+ // the whole wiring. Faster cadence than the notice watches (3 s vs 20 s) because a turn is a
407
+ // short-lived fact — a 20 s sweep would miss whole turns. Affordable: the sweep stats only
408
+ // files CHANGED since the last pass (measured on this host: ~2 ms across both runtime trees).
409
+ const activity = new ActivityBoard()
410
+ const stopTurnWatch = startTurnWatch(cfg, activity, {
411
+ env,
412
+ intervalMs: opts.turnWatchIntervalMs ?? DEFAULT_TURNWATCH_INTERVAL_MS,
413
+ onError: (err: unknown) => {
414
+ try {
415
+ const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
416
+ appendLifecycleEvent(cfg.eventLogDir, { ev: 'turnwatch-error', error: detail.replace(/\s+/g, ' ').slice(0, 600) }, { env })
417
+ } catch { /* best-effort */ }
418
+ },
419
+ })
395
420
  const handle = await startDaemon({
396
421
  wake: makeWakeFn(cfg, env),
397
422
  // M2 arm-on-outbound (see makeArmEphemeralOnDelivered): ephemeral caller's ok
@@ -418,7 +443,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
418
443
  // (the same listPeers truth as `iapeer list`), SSE events (tail of the durable
419
444
  // logs), commands over the existing verb functions. Same listeners, same bearer
420
445
  // gate; advertised in router.json as fleet:1. Contract: docs/15-fleet-api.md.
421
- fleet: buildFleetHandler({ env, board }),
446
+ fleet: buildFleetHandler({ env, board, activity }),
422
447
  supervise: {
423
448
  intervalMs: opts.superviseIntervalMs ?? DEFAULT_SUPERVISE_INTERVAL_MS,
424
449
  // idle-reap / zombie-sweep, THEN the eager fresh re-launch for any peer whose
@@ -466,7 +491,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
466
491
  // The mute-watch timer is OURS, not startDaemon's — so its teardown must hang off the
467
492
  // handle we return, or a closed daemon would leave a live timer sweeping the disk (and
468
493
  // every test that starts a daemon would hang on exit).
469
- return { ...handle, close: async () => { stopMuteWatch(); stopGoalWatch(); await handle.close() } }
494
+ return { ...handle, close: async () => { stopMuteWatch(); stopGoalWatch(); stopTurnWatch(); await handle.close() } }
470
495
  }
471
496
 
472
497
  // ─────────────────────────────────────────────────────────────────────────────
@@ -0,0 +1,52 @@
1
+ // Turn-activity durable log — the daemon's DURABLE trace of every per-runtime turn boundary
2
+ // it observes (docs/15 §Activity). Sibling to lifecycle.log / delivery.log / exits.log /
3
+ // approvals.log / notices.log under cfg.eventLogDir; tailed by the fleet-API SSE stream
4
+ // (fleetEventFiles), so `turn-started` / `turn-ended` reach every operator client with
5
+ // at-least-once semantics — the same "the log IS the push mechanism" trick notices.log uses.
6
+ //
7
+ // Volume is bounded by DESIGN, not by luck: turnWatchTick writes only on a real state CHANGE,
8
+ // never once per sweep. The watch polls every ~3 s; without that guard it would emit a line per
9
+ // peer per pass and drown the very log meant to make turn history readable. Two lines per turn
10
+ // per runtime is the ceiling.
11
+ //
12
+ // Same rotation class as its siblings: written by OUR code → self-rotates via the shared
13
+ // storage/rotatelog primitive. The dir is passed IN by the composition point, never re-resolved
14
+ // from env — a falsy dir → no-op (library/test daemons stay hermetic).
15
+
16
+ import { join } from 'path'
17
+ import { DEFAULT_LOG_KEEP, DEFAULT_LOG_MAX_BYTES, appendRotatedEvent } from '../storage/rotatelog.ts'
18
+
19
+ /** The turn-activity log inside `logDir` (sibling to notices.log). */
20
+ export function turnsLogPath(logDir: string): string {
21
+ return join(logDir, 'turns.log')
22
+ }
23
+
24
+ function envPosInt(raw: string | undefined, dflt: number): number {
25
+ const n = parseInt(raw ?? '', 10)
26
+ return Number.isFinite(n) && n > 0 ? n : dflt
27
+ }
28
+
29
+ export interface AppendTurnOptions {
30
+ /** Reads the rotation knobs IAPEER_TURNS_LOG_MAX_BYTES / _KEEP. */
31
+ env?: NodeJS.ProcessEnv
32
+ /** Stamp the line with this epoch-ms. Default Date.now(). */
33
+ nowMs?: number
34
+ }
35
+
36
+ /**
37
+ * Append one turn event line into `logDir`/turns.log. A falsy `logDir` is a no-op. Fully
38
+ * best-effort — never throws; logging must never fail a detection sweep.
39
+ */
40
+ export function appendTurnEvent(
41
+ logDir: string | undefined,
42
+ fields: Record<string, string | number | undefined>,
43
+ opts: AppendTurnOptions = {},
44
+ ): void {
45
+ if (!logDir) return
46
+ const env = opts.env ?? process.env
47
+ appendRotatedEvent(turnsLogPath(logDir), fields, {
48
+ nowMs: opts.nowMs,
49
+ maxBytes: envPosInt(env.IAPEER_TURNS_LOG_MAX_BYTES, DEFAULT_LOG_MAX_BYTES),
50
+ keep: envPosInt(env.IAPEER_TURNS_LOG_KEEP, DEFAULT_LOG_KEEP),
51
+ })
52
+ }
@@ -0,0 +1,450 @@
1
+ // Turn-watch — per-runtime TURN ACTIVITY for the fleet surface (docs/15 §Activity).
2
+ //
3
+ // WHY THIS EXISTS. The snapshot answered exactly one question about a session: is the process
4
+ // alive (`status: live | asleep | stopped`). That is LIVENESS, and it is not the operator's
5
+ // question. A peer sitting idle with an unfinished job and a peer grinding through a turn are
6
+ // both `live` — indistinguishable. The owner spotted a stalled peer before the PM twice
7
+ // because the surface could not tell "working" from "merely alive".
8
+ //
9
+ // ACTIVITY IS NOT LIVENESS, AND THEY STAY SEPARATE. `status` keeps its exact meaning; the new
10
+ // `activity` field is additive and rides ALONGSIDE it on the same per-runtime object. A client
11
+ // that ignores `activity` is unaffected. Activity is reported ONLY for a `live` runtime — for
12
+ // an asleep/stopped session there is no turn to be in, and the field is OMITTED rather than
13
+ // filled with a fake `idle`: absent means "no claim", which a dead session deserves.
14
+ //
15
+ // THE EVIDENCE IS THE RUNTIME'S OWN TURN MARKER — mechanical, never a CPU/mtime reconstruction:
16
+ //
17
+ // • codex → the rollout's `event_msg` payloads `task_started` / `task_complete`. Explicit
18
+ // turn boundaries; the last one wins. Measured across 3 days of real rollouts: 156 started
19
+ // / 147 complete — the 9-line gap is precisely the dangling-turn risk the stale bound below
20
+ // exists for. Codex emits `task_complete` even for a turn that ended on an error (measured:
21
+ // the 16.07 goal-blocking turn ends token_count → task_complete, no error event), so the
22
+ // terminal error path needs no separate marker.
23
+ //
24
+ // • claude → the transcript's assistant `stop_reason`. `tool_use` means the turn CONTINUES;
25
+ // every other terminal reason means it ended. Measured over every transcript this peer has:
26
+ // tool_use 18317, end_turn 663, stop_sequence 16, null 4. So the rule is deliberately
27
+ // ASYMMETRIC — only `tool_use` reads as working, and anything else (end_turn, stop_sequence,
28
+ // max_tokens, refusal, an unknown future value, null) reads as IDLE. An unknown reason must
29
+ // never invent work; the polarity always fails toward idle.
30
+ //
31
+ // THE THREE FALSE-WORKING TRAPS, each measured on real transcripts rather than assumed:
32
+ //
33
+ // 1. **A `user` line means working** (the model is about to answer, or a tool_result just
34
+ // landed) — but claude writes NON-model user lines too. A compaction summary is
35
+ // `type:"user"` with `isCompactSummary:true` (12 found here), and a `/compact` run while
36
+ // IDLE would leave one as the last line → a permanent phantom "working". Excluded. The
37
+ // `compact_boundary` line is `type:"system"` and never qualifies anyway. Excluding the
38
+ // summary is also CORRECT for mid-turn compaction: the prior assistant `tool_use` line
39
+ // then decides, and the turn is genuinely still running.
40
+ // 2. **Sub-agent lines.** A Task sidechain writes assistant/user lines into the SAME
41
+ // transcript with `isSidechain:true`. They belong to a different chain and must not decide
42
+ // the peer's turn state. Excluded.
43
+ // 3. **Injected meta lines** (`isMeta:true`, 13 found here) are not the model being asked
44
+ // anything. Excluded.
45
+ //
46
+ // THE STALE BOUND. A `working` claim rests on evidence that stops being written the moment a
47
+ // session hangs mid-turn — the transcript simply goes quiet. Liveness already removes the dead
48
+ // session (activity is live-only), so what remains is a LIVE session whose last recorded act
49
+ // was starting work and which has since written nothing. Past TURN_STALE_MS the claim decays to
50
+ // `unknown` rather than standing as `working` forever: unknown is the honest word for "we saw
51
+ // it start and never saw it finish". `idle` never decays — a session that ended its turn stays
52
+ // idle until something pokes it, and age proves nothing against that.
53
+ //
54
+ // Deliberately NOT the pane-log mtime, which looks like a freshness clock and is not: measured
55
+ // live 16.07, an IDLE claude pane-log advances every second (the statusline repaints) while an
56
+ // IDLE codex pane-log freezes (39s→46s). The signal means opposite things per runtime; one
57
+ // clock that lies differently on each runtime is worse than no clock.
58
+
59
+ import { existsSync, openSync, closeSync, fstatSync, readSync, readdirSync, statSync } from 'node:fs'
60
+ import { join } from 'node:path'
61
+ import type { LifecycleConfig } from '../lifecycle/index.ts'
62
+ import { appendTurnEvent } from './turnslog.ts'
63
+
64
+ /** Tail parsed per session file. The turn marker is always at the very end. */
65
+ const TAIL_BYTES = 128 * 1024
66
+ /** Head slice for codex's `session_meta` (line 1 — a long rollout pushes it out of the tail). */
67
+ const HEAD_BYTES = 64 * 1024
68
+ /** Walk depth for the codex YYYY/MM/DD tree. */
69
+ const CODEX_WALK_DEPTH = 3
70
+ /** How far back the FIRST sweep looks, to seed state for sessions already running when the
71
+ * daemon started. Bounded on purpose: a full-history pass would read every transcript ever.
72
+ * Measured cost of a 48h seed on this host: ~2 ms of walk, 97 files. */
73
+ const SEED_LOOKBACK_MS = 48 * 3_600_000
74
+ /** Overlap on the since-boundary so a write landing between sweeps is never missed. */
75
+ const SCAN_OVERLAP_MS = 2_000
76
+
77
+ /**
78
+ * How long a `working` claim outlives its last evidence before decaying to `unknown`.
79
+ * Generous on purpose: a turn legitimately writes NOTHING while one long tool runs (a deploy
80
+ * here takes minutes), and a false `unknown` on a genuinely working peer is a worse lie than a
81
+ * late one. Ten minutes is far past any tool this fleet runs and still bounds the phantom.
82
+ */
83
+ export const TURN_STALE_MS = 10 * 60_000
84
+
85
+ /** The operator-facing verdict. `unknown` = live, but the evidence cannot support a claim. */
86
+ export type TurnActivity = 'working' | 'idle' | 'unknown'
87
+
88
+ export interface TurnState {
89
+ state: 'working' | 'idle'
90
+ /** The EVIDENCE's own timestamp (the transcript line's), never our sweep clock. */
91
+ atMs: number
92
+ cwd: string
93
+ }
94
+
95
+ // ─────────────────────────────────────────────────────────────────────────────
96
+ // Pure parsers — text in, state out (fs-free: tests need no session tree)
97
+ // ─────────────────────────────────────────────────────────────────────────────
98
+
99
+ function toMs(ts: unknown): number | null {
100
+ if (typeof ts !== 'string') return null
101
+ const ms = Date.parse(ts)
102
+ return Number.isFinite(ms) ? ms : null
103
+ }
104
+
105
+ /**
106
+ * The claude transcript's turn state, or null if the file states nothing usable.
107
+ *
108
+ * Only `tool_use` continues a turn; every other stop_reason ends it (see the header on why the
109
+ * polarity is asymmetric). Compaction summaries, sidechain lines and meta lines are excluded —
110
+ * each is a measured false-working vector.
111
+ */
112
+ export function parseClaudeTurn(text: string): TurnState | null {
113
+ let found: TurnState | null = null
114
+ for (const line of text.split('\n')) {
115
+ const s = line.trim()
116
+ if (!s.startsWith('{')) continue // a partial first line from the tail cut
117
+ let d: Record<string, unknown>
118
+ try {
119
+ d = JSON.parse(s) as Record<string, unknown>
120
+ } catch {
121
+ continue // never fail a sweep over one garbled line
122
+ }
123
+ const type = d.type
124
+ if (type !== 'assistant' && type !== 'user') continue
125
+ if (d.isSidechain === true) continue // a sub-agent's chain, not the peer's turn
126
+ if (d.isCompactSummary === true) continue // compaction artifact, not the model being asked
127
+ if (d.isMeta === true) continue // injected, not a real prompt
128
+ const atMs = toMs(d.timestamp)
129
+ if (atMs === null) continue
130
+ const cwd = typeof d.cwd === 'string' ? d.cwd : ''
131
+ if (!cwd) continue // unattributable to a peer → ignore rather than guess
132
+ let state: 'working' | 'idle'
133
+ if (type === 'assistant') {
134
+ const msg = d.message as { stop_reason?: unknown } | undefined
135
+ // ONLY tool_use continues. end_turn / stop_sequence / max_tokens / refusal / an unknown
136
+ // future value / null all mean the assistant stopped.
137
+ state = msg?.stop_reason === 'tool_use' ? 'working' : 'idle'
138
+ } else {
139
+ // A real user line: a prompt to answer or a tool_result to consume — either way the
140
+ // model is on the hook. (The non-model user lines were filtered above.)
141
+ state = 'working'
142
+ }
143
+ found = { state, atMs, cwd }
144
+ }
145
+ return found
146
+ }
147
+
148
+ /**
149
+ * The codex rollout's turn state, or null. `task_started` / `task_complete` are explicit
150
+ * boundaries — the last one wins. `cwd` comes from `session_meta` (line 1), so pass the file
151
+ * HEAD when the tail may have cut it away.
152
+ */
153
+ export function parseCodexTurn(text: string, head = ''): TurnState | null {
154
+ let cwd = ''
155
+ const scanMeta = (src: string): void => {
156
+ for (const line of src.split('\n')) {
157
+ const s = line.trim()
158
+ if (!s.startsWith('{') || !s.includes('session_meta')) continue
159
+ try {
160
+ const d = JSON.parse(s) as { type?: string; payload?: { cwd?: unknown } }
161
+ if (d.type !== 'session_meta' || !d.payload) continue
162
+ if (typeof d.payload.cwd === 'string') cwd = d.payload.cwd
163
+ } catch {
164
+ /* ignore */
165
+ }
166
+ }
167
+ }
168
+ scanMeta(head)
169
+ if (!cwd) scanMeta(text)
170
+ if (!cwd) return null // unattributable to a peer
171
+
172
+ let found: TurnState | null = null
173
+ for (const line of text.split('\n')) {
174
+ const s = line.trim()
175
+ if (!s.startsWith('{') || !s.includes('task_')) continue
176
+ let d: { timestamp?: unknown; type?: unknown; payload?: { type?: unknown } }
177
+ try {
178
+ d = JSON.parse(s)
179
+ } catch {
180
+ continue
181
+ }
182
+ if (d.type !== 'event_msg') continue
183
+ const p = d.payload?.type
184
+ if (p !== 'task_started' && p !== 'task_complete') continue
185
+ const atMs = toMs(d.timestamp)
186
+ if (atMs === null) continue
187
+ found = { state: p === 'task_started' ? 'working' : 'idle', atMs, cwd }
188
+ }
189
+ return found
190
+ }
191
+
192
+ // ─────────────────────────────────────────────────────────────────────────────
193
+ // The board — in-memory (personality, runtime) → last known turn state
194
+ // ─────────────────────────────────────────────────────────────────────────────
195
+
196
+ function key(personality: string, runtime: string): string {
197
+ return `${runtime}-${personality}` // the identity scheme (docs/03)
198
+ }
199
+
200
+ /**
201
+ * Last known turn state per (personality, runtime). In-memory and ephemeral, exactly like the
202
+ * notice board: a restart re-seeds from the session files, which are the durable truth.
203
+ */
204
+ export class ActivityBoard {
205
+ private readonly states = new Map<string, TurnState>()
206
+
207
+ /** Record an observation. Returns the PREVIOUS state, so the caller can log only real
208
+ * transitions rather than one line per sweep. */
209
+ set(personality: string, runtime: string, next: TurnState): TurnState | undefined {
210
+ const k = key(personality, runtime)
211
+ const prev = this.states.get(k)
212
+ // Evidence older than what we already hold is a stale re-read — never let it rewind state.
213
+ if (prev && next.atMs < prev.atMs) return prev
214
+ this.states.set(k, next)
215
+ return prev
216
+ }
217
+
218
+ /** The operator-facing verdict, with the stale bound applied at READ time (so a claim decays
219
+ * even while nothing new is swept). `undefined` → we have no evidence at all. */
220
+ get(personality: string, runtime: string, nowMs: number): TurnActivity | undefined {
221
+ const st = this.states.get(key(personality, runtime))
222
+ if (!st) return undefined
223
+ if (st.state === 'idle') return 'idle' // idle never decays — age proves nothing against it
224
+ return nowMs - st.atMs > TURN_STALE_MS ? 'unknown' : 'working'
225
+ }
226
+
227
+ /** The raw record (peer-detail / tests). */
228
+ raw(personality: string, runtime: string): TurnState | undefined {
229
+ return this.states.get(key(personality, runtime))
230
+ }
231
+
232
+ size(): number {
233
+ return this.states.size
234
+ }
235
+ }
236
+
237
+ // ─────────────────────────────────────────────────────────────────────────────
238
+ // The sweep
239
+ // ─────────────────────────────────────────────────────────────────────────────
240
+
241
+ function readTail(path: string): { tail: string; head: string } | null {
242
+ try {
243
+ const fd = openSync(path, 'r')
244
+ try {
245
+ const size = fstatSync(fd).size
246
+ if (size <= 0) return null
247
+ const off = Math.max(0, size - TAIL_BYTES)
248
+ const buf = Buffer.alloc(size - off)
249
+ readSync(fd, buf, 0, buf.length, off)
250
+ let head = ''
251
+ if (off > 0) {
252
+ const hb = Buffer.alloc(Math.min(HEAD_BYTES, off))
253
+ readSync(fd, hb, 0, hb.length, 0)
254
+ head = hb.toString('utf8')
255
+ }
256
+ return { tail: buf.toString('utf8'), head }
257
+ } finally {
258
+ closeSync(fd)
259
+ }
260
+ } catch {
261
+ return null
262
+ }
263
+ }
264
+
265
+ function safeReaddir(dir: string): string[] {
266
+ try {
267
+ return readdirSync(dir)
268
+ } catch {
269
+ return []
270
+ }
271
+ }
272
+
273
+ /** *.jsonl under `root` (depth-bounded) whose mtime is at/after `sinceMs`. */
274
+ function changedJsonl(root: string, sinceMs: number, depth: number): string[] {
275
+ if (!existsSync(root)) return []
276
+ const out: string[] = []
277
+ const walk = (dir: string, left: number): void => {
278
+ for (const name of safeReaddir(dir)) {
279
+ const p = join(dir, name)
280
+ if (name.endsWith('.jsonl')) {
281
+ let m: number | null = null
282
+ try {
283
+ m = statSync(p).mtimeMs
284
+ } catch {
285
+ continue
286
+ }
287
+ if (m >= sinceMs) out.push(p)
288
+ continue
289
+ }
290
+ if (left <= 0 || name.startsWith('.')) continue
291
+ try {
292
+ if (statSync(p).isDirectory()) walk(p, left - 1)
293
+ } catch {
294
+ continue
295
+ }
296
+ }
297
+ }
298
+ walk(root, depth)
299
+ return out
300
+ }
301
+
302
+ export interface TurnWatchPaths {
303
+ /** ~/.claude/projects — INJECTED, never re-resolved from env inside a sweep. */
304
+ claudeProjectsDir: string
305
+ /** ~/.codex/sessions — same rule. */
306
+ codexSessionsDir: string
307
+ }
308
+
309
+ export function resolveTurnWatchPaths(env: NodeJS.ProcessEnv): TurnWatchPaths {
310
+ const home = env.HOME?.trim() || ''
311
+ return {
312
+ claudeProjectsDir: env.IAPEER_CLAUDE_PROJECTS_DIR?.trim() || join(home, '.claude', 'projects'),
313
+ codexSessionsDir: env.IAPEER_CODEX_SESSIONS_DIR?.trim() || join(home, '.codex', 'sessions'),
314
+ }
315
+ }
316
+
317
+ export interface TurnScanDeps extends TurnWatchPaths {
318
+ /** personality ← cwd. Built from the registry by the composition point. */
319
+ peerByCwd: (cwd: string) => string | undefined
320
+ }
321
+
322
+ export interface TurnObservation extends TurnState {
323
+ personality: string
324
+ runtime: 'claude' | 'codex'
325
+ }
326
+
327
+ /** One sweep: the turn state of every peer session file touched since `sinceMs`. */
328
+ export function scanTurnStates(deps: TurnScanDeps, sinceMs: number): TurnObservation[] {
329
+ const floor = sinceMs - SCAN_OVERLAP_MS
330
+ const out: TurnObservation[] = []
331
+ const push = (st: TurnState | null, runtime: 'claude' | 'codex'): void => {
332
+ if (!st) return
333
+ const personality = deps.peerByCwd(st.cwd)
334
+ if (!personality) return // a human's own session — never reported
335
+ out.push({ ...st, personality, runtime })
336
+ }
337
+ for (const file of changedJsonl(deps.claudeProjectsDir, floor, 1)) {
338
+ const t = readTail(file)
339
+ if (t) push(parseClaudeTurn(t.tail), 'claude')
340
+ }
341
+ for (const file of changedJsonl(deps.codexSessionsDir, floor, CODEX_WALK_DEPTH)) {
342
+ const t = readTail(file)
343
+ if (t) push(parseCodexTurn(t.tail, t.head), 'codex')
344
+ }
345
+ return out
346
+ }
347
+
348
+ export interface TurnWatchDeps extends TurnScanDeps {
349
+ board: ActivityBoard
350
+ /** cfg.eventLogDir — turns.log (falsy → no durable log, hermetic). */
351
+ logDir?: string
352
+ env?: NodeJS.ProcessEnv
353
+ now?: () => number
354
+ onError?: (err: unknown) => void
355
+ }
356
+
357
+ /**
358
+ * Run one watch tick: sweep, fold into the board, log REAL transitions only.
359
+ * Returns how many transitions were logged plus the new since-boundary.
360
+ */
361
+ export function turnWatchTick(deps: TurnWatchDeps, sinceMs: number): { transitions: number; sinceMs: number } {
362
+ const now = deps.now ?? Date.now
363
+ const startedMs = now()
364
+ let transitions = 0
365
+ try {
366
+ for (const o of scanTurnStates(deps, sinceMs)) {
367
+ const prev = deps.board.set(o.personality, o.runtime, { state: o.state, atMs: o.atMs, cwd: o.cwd })
368
+ // Only a genuine change is an event. Without this the 3 s sweep would write a line per
369
+ // pass per peer and bury the log it is supposed to make readable.
370
+ if (prev?.state === o.state) continue
371
+ // A rewound observation (older evidence than held) is not a transition either.
372
+ if (prev && o.atMs < prev.atMs) continue
373
+ transitions += 1
374
+ appendTurnEvent(
375
+ deps.logDir,
376
+ {
377
+ ev: o.state === 'working' ? 'turn-started' : 'turn-ended',
378
+ personality: o.personality,
379
+ runtime: o.runtime,
380
+ from: prev?.state,
381
+ at: o.atMs,
382
+ at_iso: new Date(o.atMs).toISOString(),
383
+ },
384
+ { env: deps.env, nowMs: startedMs },
385
+ )
386
+ }
387
+ } catch (e) {
388
+ deps.onError?.(e)
389
+ }
390
+ return { transitions, sinceMs: startedMs }
391
+ }
392
+
393
+ export interface StartTurnWatchOptions {
394
+ env?: NodeJS.ProcessEnv
395
+ intervalMs?: number
396
+ paths?: TurnWatchPaths
397
+ listPeers?: () => Array<{ personality: string; cwd: string }>
398
+ now?: () => number
399
+ onError?: (err: unknown) => void
400
+ }
401
+
402
+ /**
403
+ * Start the turn-watch timer. Returns its stop function — the caller OWNS teardown.
404
+ * The FIRST sweep reaches back SEED_LOOKBACK_MS so sessions already mid-turn when the daemon
405
+ * started are known immediately; every later sweep is incremental (changed files only).
406
+ */
407
+ export function startTurnWatch(cfg: LifecycleConfig, board: ActivityBoard, opts: StartTurnWatchOptions = {}): () => void {
408
+ const env = opts.env ?? process.env
409
+ const paths = opts.paths ?? resolveTurnWatchPaths(env)
410
+ const now = opts.now ?? Date.now
411
+ let listPeers = opts.listPeers
412
+ let since = now() - SEED_LOOKBACK_MS
413
+ let running = false
414
+
415
+ const sweep = async (): Promise<void> => {
416
+ if (running) return // a slow sweep must never overlap itself into a stampede
417
+ running = true
418
+ try {
419
+ if (!listPeers) {
420
+ const cli = await import('../cli/index.ts')
421
+ listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
422
+ }
423
+ const rows = listPeers()
424
+ // Case-insensitive, trailing-slash-tolerant — same rule and reason as mutewatch.
425
+ const byCwd = new Map<string, string>()
426
+ for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
427
+ since = turnWatchTick(
428
+ {
429
+ ...paths,
430
+ board,
431
+ logDir: cfg.eventLogDir,
432
+ env,
433
+ now,
434
+ onError: opts.onError,
435
+ peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
436
+ },
437
+ since,
438
+ ).sinceMs
439
+ } catch (e) {
440
+ opts.onError?.(e)
441
+ } finally {
442
+ running = false
443
+ }
444
+ }
445
+
446
+ void sweep() // seed immediately — do not make the operator wait one interval for the truth
447
+ const timer = setInterval(() => void sweep(), opts.intervalMs ?? 3_000)
448
+ ;(timer as { unref?: () => void }).unref?.()
449
+ return () => clearInterval(timer)
450
+ }