@agfpd/iapeer 0.4.97 → 0.4.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/15-fleet-api.md +62 -2
- package/package.json +1 -1
- package/src/daemon/fleet.ts +54 -3
- package/src/daemon/main.ts +27 -2
- package/src/daemon/turnslog.ts +52 -0
- package/src/daemon/turnwatch.ts +450 -0
package/docs/15-fleet-api.md
CHANGED
|
@@ -48,7 +48,7 @@ The full fleet state. The peer rows are produced by the **same in-process functi
|
|
|
48
48
|
"default_runtime": "claude",
|
|
49
49
|
"cwd": "/Users/me/.iapeer/peers/boris",
|
|
50
50
|
"runtimes": [
|
|
51
|
-
{ "runtime": "claude", "status": "live", "attached": true },
|
|
51
|
+
{ "runtime": "claude", "status": "live", "attached": true, "activity": "working" },
|
|
52
52
|
{ "runtime": "codex", "status": "asleep" }
|
|
53
53
|
],
|
|
54
54
|
"last_active_runtime": "claude",
|
|
@@ -78,6 +78,7 @@ Field notes:
|
|
|
78
78
|
|
|
79
79
|
- `runtimes[].status` — `live` (session up) · `asleep` (wakeable on demand) · `stopped` (operator stop flag; the daemon won't wake it). Same glyph semantics as the CLI: `●` / `○` / `✕`.
|
|
80
80
|
- `runtimes[].attached` — present (`true`) only when a **human operator** is attached to that live hosted session.
|
|
81
|
+
- `runtimes[].activity` — **turn activity**: is this runtime *in a turn* right now. See «Activity» below. Additive; absent on a pre-activity daemon, on every non-`live` runtime, and whenever the daemon has no evidence — clients MUST treat absence as **no claim**, never as `idle`.
|
|
81
82
|
- `attached` — any runtime attached (peer-level convenience).
|
|
82
83
|
- `launchd_managed` — launchd owns this peer's lifecycle (H4): the daemon never wakes/reaps it, and wake/stop/start commands are guarded accordingly.
|
|
83
84
|
- `wake_policy` — `warm` (default: wake-on-message, idle-reap, resume) or `ephemeral` (stateless worker: serial task queue, die-after-reply).
|
|
@@ -90,6 +91,65 @@ Field notes:
|
|
|
90
91
|
- `approvals` (top-level, docs/17-approval) — the pending human-approval queue, the SAME items `GET /fleet/v1/approvals` returns, carried in the snapshot so a client that already re-fetches `/snapshot` on every event renders the queue with no extra call. **ADDITIVE + OMITTED when the queue is empty** — a client MUST treat its absence as an empty queue (the same rule as `approval_mode`). Each item: `id` (broker id, the target of `POST /fleet/v1/approvals/<id>/(approve|deny)`), `personality` + `runtime` (whose action), `kind` (`tool`|`plan`|`question`|`circuit-breaker`), `tool` (the tool / breaker name), `summary` (a one-line badge string), `content` (the FULL verbatim action — command / diff / plan; criterion #7), `createdMs` / `expiresMs`. The broker's item is a superset — unknown extra fields (`title`, `approvers`) MUST be ignored (obligation 2).
|
|
91
92
|
- `notices` (top-level, docs/19-notices) — live OWNER-facing notices: things the daemon noticed that the peer itself could not report, the v1 case being a structural API error that left it MUTE (`kind: peer-mute`). The SAME items `GET /fleet/v1/notices` returns. **ADDITIVE + OMITTED when the board is empty** — absence ⇒ empty board (the same rule as `approvals`). **READ-ONLY**: a notice is one-way, there is nothing to approve or deny. Each item: `id`, `personality` + `runtime` (who went mute), `kind` (`peer-mute`; treat unknown kinds as renderable), `errorType` (the RUNTIME's own value — `rate_limit`, `overloaded`, `rate_limit_reached`, … — free-form, do not switch exhaustively), `model?`, `resetsAtMs?`, `summary`, `content` (verbatim for claude), `sessionId?`, `createdMs` / `lastMs` / `expiresMs`, `count` (detections folded in — render as `×N`). **`model` and `resetsAtMs` are ABSENT when the runtime did not state them** — absence means *the runtime did not say*, NOT *there is no limit*; a client MUST render unknown and MUST NOT substitute the 5h/7d reset, which belongs to a different limit (docs/19 §3).
|
|
92
93
|
|
|
94
|
+
## Activity — is this runtime in a turn?
|
|
95
|
+
|
|
96
|
+
`runtimes[].activity` answers the operator's actual question, which `status` never did: a peer
|
|
97
|
+
idling with an unfinished job and a peer grinding through a turn are BOTH `live`.
|
|
98
|
+
|
|
99
|
+
**Activity is not liveness, and the two never merge.** `status` keeps its exact prior meaning.
|
|
100
|
+
`activity` is an additive sibling on the same per-runtime object.
|
|
101
|
+
|
|
102
|
+
| value | meaning |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `working` | a turn is in flight on this runtime |
|
|
105
|
+
| `idle` | live, but not in a turn |
|
|
106
|
+
| `unknown` | live; the turn was seen to start, never seen to finish, and the evidence has gone stale |
|
|
107
|
+
| *absent* | **no claim** — see below |
|
|
108
|
+
|
|
109
|
+
`activity` is **ABSENT** (never `idle`) when any of these hold, and a client MUST NOT infer
|
|
110
|
+
idleness from absence:
|
|
111
|
+
|
|
112
|
+
- the daemon predates this field (obligation 1);
|
|
113
|
+
- `status !== 'live'` — an asleep/stopped session has no turn to be in;
|
|
114
|
+
- the daemon has no evidence for that live runtime yet.
|
|
115
|
+
|
|
116
|
+
### Where the value comes from
|
|
117
|
+
|
|
118
|
+
The **runtime's own turn marker**, read from its session file. Never CPU, never a pane-log
|
|
119
|
+
mtime, never a reconstruction:
|
|
120
|
+
|
|
121
|
+
- **codex** — the rollout's `event_msg` payloads `task_started` → `working`, `task_complete` →
|
|
122
|
+
`idle`; last one wins. Codex emits `task_complete` for a turn that ends on an error too, so
|
|
123
|
+
the terminal-error path needs no separate marker.
|
|
124
|
+
- **claude** — the transcript's assistant `stop_reason`. **Only `tool_use` means the turn
|
|
125
|
+
continues**; every other terminal reason (`end_turn`, `stop_sequence`, `max_tokens`, a future
|
|
126
|
+
value, `null`) means it ended. The polarity is deliberately asymmetric: an unrecognised reason
|
|
127
|
+
must never invent work. A genuine `user` line also reads `working` (a prompt to answer or a
|
|
128
|
+
tool_result to consume), but compaction summaries (`isCompactSummary`), sub-agent sidechains
|
|
129
|
+
(`isSidechain`) and injected meta lines (`isMeta`) are excluded — each is a measured
|
|
130
|
+
false-`working` vector, the worst being a `/compact` run while idle leaving a phantom
|
|
131
|
+
`working` forever.
|
|
132
|
+
|
|
133
|
+
### Bounds a client can rely on
|
|
134
|
+
|
|
135
|
+
- **Lag** — the daemon re-reads changed session files every **~3 s**, so an `idle`↔`working`
|
|
136
|
+
flip appears within roughly one interval plus fs latency. A turn shorter than the interval may
|
|
137
|
+
never be observed at all; activity is a *state* to render, not an event ledger to count.
|
|
138
|
+
- **Stale bound** — a `working` claim whose evidence is older than **10 min** decays to
|
|
139
|
+
`unknown`. `idle` NEVER decays: a session that ended its turn stays idle until something pokes
|
|
140
|
+
it, and age proves nothing against that.
|
|
141
|
+
- **Restart** — the board is in-memory and re-seeds from the session files (a 48 h look-back) on
|
|
142
|
+
daemon start, so a restart cannot preserve a phantom `working`; a stale one decays by the rule
|
|
143
|
+
above, and a dead session drops out via `status` first.
|
|
144
|
+
|
|
145
|
+
### Events
|
|
146
|
+
|
|
147
|
+
`turns.log` → the SSE stream, `src: "turns"`, `ev`: **`turn-started`** / **`turn-ended`**, with
|
|
148
|
+
`personality`, `runtime`, `from` (the previous state, absent on first observation), `at` +
|
|
149
|
+
`at_iso` (the MARKER's own timestamp, not the sweep's). Emitted only on a real state **change** —
|
|
150
|
+
two lines per turn per runtime, not one per sweep. Treat the snapshot as the source of truth and
|
|
151
|
+
these as a push hint (obligation: the stream is at-least-once).
|
|
152
|
+
|
|
93
153
|
## GET /fleet/v1/peers/<personality>
|
|
94
154
|
|
|
95
155
|
One peer's card: the same object as the snapshot row, plus its recent history.
|
|
@@ -122,7 +182,7 @@ GET /fleet/v1/events?replay=50
|
|
|
122
182
|
- A comment `: connected` marks the transition from replay to live; `: hb` comments every ~15 s keep the connection alive.
|
|
123
183
|
- Latency: live events surface within ~0.5 s (log-tail poll).
|
|
124
184
|
|
|
125
|
-
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently).
|
|
185
|
+
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently), `turn-started` / `turn-ended` (from `turns.log`, `src: turns` — a runtime entered/left a turn; see «Activity». One line per real state CHANGE, never one per sweep).
|
|
126
186
|
|
|
127
187
|
What the headline states look like on the stream: a peer **waking** is `ev=wake`; a peer **parked to sleep** is `ev=supervise action=reaped-idle`; an operator **stop/start** is `ev=stopped` / `ev=started`; a **death** is `ev=session-exit` (immediately, from the supervisor) followed by the daemon's classification on its next pass.
|
|
128
188
|
|
package/package.json
CHANGED
package/src/daemon/fleet.ts
CHANGED
|
@@ -64,6 +64,8 @@ import { approvalsLogPath } from './approvalslog.ts'
|
|
|
64
64
|
import { ApprovalBroker } from './approvals.ts'
|
|
65
65
|
import { noticesLogPath } from './noticeslog.ts'
|
|
66
66
|
import { NoticeBoard } from './notices.ts'
|
|
67
|
+
import { turnsLogPath } from './turnslog.ts'
|
|
68
|
+
import { ActivityBoard, type TurnActivity } from './turnwatch.ts'
|
|
67
69
|
import { callTool, type CallToolDeps } from './index.ts'
|
|
68
70
|
|
|
69
71
|
/** The API generation baked into every snapshot (additive evolution within v1 —
|
|
@@ -79,6 +81,18 @@ export interface FleetRuntimeStatus {
|
|
|
79
81
|
status: 'live' | 'asleep' | 'stopped'
|
|
80
82
|
/** Present (true) iff a HUMAN operator is attached to this live hosted session. */
|
|
81
83
|
attached?: boolean
|
|
84
|
+
/** Whether this runtime is IN A TURN right now — a SEPARATE concept from `status`, which
|
|
85
|
+
* stays pure liveness (docs/15 §Activity). Read from the runtime's own turn marker
|
|
86
|
+
* (codex `task_started`/`task_complete`; claude assistant `stop_reason`), never from CPU or
|
|
87
|
+
* an mtime proxy.
|
|
88
|
+
*
|
|
89
|
+
* ABSENT unless `status === 'live'`: an asleep/stopped session has no turn to be in, and a
|
|
90
|
+
* fabricated `idle` would read as a claim we have not earned. Also absent for a live session
|
|
91
|
+
* the daemon has no evidence for yet.
|
|
92
|
+
*
|
|
93
|
+
* `unknown` = live, saw the turn start, never saw it finish, and the evidence has gone
|
|
94
|
+
* stale (TURN_STALE_MS) — an honest "no claim", never a phantom `working`. */
|
|
95
|
+
activity?: TurnActivity
|
|
82
96
|
}
|
|
83
97
|
|
|
84
98
|
export interface FleetPeer {
|
|
@@ -208,12 +222,25 @@ export function buildFleetSnapshot(
|
|
|
208
222
|
nowMs: number = Date.now(),
|
|
209
223
|
approvals: FleetApproval[] = [],
|
|
210
224
|
notices: FleetNotice[] = [],
|
|
225
|
+
/** Per-runtime turn activity (docs/15 §Activity). Injected — the snapshot never scans
|
|
226
|
+
* session files itself; the turn-watch owns that and keeps an in-memory board, so building
|
|
227
|
+
* a snapshot stays O(peers). Absent lookup (library/test daemons) → the field is simply
|
|
228
|
+
* never emitted, which is exactly its "no claim" semantics. */
|
|
229
|
+
activityOf?: (personality: string, runtime: string) => TurnActivity | undefined,
|
|
211
230
|
): FleetSnapshot {
|
|
212
231
|
const peers: FleetPeer[] = ops.listPeers({ env }).map(r => {
|
|
213
232
|
const runtimes: FleetRuntimeStatus[] = r.runtimes.map(s => {
|
|
214
233
|
const attached =
|
|
215
234
|
s.status === 'live' && hasAttachedSupervisorClient(buildProcessAddress(s.runtime, r.personality), env)
|
|
216
|
-
|
|
235
|
+
// Live-only, by design: see FleetRuntimeStatus.activity on why a dead session gets no
|
|
236
|
+
// claim rather than a fabricated `idle`.
|
|
237
|
+
const activity = s.status === 'live' ? activityOf?.(r.personality, s.runtime) : undefined
|
|
238
|
+
return {
|
|
239
|
+
runtime: s.runtime,
|
|
240
|
+
status: s.status,
|
|
241
|
+
...(attached ? { attached: true } : {}),
|
|
242
|
+
...(activity ? { activity } : {}),
|
|
243
|
+
}
|
|
217
244
|
})
|
|
218
245
|
let ephemeral = false
|
|
219
246
|
try {
|
|
@@ -296,6 +323,10 @@ export function fleetEventFiles(cfg: LifecycleConfig): Array<{ path: string; src
|
|
|
296
323
|
// path — a face subscribed to /fleet/v1/events receives owner notices with no
|
|
297
324
|
// further wiring, exactly as it already receives approvals.
|
|
298
325
|
{ path: noticesLogPath(cfg.eventLogDir), src: 'notices' },
|
|
326
|
+
// Turn-activity events (docs/15 §Activity): turn-started / turn-ended. Same trick — the
|
|
327
|
+
// log IS the push path. Written on a real state CHANGE only, so this is two lines per turn
|
|
328
|
+
// per runtime, not one per sweep.
|
|
329
|
+
{ path: turnsLogPath(cfg.eventLogDir), src: 'turns' },
|
|
299
330
|
]
|
|
300
331
|
}
|
|
301
332
|
|
|
@@ -517,6 +548,11 @@ export interface FleetHandlerOptions {
|
|
|
517
548
|
* mute-watch raises into — that shared instance IS the wiring: the watch writes, this
|
|
518
549
|
* handler reads. Default: a fresh one (a handler with no watch simply stays empty). */
|
|
519
550
|
board?: NoticeBoard
|
|
551
|
+
/** The turn-activity board (docs/15 §Activity). Same sharing contract as `board`: the
|
|
552
|
+
* composition point passes the SAME instance the turn-watch folds observations into.
|
|
553
|
+
* Default: a fresh one — a handler with no watch reports no activity at all, which is the
|
|
554
|
+
* correct "no claim" rather than a fabricated idle. */
|
|
555
|
+
activity?: ActivityBoard
|
|
520
556
|
}
|
|
521
557
|
|
|
522
558
|
export type FleetHandler = (
|
|
@@ -542,6 +578,11 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
542
578
|
// The notice board — one per daemon; the mute-watch RAISES into it and the /notices reads
|
|
543
579
|
// below serve it. Same shape as the broker above: one in-memory store, many interfaces.
|
|
544
580
|
const board = opts.board ?? new NoticeBoard({ logDir: cfg.eventLogDir, env })
|
|
581
|
+
// The turn-activity board — same one-store-many-interfaces shape. The turn-watch folds
|
|
582
|
+
// observations in; the snapshot below reads it. Reading is O(1): no file touched per request.
|
|
583
|
+
const activity = opts.activity ?? new ActivityBoard()
|
|
584
|
+
const activityOf = (personality: string, runtime: string): TurnActivity | undefined =>
|
|
585
|
+
activity.get(personality, runtime, Date.now())
|
|
545
586
|
let opsPromise: Promise<FleetOps> | undefined
|
|
546
587
|
const getOps = (): Promise<FleetOps> => {
|
|
547
588
|
if (opts.ops) return Promise.resolve(opts.ops)
|
|
@@ -564,7 +605,7 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
564
605
|
return sendJson(
|
|
565
606
|
res,
|
|
566
607
|
200,
|
|
567
|
-
buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list()),
|
|
608
|
+
buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list(), activityOf),
|
|
568
609
|
)
|
|
569
610
|
}
|
|
570
611
|
if (path === '/fleet/v1/events') {
|
|
@@ -576,7 +617,17 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
576
617
|
if (peerMatch) {
|
|
577
618
|
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/peers/<peer>' })
|
|
578
619
|
const personality = decodeURIComponent(peerMatch[1]!)
|
|
579
|
-
|
|
620
|
+
// activityOf passed here too: peer-detail must not disagree with the list it came from.
|
|
621
|
+
const snapshot = buildFleetSnapshot(
|
|
622
|
+
env,
|
|
623
|
+
cfg,
|
|
624
|
+
await getOps(),
|
|
625
|
+
startedAtMs,
|
|
626
|
+
undefined,
|
|
627
|
+
undefined,
|
|
628
|
+
undefined,
|
|
629
|
+
activityOf,
|
|
630
|
+
)
|
|
580
631
|
const peer = snapshot.peers.find(p => p.personality === personality)
|
|
581
632
|
if (!peer) return sendJson(res, 404, { error: `unknown peer "${personality}"` })
|
|
582
633
|
const events = readRecentEvents(files, 50, personality)
|
package/src/daemon/main.ts
CHANGED
|
@@ -70,6 +70,7 @@ import { buildFleetHandler } from './fleet.ts'
|
|
|
70
70
|
import { NoticeBoard } from './notices.ts'
|
|
71
71
|
import { startMuteWatch } from './mutewatch.ts'
|
|
72
72
|
import { startGoalWatch } from './goalwatch.ts'
|
|
73
|
+
import { ActivityBoard, startTurnWatch } from './turnwatch.ts'
|
|
73
74
|
import { defaultDaemonSocketPath, startDaemon, type DaemonHandle } from './index.ts'
|
|
74
75
|
|
|
75
76
|
/** Default TCP loopback port for the always-on router. Real http MCP clients
|
|
@@ -107,6 +108,12 @@ export const DEFAULT_MUTEWATCH_INTERVAL_MS = 20_000
|
|
|
107
108
|
* DB, and the threads join runs ONLY when a stalled row is actually present. */
|
|
108
109
|
export const DEFAULT_GOALWATCH_INTERVAL_MS = 20_000
|
|
109
110
|
|
|
111
|
+
/** Turn-watch cadence (docs/15 §Activity) — how often the daemon re-reads the turn markers of
|
|
112
|
+
* session files that CHANGED. Much faster than the notice watches because the fact is
|
|
113
|
+
* short-lived: a turn can be shorter than 20 s, so a 20 s sweep would report `idle` straight
|
|
114
|
+
* through one. This bounds the snapshot's idle↔working lag to roughly one interval. */
|
|
115
|
+
export const DEFAULT_TURNWATCH_INTERVAL_MS = 3_000
|
|
116
|
+
|
|
110
117
|
// This module's own path — the launchd plist runs `bun <this>` as the daemon.
|
|
111
118
|
const DAEMON_MAIN_PATH = fileURLToPath(import.meta.url)
|
|
112
119
|
|
|
@@ -339,6 +346,8 @@ export interface ConfiguredDaemonOptions {
|
|
|
339
346
|
muteWatchIntervalMs?: number
|
|
340
347
|
/** Goal-watch sweep cadence (default DEFAULT_GOALWATCH_INTERVAL_MS). */
|
|
341
348
|
goalWatchIntervalMs?: number
|
|
349
|
+
/** Turn-watch sweep cadence (default DEFAULT_TURNWATCH_INTERVAL_MS). */
|
|
350
|
+
turnWatchIntervalMs?: number
|
|
342
351
|
/** Write the router.json discovery file (default true for production). */
|
|
343
352
|
discovery?: boolean
|
|
344
353
|
rootDir?: string
|
|
@@ -392,6 +401,22 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
392
401
|
} catch { /* best-effort */ }
|
|
393
402
|
},
|
|
394
403
|
})
|
|
404
|
+
// Turn-activity board + watch (docs/15 §Activity). Constructed HERE for the same reason as
|
|
405
|
+
// the notice board: the watch WRITES it and the fleet handler READS it, and that sharing is
|
|
406
|
+
// the whole wiring. Faster cadence than the notice watches (3 s vs 20 s) because a turn is a
|
|
407
|
+
// short-lived fact — a 20 s sweep would miss whole turns. Affordable: the sweep stats only
|
|
408
|
+
// files CHANGED since the last pass (measured on this host: ~2 ms across both runtime trees).
|
|
409
|
+
const activity = new ActivityBoard()
|
|
410
|
+
const stopTurnWatch = startTurnWatch(cfg, activity, {
|
|
411
|
+
env,
|
|
412
|
+
intervalMs: opts.turnWatchIntervalMs ?? DEFAULT_TURNWATCH_INTERVAL_MS,
|
|
413
|
+
onError: (err: unknown) => {
|
|
414
|
+
try {
|
|
415
|
+
const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
416
|
+
appendLifecycleEvent(cfg.eventLogDir, { ev: 'turnwatch-error', error: detail.replace(/\s+/g, ' ').slice(0, 600) }, { env })
|
|
417
|
+
} catch { /* best-effort */ }
|
|
418
|
+
},
|
|
419
|
+
})
|
|
395
420
|
const handle = await startDaemon({
|
|
396
421
|
wake: makeWakeFn(cfg, env),
|
|
397
422
|
// M2 arm-on-outbound (see makeArmEphemeralOnDelivered): ephemeral caller's ok
|
|
@@ -418,7 +443,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
418
443
|
// (the same listPeers truth as `iapeer list`), SSE events (tail of the durable
|
|
419
444
|
// logs), commands over the existing verb functions. Same listeners, same bearer
|
|
420
445
|
// gate; advertised in router.json as fleet:1. Contract: docs/15-fleet-api.md.
|
|
421
|
-
fleet: buildFleetHandler({ env, board }),
|
|
446
|
+
fleet: buildFleetHandler({ env, board, activity }),
|
|
422
447
|
supervise: {
|
|
423
448
|
intervalMs: opts.superviseIntervalMs ?? DEFAULT_SUPERVISE_INTERVAL_MS,
|
|
424
449
|
// idle-reap / zombie-sweep, THEN the eager fresh re-launch for any peer whose
|
|
@@ -466,7 +491,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
466
491
|
// The mute-watch timer is OURS, not startDaemon's — so its teardown must hang off the
|
|
467
492
|
// handle we return, or a closed daemon would leave a live timer sweeping the disk (and
|
|
468
493
|
// every test that starts a daemon would hang on exit).
|
|
469
|
-
return { ...handle, close: async () => { stopMuteWatch(); stopGoalWatch(); await handle.close() } }
|
|
494
|
+
return { ...handle, close: async () => { stopMuteWatch(); stopGoalWatch(); stopTurnWatch(); await handle.close() } }
|
|
470
495
|
}
|
|
471
496
|
|
|
472
497
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Turn-activity durable log — the daemon's DURABLE trace of every per-runtime turn boundary
|
|
2
|
+
// it observes (docs/15 §Activity). Sibling to lifecycle.log / delivery.log / exits.log /
|
|
3
|
+
// approvals.log / notices.log under cfg.eventLogDir; tailed by the fleet-API SSE stream
|
|
4
|
+
// (fleetEventFiles), so `turn-started` / `turn-ended` reach every operator client with
|
|
5
|
+
// at-least-once semantics — the same "the log IS the push mechanism" trick notices.log uses.
|
|
6
|
+
//
|
|
7
|
+
// Volume is bounded by DESIGN, not by luck: turnWatchTick writes only on a real state CHANGE,
|
|
8
|
+
// never once per sweep. The watch polls every ~3 s; without that guard it would emit a line per
|
|
9
|
+
// peer per pass and drown the very log meant to make turn history readable. Two lines per turn
|
|
10
|
+
// per runtime is the ceiling.
|
|
11
|
+
//
|
|
12
|
+
// Same rotation class as its siblings: written by OUR code → self-rotates via the shared
|
|
13
|
+
// storage/rotatelog primitive. The dir is passed IN by the composition point, never re-resolved
|
|
14
|
+
// from env — a falsy dir → no-op (library/test daemons stay hermetic).
|
|
15
|
+
|
|
16
|
+
import { join } from 'path'
|
|
17
|
+
import { DEFAULT_LOG_KEEP, DEFAULT_LOG_MAX_BYTES, appendRotatedEvent } from '../storage/rotatelog.ts'
|
|
18
|
+
|
|
19
|
+
/** The turn-activity log inside `logDir` (sibling to notices.log). */
|
|
20
|
+
export function turnsLogPath(logDir: string): string {
|
|
21
|
+
return join(logDir, 'turns.log')
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
25
|
+
const n = parseInt(raw ?? '', 10)
|
|
26
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface AppendTurnOptions {
|
|
30
|
+
/** Reads the rotation knobs IAPEER_TURNS_LOG_MAX_BYTES / _KEEP. */
|
|
31
|
+
env?: NodeJS.ProcessEnv
|
|
32
|
+
/** Stamp the line with this epoch-ms. Default Date.now(). */
|
|
33
|
+
nowMs?: number
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Append one turn event line into `logDir`/turns.log. A falsy `logDir` is a no-op. Fully
|
|
38
|
+
* best-effort — never throws; logging must never fail a detection sweep.
|
|
39
|
+
*/
|
|
40
|
+
export function appendTurnEvent(
|
|
41
|
+
logDir: string | undefined,
|
|
42
|
+
fields: Record<string, string | number | undefined>,
|
|
43
|
+
opts: AppendTurnOptions = {},
|
|
44
|
+
): void {
|
|
45
|
+
if (!logDir) return
|
|
46
|
+
const env = opts.env ?? process.env
|
|
47
|
+
appendRotatedEvent(turnsLogPath(logDir), fields, {
|
|
48
|
+
nowMs: opts.nowMs,
|
|
49
|
+
maxBytes: envPosInt(env.IAPEER_TURNS_LOG_MAX_BYTES, DEFAULT_LOG_MAX_BYTES),
|
|
50
|
+
keep: envPosInt(env.IAPEER_TURNS_LOG_KEEP, DEFAULT_LOG_KEEP),
|
|
51
|
+
})
|
|
52
|
+
}
|
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
// Turn-watch — per-runtime TURN ACTIVITY for the fleet surface (docs/15 §Activity).
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS. The snapshot answered exactly one question about a session: is the process
|
|
4
|
+
// alive (`status: live | asleep | stopped`). That is LIVENESS, and it is not the operator's
|
|
5
|
+
// question. A peer sitting idle with an unfinished job and a peer grinding through a turn are
|
|
6
|
+
// both `live` — indistinguishable. The owner spotted a stalled peer before the PM twice
|
|
7
|
+
// because the surface could not tell "working" from "merely alive".
|
|
8
|
+
//
|
|
9
|
+
// ACTIVITY IS NOT LIVENESS, AND THEY STAY SEPARATE. `status` keeps its exact meaning; the new
|
|
10
|
+
// `activity` field is additive and rides ALONGSIDE it on the same per-runtime object. A client
|
|
11
|
+
// that ignores `activity` is unaffected. Activity is reported ONLY for a `live` runtime — for
|
|
12
|
+
// an asleep/stopped session there is no turn to be in, and the field is OMITTED rather than
|
|
13
|
+
// filled with a fake `idle`: absent means "no claim", which a dead session deserves.
|
|
14
|
+
//
|
|
15
|
+
// THE EVIDENCE IS THE RUNTIME'S OWN TURN MARKER — mechanical, never a CPU/mtime reconstruction:
|
|
16
|
+
//
|
|
17
|
+
// • codex → the rollout's `event_msg` payloads `task_started` / `task_complete`. Explicit
|
|
18
|
+
// turn boundaries; the last one wins. Measured across 3 days of real rollouts: 156 started
|
|
19
|
+
// / 147 complete — the 9-line gap is precisely the dangling-turn risk the stale bound below
|
|
20
|
+
// exists for. Codex emits `task_complete` even for a turn that ended on an error (measured:
|
|
21
|
+
// the 16.07 goal-blocking turn ends token_count → task_complete, no error event), so the
|
|
22
|
+
// terminal error path needs no separate marker.
|
|
23
|
+
//
|
|
24
|
+
// • claude → the transcript's assistant `stop_reason`. `tool_use` means the turn CONTINUES;
|
|
25
|
+
// every other terminal reason means it ended. Measured over every transcript this peer has:
|
|
26
|
+
// tool_use 18317, end_turn 663, stop_sequence 16, null 4. So the rule is deliberately
|
|
27
|
+
// ASYMMETRIC — only `tool_use` reads as working, and anything else (end_turn, stop_sequence,
|
|
28
|
+
// max_tokens, refusal, an unknown future value, null) reads as IDLE. An unknown reason must
|
|
29
|
+
// never invent work; the polarity always fails toward idle.
|
|
30
|
+
//
|
|
31
|
+
// THE THREE FALSE-WORKING TRAPS, each measured on real transcripts rather than assumed:
|
|
32
|
+
//
|
|
33
|
+
// 1. **A `user` line means working** (the model is about to answer, or a tool_result just
|
|
34
|
+
// landed) — but claude writes NON-model user lines too. A compaction summary is
|
|
35
|
+
// `type:"user"` with `isCompactSummary:true` (12 found here), and a `/compact` run while
|
|
36
|
+
// IDLE would leave one as the last line → a permanent phantom "working". Excluded. The
|
|
37
|
+
// `compact_boundary` line is `type:"system"` and never qualifies anyway. Excluding the
|
|
38
|
+
// summary is also CORRECT for mid-turn compaction: the prior assistant `tool_use` line
|
|
39
|
+
// then decides, and the turn is genuinely still running.
|
|
40
|
+
// 2. **Sub-agent lines.** A Task sidechain writes assistant/user lines into the SAME
|
|
41
|
+
// transcript with `isSidechain:true`. They belong to a different chain and must not decide
|
|
42
|
+
// the peer's turn state. Excluded.
|
|
43
|
+
// 3. **Injected meta lines** (`isMeta:true`, 13 found here) are not the model being asked
|
|
44
|
+
// anything. Excluded.
|
|
45
|
+
//
|
|
46
|
+
// THE STALE BOUND. A `working` claim rests on evidence that stops being written the moment a
|
|
47
|
+
// session hangs mid-turn — the transcript simply goes quiet. Liveness already removes the dead
|
|
48
|
+
// session (activity is live-only), so what remains is a LIVE session whose last recorded act
|
|
49
|
+
// was starting work and which has since written nothing. Past TURN_STALE_MS the claim decays to
|
|
50
|
+
// `unknown` rather than standing as `working` forever: unknown is the honest word for "we saw
|
|
51
|
+
// it start and never saw it finish". `idle` never decays — a session that ended its turn stays
|
|
52
|
+
// idle until something pokes it, and age proves nothing against that.
|
|
53
|
+
//
|
|
54
|
+
// Deliberately NOT the pane-log mtime, which looks like a freshness clock and is not: measured
|
|
55
|
+
// live 16.07, an IDLE claude pane-log advances every second (the statusline repaints) while an
|
|
56
|
+
// IDLE codex pane-log freezes (39s→46s). The signal means opposite things per runtime; one
|
|
57
|
+
// clock that lies differently on each runtime is worse than no clock.
|
|
58
|
+
|
|
59
|
+
import { existsSync, openSync, closeSync, fstatSync, readSync, readdirSync, statSync } from 'node:fs'
|
|
60
|
+
import { join } from 'node:path'
|
|
61
|
+
import type { LifecycleConfig } from '../lifecycle/index.ts'
|
|
62
|
+
import { appendTurnEvent } from './turnslog.ts'
|
|
63
|
+
|
|
64
|
+
/** Tail parsed per session file. The turn marker is always at the very end. */
|
|
65
|
+
const TAIL_BYTES = 128 * 1024
|
|
66
|
+
/** Head slice for codex's `session_meta` (line 1 — a long rollout pushes it out of the tail). */
|
|
67
|
+
const HEAD_BYTES = 64 * 1024
|
|
68
|
+
/** Walk depth for the codex YYYY/MM/DD tree. */
|
|
69
|
+
const CODEX_WALK_DEPTH = 3
|
|
70
|
+
/** How far back the FIRST sweep looks, to seed state for sessions already running when the
|
|
71
|
+
* daemon started. Bounded on purpose: a full-history pass would read every transcript ever.
|
|
72
|
+
* Measured cost of a 48h seed on this host: ~2 ms of walk, 97 files. */
|
|
73
|
+
const SEED_LOOKBACK_MS = 48 * 3_600_000
|
|
74
|
+
/** Overlap on the since-boundary so a write landing between sweeps is never missed. */
|
|
75
|
+
const SCAN_OVERLAP_MS = 2_000
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* How long a `working` claim outlives its last evidence before decaying to `unknown`.
|
|
79
|
+
* Generous on purpose: a turn legitimately writes NOTHING while one long tool runs (a deploy
|
|
80
|
+
* here takes minutes), and a false `unknown` on a genuinely working peer is a worse lie than a
|
|
81
|
+
* late one. Ten minutes is far past any tool this fleet runs and still bounds the phantom.
|
|
82
|
+
*/
|
|
83
|
+
export const TURN_STALE_MS = 10 * 60_000
|
|
84
|
+
|
|
85
|
+
/** The operator-facing verdict. `unknown` = live, but the evidence cannot support a claim. */
|
|
86
|
+
export type TurnActivity = 'working' | 'idle' | 'unknown'
|
|
87
|
+
|
|
88
|
+
export interface TurnState {
|
|
89
|
+
state: 'working' | 'idle'
|
|
90
|
+
/** The EVIDENCE's own timestamp (the transcript line's), never our sweep clock. */
|
|
91
|
+
atMs: number
|
|
92
|
+
cwd: string
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
96
|
+
// Pure parsers — text in, state out (fs-free: tests need no session tree)
|
|
97
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
98
|
+
|
|
99
|
+
function toMs(ts: unknown): number | null {
|
|
100
|
+
if (typeof ts !== 'string') return null
|
|
101
|
+
const ms = Date.parse(ts)
|
|
102
|
+
return Number.isFinite(ms) ? ms : null
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The claude transcript's turn state, or null if the file states nothing usable.
|
|
107
|
+
*
|
|
108
|
+
* Only `tool_use` continues a turn; every other stop_reason ends it (see the header on why the
|
|
109
|
+
* polarity is asymmetric). Compaction summaries, sidechain lines and meta lines are excluded —
|
|
110
|
+
* each is a measured false-working vector.
|
|
111
|
+
*/
|
|
112
|
+
export function parseClaudeTurn(text: string): TurnState | null {
|
|
113
|
+
let found: TurnState | null = null
|
|
114
|
+
for (const line of text.split('\n')) {
|
|
115
|
+
const s = line.trim()
|
|
116
|
+
if (!s.startsWith('{')) continue // a partial first line from the tail cut
|
|
117
|
+
let d: Record<string, unknown>
|
|
118
|
+
try {
|
|
119
|
+
d = JSON.parse(s) as Record<string, unknown>
|
|
120
|
+
} catch {
|
|
121
|
+
continue // never fail a sweep over one garbled line
|
|
122
|
+
}
|
|
123
|
+
const type = d.type
|
|
124
|
+
if (type !== 'assistant' && type !== 'user') continue
|
|
125
|
+
if (d.isSidechain === true) continue // a sub-agent's chain, not the peer's turn
|
|
126
|
+
if (d.isCompactSummary === true) continue // compaction artifact, not the model being asked
|
|
127
|
+
if (d.isMeta === true) continue // injected, not a real prompt
|
|
128
|
+
const atMs = toMs(d.timestamp)
|
|
129
|
+
if (atMs === null) continue
|
|
130
|
+
const cwd = typeof d.cwd === 'string' ? d.cwd : ''
|
|
131
|
+
if (!cwd) continue // unattributable to a peer → ignore rather than guess
|
|
132
|
+
let state: 'working' | 'idle'
|
|
133
|
+
if (type === 'assistant') {
|
|
134
|
+
const msg = d.message as { stop_reason?: unknown } | undefined
|
|
135
|
+
// ONLY tool_use continues. end_turn / stop_sequence / max_tokens / refusal / an unknown
|
|
136
|
+
// future value / null all mean the assistant stopped.
|
|
137
|
+
state = msg?.stop_reason === 'tool_use' ? 'working' : 'idle'
|
|
138
|
+
} else {
|
|
139
|
+
// A real user line: a prompt to answer or a tool_result to consume — either way the
|
|
140
|
+
// model is on the hook. (The non-model user lines were filtered above.)
|
|
141
|
+
state = 'working'
|
|
142
|
+
}
|
|
143
|
+
found = { state, atMs, cwd }
|
|
144
|
+
}
|
|
145
|
+
return found
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The codex rollout's turn state, or null. `task_started` / `task_complete` are explicit
|
|
150
|
+
* boundaries — the last one wins. `cwd` comes from `session_meta` (line 1), so pass the file
|
|
151
|
+
* HEAD when the tail may have cut it away.
|
|
152
|
+
*/
|
|
153
|
+
export function parseCodexTurn(text: string, head = ''): TurnState | null {
|
|
154
|
+
let cwd = ''
|
|
155
|
+
const scanMeta = (src: string): void => {
|
|
156
|
+
for (const line of src.split('\n')) {
|
|
157
|
+
const s = line.trim()
|
|
158
|
+
if (!s.startsWith('{') || !s.includes('session_meta')) continue
|
|
159
|
+
try {
|
|
160
|
+
const d = JSON.parse(s) as { type?: string; payload?: { cwd?: unknown } }
|
|
161
|
+
if (d.type !== 'session_meta' || !d.payload) continue
|
|
162
|
+
if (typeof d.payload.cwd === 'string') cwd = d.payload.cwd
|
|
163
|
+
} catch {
|
|
164
|
+
/* ignore */
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
scanMeta(head)
|
|
169
|
+
if (!cwd) scanMeta(text)
|
|
170
|
+
if (!cwd) return null // unattributable to a peer
|
|
171
|
+
|
|
172
|
+
let found: TurnState | null = null
|
|
173
|
+
for (const line of text.split('\n')) {
|
|
174
|
+
const s = line.trim()
|
|
175
|
+
if (!s.startsWith('{') || !s.includes('task_')) continue
|
|
176
|
+
let d: { timestamp?: unknown; type?: unknown; payload?: { type?: unknown } }
|
|
177
|
+
try {
|
|
178
|
+
d = JSON.parse(s)
|
|
179
|
+
} catch {
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
if (d.type !== 'event_msg') continue
|
|
183
|
+
const p = d.payload?.type
|
|
184
|
+
if (p !== 'task_started' && p !== 'task_complete') continue
|
|
185
|
+
const atMs = toMs(d.timestamp)
|
|
186
|
+
if (atMs === null) continue
|
|
187
|
+
found = { state: p === 'task_started' ? 'working' : 'idle', atMs, cwd }
|
|
188
|
+
}
|
|
189
|
+
return found
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
193
|
+
// The board — in-memory (personality, runtime) → last known turn state
|
|
194
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
195
|
+
|
|
196
|
+
function key(personality: string, runtime: string): string {
|
|
197
|
+
return `${runtime}-${personality}` // the identity scheme (docs/03)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Last known turn state per (personality, runtime). In-memory and ephemeral, exactly like the
|
|
202
|
+
* notice board: a restart re-seeds from the session files, which are the durable truth.
|
|
203
|
+
*/
|
|
204
|
+
export class ActivityBoard {
|
|
205
|
+
private readonly states = new Map<string, TurnState>()
|
|
206
|
+
|
|
207
|
+
/** Record an observation. Returns the PREVIOUS state, so the caller can log only real
|
|
208
|
+
* transitions rather than one line per sweep. */
|
|
209
|
+
set(personality: string, runtime: string, next: TurnState): TurnState | undefined {
|
|
210
|
+
const k = key(personality, runtime)
|
|
211
|
+
const prev = this.states.get(k)
|
|
212
|
+
// Evidence older than what we already hold is a stale re-read — never let it rewind state.
|
|
213
|
+
if (prev && next.atMs < prev.atMs) return prev
|
|
214
|
+
this.states.set(k, next)
|
|
215
|
+
return prev
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** The operator-facing verdict, with the stale bound applied at READ time (so a claim decays
|
|
219
|
+
* even while nothing new is swept). `undefined` → we have no evidence at all. */
|
|
220
|
+
get(personality: string, runtime: string, nowMs: number): TurnActivity | undefined {
|
|
221
|
+
const st = this.states.get(key(personality, runtime))
|
|
222
|
+
if (!st) return undefined
|
|
223
|
+
if (st.state === 'idle') return 'idle' // idle never decays — age proves nothing against it
|
|
224
|
+
return nowMs - st.atMs > TURN_STALE_MS ? 'unknown' : 'working'
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** The raw record (peer-detail / tests). */
|
|
228
|
+
raw(personality: string, runtime: string): TurnState | undefined {
|
|
229
|
+
return this.states.get(key(personality, runtime))
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
size(): number {
|
|
233
|
+
return this.states.size
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
238
|
+
// The sweep
|
|
239
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
240
|
+
|
|
241
|
+
function readTail(path: string): { tail: string; head: string } | null {
|
|
242
|
+
try {
|
|
243
|
+
const fd = openSync(path, 'r')
|
|
244
|
+
try {
|
|
245
|
+
const size = fstatSync(fd).size
|
|
246
|
+
if (size <= 0) return null
|
|
247
|
+
const off = Math.max(0, size - TAIL_BYTES)
|
|
248
|
+
const buf = Buffer.alloc(size - off)
|
|
249
|
+
readSync(fd, buf, 0, buf.length, off)
|
|
250
|
+
let head = ''
|
|
251
|
+
if (off > 0) {
|
|
252
|
+
const hb = Buffer.alloc(Math.min(HEAD_BYTES, off))
|
|
253
|
+
readSync(fd, hb, 0, hb.length, 0)
|
|
254
|
+
head = hb.toString('utf8')
|
|
255
|
+
}
|
|
256
|
+
return { tail: buf.toString('utf8'), head }
|
|
257
|
+
} finally {
|
|
258
|
+
closeSync(fd)
|
|
259
|
+
}
|
|
260
|
+
} catch {
|
|
261
|
+
return null
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function safeReaddir(dir: string): string[] {
|
|
266
|
+
try {
|
|
267
|
+
return readdirSync(dir)
|
|
268
|
+
} catch {
|
|
269
|
+
return []
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** *.jsonl under `root` (depth-bounded) whose mtime is at/after `sinceMs`. */
|
|
274
|
+
function changedJsonl(root: string, sinceMs: number, depth: number): string[] {
|
|
275
|
+
if (!existsSync(root)) return []
|
|
276
|
+
const out: string[] = []
|
|
277
|
+
const walk = (dir: string, left: number): void => {
|
|
278
|
+
for (const name of safeReaddir(dir)) {
|
|
279
|
+
const p = join(dir, name)
|
|
280
|
+
if (name.endsWith('.jsonl')) {
|
|
281
|
+
let m: number | null = null
|
|
282
|
+
try {
|
|
283
|
+
m = statSync(p).mtimeMs
|
|
284
|
+
} catch {
|
|
285
|
+
continue
|
|
286
|
+
}
|
|
287
|
+
if (m >= sinceMs) out.push(p)
|
|
288
|
+
continue
|
|
289
|
+
}
|
|
290
|
+
if (left <= 0 || name.startsWith('.')) continue
|
|
291
|
+
try {
|
|
292
|
+
if (statSync(p).isDirectory()) walk(p, left - 1)
|
|
293
|
+
} catch {
|
|
294
|
+
continue
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
walk(root, depth)
|
|
299
|
+
return out
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export interface TurnWatchPaths {
|
|
303
|
+
/** ~/.claude/projects — INJECTED, never re-resolved from env inside a sweep. */
|
|
304
|
+
claudeProjectsDir: string
|
|
305
|
+
/** ~/.codex/sessions — same rule. */
|
|
306
|
+
codexSessionsDir: string
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export function resolveTurnWatchPaths(env: NodeJS.ProcessEnv): TurnWatchPaths {
|
|
310
|
+
const home = env.HOME?.trim() || ''
|
|
311
|
+
return {
|
|
312
|
+
claudeProjectsDir: env.IAPEER_CLAUDE_PROJECTS_DIR?.trim() || join(home, '.claude', 'projects'),
|
|
313
|
+
codexSessionsDir: env.IAPEER_CODEX_SESSIONS_DIR?.trim() || join(home, '.codex', 'sessions'),
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export interface TurnScanDeps extends TurnWatchPaths {
|
|
318
|
+
/** personality ← cwd. Built from the registry by the composition point. */
|
|
319
|
+
peerByCwd: (cwd: string) => string | undefined
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
export interface TurnObservation extends TurnState {
|
|
323
|
+
personality: string
|
|
324
|
+
runtime: 'claude' | 'codex'
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** One sweep: the turn state of every peer session file touched since `sinceMs`. */
|
|
328
|
+
export function scanTurnStates(deps: TurnScanDeps, sinceMs: number): TurnObservation[] {
|
|
329
|
+
const floor = sinceMs - SCAN_OVERLAP_MS
|
|
330
|
+
const out: TurnObservation[] = []
|
|
331
|
+
const push = (st: TurnState | null, runtime: 'claude' | 'codex'): void => {
|
|
332
|
+
if (!st) return
|
|
333
|
+
const personality = deps.peerByCwd(st.cwd)
|
|
334
|
+
if (!personality) return // a human's own session — never reported
|
|
335
|
+
out.push({ ...st, personality, runtime })
|
|
336
|
+
}
|
|
337
|
+
for (const file of changedJsonl(deps.claudeProjectsDir, floor, 1)) {
|
|
338
|
+
const t = readTail(file)
|
|
339
|
+
if (t) push(parseClaudeTurn(t.tail), 'claude')
|
|
340
|
+
}
|
|
341
|
+
for (const file of changedJsonl(deps.codexSessionsDir, floor, CODEX_WALK_DEPTH)) {
|
|
342
|
+
const t = readTail(file)
|
|
343
|
+
if (t) push(parseCodexTurn(t.tail, t.head), 'codex')
|
|
344
|
+
}
|
|
345
|
+
return out
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export interface TurnWatchDeps extends TurnScanDeps {
|
|
349
|
+
board: ActivityBoard
|
|
350
|
+
/** cfg.eventLogDir — turns.log (falsy → no durable log, hermetic). */
|
|
351
|
+
logDir?: string
|
|
352
|
+
env?: NodeJS.ProcessEnv
|
|
353
|
+
now?: () => number
|
|
354
|
+
onError?: (err: unknown) => void
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Run one watch tick: sweep, fold into the board, log REAL transitions only.
|
|
359
|
+
* Returns how many transitions were logged plus the new since-boundary.
|
|
360
|
+
*/
|
|
361
|
+
export function turnWatchTick(deps: TurnWatchDeps, sinceMs: number): { transitions: number; sinceMs: number } {
|
|
362
|
+
const now = deps.now ?? Date.now
|
|
363
|
+
const startedMs = now()
|
|
364
|
+
let transitions = 0
|
|
365
|
+
try {
|
|
366
|
+
for (const o of scanTurnStates(deps, sinceMs)) {
|
|
367
|
+
const prev = deps.board.set(o.personality, o.runtime, { state: o.state, atMs: o.atMs, cwd: o.cwd })
|
|
368
|
+
// Only a genuine change is an event. Without this the 3 s sweep would write a line per
|
|
369
|
+
// pass per peer and bury the log it is supposed to make readable.
|
|
370
|
+
if (prev?.state === o.state) continue
|
|
371
|
+
// A rewound observation (older evidence than held) is not a transition either.
|
|
372
|
+
if (prev && o.atMs < prev.atMs) continue
|
|
373
|
+
transitions += 1
|
|
374
|
+
appendTurnEvent(
|
|
375
|
+
deps.logDir,
|
|
376
|
+
{
|
|
377
|
+
ev: o.state === 'working' ? 'turn-started' : 'turn-ended',
|
|
378
|
+
personality: o.personality,
|
|
379
|
+
runtime: o.runtime,
|
|
380
|
+
from: prev?.state,
|
|
381
|
+
at: o.atMs,
|
|
382
|
+
at_iso: new Date(o.atMs).toISOString(),
|
|
383
|
+
},
|
|
384
|
+
{ env: deps.env, nowMs: startedMs },
|
|
385
|
+
)
|
|
386
|
+
}
|
|
387
|
+
} catch (e) {
|
|
388
|
+
deps.onError?.(e)
|
|
389
|
+
}
|
|
390
|
+
return { transitions, sinceMs: startedMs }
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export interface StartTurnWatchOptions {
|
|
394
|
+
env?: NodeJS.ProcessEnv
|
|
395
|
+
intervalMs?: number
|
|
396
|
+
paths?: TurnWatchPaths
|
|
397
|
+
listPeers?: () => Array<{ personality: string; cwd: string }>
|
|
398
|
+
now?: () => number
|
|
399
|
+
onError?: (err: unknown) => void
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* Start the turn-watch timer. Returns its stop function — the caller OWNS teardown.
|
|
404
|
+
* The FIRST sweep reaches back SEED_LOOKBACK_MS so sessions already mid-turn when the daemon
|
|
405
|
+
* started are known immediately; every later sweep is incremental (changed files only).
|
|
406
|
+
*/
|
|
407
|
+
export function startTurnWatch(cfg: LifecycleConfig, board: ActivityBoard, opts: StartTurnWatchOptions = {}): () => void {
|
|
408
|
+
const env = opts.env ?? process.env
|
|
409
|
+
const paths = opts.paths ?? resolveTurnWatchPaths(env)
|
|
410
|
+
const now = opts.now ?? Date.now
|
|
411
|
+
let listPeers = opts.listPeers
|
|
412
|
+
let since = now() - SEED_LOOKBACK_MS
|
|
413
|
+
let running = false
|
|
414
|
+
|
|
415
|
+
const sweep = async (): Promise<void> => {
|
|
416
|
+
if (running) return // a slow sweep must never overlap itself into a stampede
|
|
417
|
+
running = true
|
|
418
|
+
try {
|
|
419
|
+
if (!listPeers) {
|
|
420
|
+
const cli = await import('../cli/index.ts')
|
|
421
|
+
listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
|
|
422
|
+
}
|
|
423
|
+
const rows = listPeers()
|
|
424
|
+
// Case-insensitive, trailing-slash-tolerant — same rule and reason as mutewatch.
|
|
425
|
+
const byCwd = new Map<string, string>()
|
|
426
|
+
for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
|
|
427
|
+
since = turnWatchTick(
|
|
428
|
+
{
|
|
429
|
+
...paths,
|
|
430
|
+
board,
|
|
431
|
+
logDir: cfg.eventLogDir,
|
|
432
|
+
env,
|
|
433
|
+
now,
|
|
434
|
+
onError: opts.onError,
|
|
435
|
+
peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
|
|
436
|
+
},
|
|
437
|
+
since,
|
|
438
|
+
).sinceMs
|
|
439
|
+
} catch (e) {
|
|
440
|
+
opts.onError?.(e)
|
|
441
|
+
} finally {
|
|
442
|
+
running = false
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
void sweep() // seed immediately — do not make the operator wait one interval for the truth
|
|
447
|
+
const timer = setInterval(() => void sweep(), opts.intervalMs ?? 3_000)
|
|
448
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
449
|
+
return () => clearInterval(timer)
|
|
450
|
+
}
|