@agfpd/iapeer 0.4.96 → 0.4.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/15-fleet-api.md +62 -2
- package/docs/19-notices.md +78 -8
- package/package.json +1 -1
- package/src/daemon/fleet.ts +54 -3
- package/src/daemon/goalwatch.ts +385 -0
- package/src/daemon/main.ts +50 -2
- package/src/daemon/turnslog.ts +52 -0
- package/src/daemon/turnwatch.ts +450 -0
package/docs/15-fleet-api.md
CHANGED
|
@@ -48,7 +48,7 @@ The full fleet state. The peer rows are produced by the **same in-process functi
|
|
|
48
48
|
"default_runtime": "claude",
|
|
49
49
|
"cwd": "/Users/me/.iapeer/peers/boris",
|
|
50
50
|
"runtimes": [
|
|
51
|
-
{ "runtime": "claude", "status": "live", "attached": true },
|
|
51
|
+
{ "runtime": "claude", "status": "live", "attached": true, "activity": "working" },
|
|
52
52
|
{ "runtime": "codex", "status": "asleep" }
|
|
53
53
|
],
|
|
54
54
|
"last_active_runtime": "claude",
|
|
@@ -78,6 +78,7 @@ Field notes:
|
|
|
78
78
|
|
|
79
79
|
- `runtimes[].status` — `live` (session up) · `asleep` (wakeable on demand) · `stopped` (operator stop flag; the daemon won't wake it). Same glyph semantics as the CLI: `●` / `○` / `✕`.
|
|
80
80
|
- `runtimes[].attached` — present (`true`) only when a **human operator** is attached to that live hosted session.
|
|
81
|
+
- `runtimes[].activity` — **turn activity**: is this runtime *in a turn* right now. See «Activity» below. Additive; absent on a pre-activity daemon, on every non-`live` runtime, and whenever the daemon has no evidence — clients MUST treat absence as **no claim**, never as `idle`.
|
|
81
82
|
- `attached` — any runtime attached (peer-level convenience).
|
|
82
83
|
- `launchd_managed` — launchd owns this peer's lifecycle (H4): the daemon never wakes/reaps it, and wake/stop/start commands are guarded accordingly.
|
|
83
84
|
- `wake_policy` — `warm` (default: wake-on-message, idle-reap, resume) or `ephemeral` (stateless worker: serial task queue, die-after-reply).
|
|
@@ -90,6 +91,65 @@ Field notes:
|
|
|
90
91
|
- `approvals` (top-level, docs/17-approval) — the pending human-approval queue, the SAME items `GET /fleet/v1/approvals` returns, carried in the snapshot so a client that already re-fetches `/snapshot` on every event renders the queue with no extra call. **ADDITIVE + OMITTED when the queue is empty** — a client MUST treat its absence as an empty queue (the same rule as `approval_mode`). Each item: `id` (broker id, the target of `POST /fleet/v1/approvals/<id>/(approve|deny)`), `personality` + `runtime` (whose action), `kind` (`tool`|`plan`|`question`|`circuit-breaker`), `tool` (the tool / breaker name), `summary` (a one-line badge string), `content` (the FULL verbatim action — command / diff / plan; criterion #7), `createdMs` / `expiresMs`. The broker's item is a superset — unknown extra fields (`title`, `approvers`) MUST be ignored (obligation 2).
|
|
91
92
|
- `notices` (top-level, docs/19-notices) — live OWNER-facing notices: things the daemon noticed that the peer itself could not report, the v1 case being a structural API error that left it MUTE (`kind: peer-mute`). The SAME items `GET /fleet/v1/notices` returns. **ADDITIVE + OMITTED when the board is empty** — absence ⇒ empty board (the same rule as `approvals`). **READ-ONLY**: a notice is one-way, there is nothing to approve or deny. Each item: `id`, `personality` + `runtime` (who went mute), `kind` (`peer-mute`; treat unknown kinds as renderable), `errorType` (the RUNTIME's own value — `rate_limit`, `overloaded`, `rate_limit_reached`, … — free-form, do not switch exhaustively), `model?`, `resetsAtMs?`, `summary`, `content` (verbatim for claude), `sessionId?`, `createdMs` / `lastMs` / `expiresMs`, `count` (detections folded in — render as `×N`). **`model` and `resetsAtMs` are ABSENT when the runtime did not state them** — absence means *the runtime did not say*, NOT *there is no limit*; a client MUST render unknown and MUST NOT substitute the 5h/7d reset, which belongs to a different limit (docs/19 §3).
|
|
92
93
|
|
|
94
|
+
## Activity — is this runtime in a turn?
|
|
95
|
+
|
|
96
|
+
`runtimes[].activity` answers the operator's actual question, which `status` never did: a peer
|
|
97
|
+
idling with an unfinished job and a peer grinding through a turn are BOTH `live`.
|
|
98
|
+
|
|
99
|
+
**Activity is not liveness, and the two never merge.** `status` keeps its exact prior meaning.
|
|
100
|
+
`activity` is an additive sibling on the same per-runtime object.
|
|
101
|
+
|
|
102
|
+
| value | meaning |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `working` | a turn is in flight on this runtime |
|
|
105
|
+
| `idle` | live, but not in a turn |
|
|
106
|
+
| `unknown` | live; the turn was seen to start, never seen to finish, and the evidence has gone stale |
|
|
107
|
+
| *absent* | **no claim** — see below |
|
|
108
|
+
|
|
109
|
+
`activity` is **ABSENT** (never `idle`) when any of these hold, and a client MUST NOT infer
|
|
110
|
+
idleness from absence:
|
|
111
|
+
|
|
112
|
+
- the daemon predates this field (obligation 1);
|
|
113
|
+
- `status !== 'live'` — an asleep/stopped session has no turn to be in;
|
|
114
|
+
- the daemon has no evidence for that live runtime yet.
|
|
115
|
+
|
|
116
|
+
### Where the value comes from
|
|
117
|
+
|
|
118
|
+
The **runtime's own turn marker**, read from its session file. Never CPU, never a pane-log
|
|
119
|
+
mtime, never a reconstruction:
|
|
120
|
+
|
|
121
|
+
- **codex** — the rollout's `event_msg` payloads `task_started` → `working`, `task_complete` →
|
|
122
|
+
`idle`; last one wins. Codex emits `task_complete` for a turn that ends on an error too, so
|
|
123
|
+
the terminal-error path needs no separate marker.
|
|
124
|
+
- **claude** — the transcript's assistant `stop_reason`. **Only `tool_use` means the turn
|
|
125
|
+
continues**; every other terminal reason (`end_turn`, `stop_sequence`, `max_tokens`, a future
|
|
126
|
+
value, `null`) means it ended. The polarity is deliberately asymmetric: an unrecognised reason
|
|
127
|
+
must never invent work. A genuine `user` line also reads `working` (a prompt to answer or a
|
|
128
|
+
tool_result to consume), but compaction summaries (`isCompactSummary`), sub-agent sidechains
|
|
129
|
+
(`isSidechain`) and injected meta lines (`isMeta`) are excluded — each is a measured
|
|
130
|
+
false-`working` vector, the worst being a `/compact` run while idle leaving a phantom
|
|
131
|
+
`working` forever.
|
|
132
|
+
|
|
133
|
+
### Bounds a client can rely on
|
|
134
|
+
|
|
135
|
+
- **Lag** — the daemon re-reads changed session files every **~3 s**, so an `idle`↔`working`
|
|
136
|
+
flip appears within roughly one interval plus fs latency. A turn shorter than the interval may
|
|
137
|
+
never be observed at all; activity is a *state* to render, not an event ledger to count.
|
|
138
|
+
- **Stale bound** — a `working` claim whose evidence is older than **10 min** decays to
|
|
139
|
+
`unknown`. `idle` NEVER decays: a session that ended its turn stays idle until something pokes
|
|
140
|
+
it, and age proves nothing against that.
|
|
141
|
+
- **Restart** — the board is in-memory and re-seeds from the session files (a 48 h look-back) on
|
|
142
|
+
daemon start, so a restart cannot preserve a phantom `working`; a stale one decays by the rule
|
|
143
|
+
above, and a dead session drops out via `status` first.
|
|
144
|
+
|
|
145
|
+
### Events
|
|
146
|
+
|
|
147
|
+
`turns.log` → the SSE stream, `src: "turns"`, `ev`: **`turn-started`** / **`turn-ended`**, with
|
|
148
|
+
`personality`, `runtime`, `from` (the previous state, absent on first observation), `at` +
|
|
149
|
+
`at_iso` (the MARKER's own timestamp, not the sweep's). Emitted only on a real state **change** —
|
|
150
|
+
two lines per turn per runtime, not one per sweep. Treat the snapshot as the source of truth and
|
|
151
|
+
these as a push hint (obligation: the stream is at-least-once).
|
|
152
|
+
|
|
93
153
|
## GET /fleet/v1/peers/<personality>
|
|
94
154
|
|
|
95
155
|
One peer's card: the same object as the snapshot row, plus its recent history.
|
|
@@ -122,7 +182,7 @@ GET /fleet/v1/events?replay=50
|
|
|
122
182
|
- A comment `: connected` marks the transition from replay to live; `: hb` comments every ~15 s keep the connection alive.
|
|
123
183
|
- Latency: live events surface within ~0.5 s (log-tail poll).
|
|
124
184
|
|
|
125
|
-
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently).
|
|
185
|
+
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently), `turn-started` / `turn-ended` (from `turns.log`, `src: turns` — a runtime entered/left a turn; see «Activity». One line per real state CHANGE, never one per sweep).
|
|
126
186
|
|
|
127
187
|
What the headline states look like on the stream: a peer **waking** is `ev=wake`; a peer **parked to sleep** is `ev=supervise action=reaped-idle`; an operator **stop/start** is `ev=stopped` / `ev=started`; a **death** is `ev=session-exit` (immediately, from the supervisor) followed by the daemon's classification on its next pass.
|
|
128
188
|
|
package/docs/19-notices.md
CHANGED
|
@@ -29,7 +29,8 @@ renders approvals renders notices with the same plumbing.
|
|
|
29
29
|
|
|
30
30
|
## 2. The signal: `peer-mute`
|
|
31
31
|
|
|
32
|
-
The
|
|
32
|
+
The first notice kind (§2a adds `peer-goal-stalled`). It means: *a structural API error left
|
|
33
|
+
this peer unable to answer.*
|
|
33
34
|
|
|
34
35
|
The daemon does not infer this from silence — silence is unfalsifiable. It reads the fact
|
|
35
36
|
**structurally** out of the runtime's own session file. Neither runtime is scraped from the
|
|
@@ -66,6 +67,63 @@ detector keys on **non-null**, never on the variant strings (`rate_limit_reached
|
|
|
66
67
|
`workspace_owner_usage_limit_reached`, `workspace_member_credits_depleted`, … — read out of
|
|
67
68
|
the 0.144.1 binary), so a variant we have never seen still detects and reports itself.
|
|
68
69
|
|
|
70
|
+
## 2a. The signal: `peer-goal-stalled` (codex only)
|
|
71
|
+
|
|
72
|
+
The second notice kind. It means: *this peer's pinned objective stopped being worked, and
|
|
73
|
+
will not resume on its own.* Same class as `peer-mute` — peer alive, every health signal
|
|
74
|
+
green, nobody told — different evidence and a different cause.
|
|
75
|
+
|
|
76
|
+
**What the harness is.** Codex 0.144.x pins an objective to a thread (`create_goal`) and then
|
|
77
|
+
continues it AUTOMATICALLY: when the thread goes idle it injects a synthetic
|
|
78
|
+
`<codex_internal_context source="goal">` user message and runs another turn. iapeer owns no
|
|
79
|
+
part of this and writes nothing into it.
|
|
80
|
+
|
|
81
|
+
**Why it stalls.** `ext/goal/src/extension.rs::on_turn_error` maps every terminal turn error
|
|
82
|
+
that is not a usage limit to `ActiveGoalStopReason::TurnError`, which `ext/goal/src/runtime.rs`
|
|
83
|
+
writes as status `blocked`. That is the **only** path that reaches `blocked` without the model
|
|
84
|
+
calling `update_goal`. Once the status leaves `active`, `clear_active_goal()` runs and
|
|
85
|
+
automatic continuation stops **permanently**: the objective is abandoned while the peer stays
|
|
86
|
+
healthy and keeps taking unrelated turns.
|
|
87
|
+
|
|
88
|
+
**Where the truth is — the state DB, not the rollout.** Unlike `peer-mute`, this fact is *not*
|
|
89
|
+
in the session file. A live capture (16.07.2026, `zapret2-oneclick`, thread `019f6709…`) holds
|
|
90
|
+
exactly one `thread_goal_updated` event in 1971 rollout lines — the `active` one at resume —
|
|
91
|
+
and **none** for the transition to `blocked`. The transition exists only in codex's goal store:
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
~/.codex/goals_<n>.sqlite thread_goals(thread_id, goal_id, objective, status,
|
|
95
|
+
token_budget, tokens_used, time_used_seconds,
|
|
96
|
+
created_at_ms, updated_at_ms)
|
|
97
|
+
~/.codex/state_<n>.sqlite threads(id, cwd, …) ← the only place a thread's cwd lives
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Both are read **read-only**, and the filename generation is globbed (highest `_<n>` wins) —
|
|
101
|
+
`state_5` says codex has rolled it before, so a pinned literal name would silently read a
|
|
102
|
+
stale DB the day it rolls again. Attribution is the §6 rule unchanged: `thread_id` → `cwd`
|
|
103
|
+
(state DB) → personality (registry). A thread whose cwd is no peer's is a human's own codex
|
|
104
|
+
and is never notified.
|
|
105
|
+
|
|
106
|
+
**The reported statuses** are codex's own values, verbatim on `errorType`:
|
|
107
|
+
|
|
108
|
+
| status | meaning | notified |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `blocked` | a turn error stopped continuation | **yes** |
|
|
111
|
+
| `usage_limited` | a usage wall stopped continuation | **yes** |
|
|
112
|
+
| `budget_limited` | the token budget stopped continuation | **yes** |
|
|
113
|
+
| `active` | being worked | no — healthy |
|
|
114
|
+
| `complete` | the model proved it done | no — success is not news |
|
|
115
|
+
| `paused` | the human's own choice | no — his own click is not news |
|
|
116
|
+
|
|
117
|
+
`blocked` and `usage_limited` stay distinct rather than flattening into one "stalled": they
|
|
118
|
+
are different facts with different remedies, and they carry different dedup identities.
|
|
119
|
+
|
|
120
|
+
**One-shot without a state file.** A stalled goal's row is never touched again — its
|
|
121
|
+
accounting is `ActiveOnly`, measured live: through 17 further minutes of turns the blocked
|
|
122
|
+
row's `tokens_used`/`updated_at_ms` stayed frozen at the block instant. So `updated_at_ms` is
|
|
123
|
+
a stable *transition* clock, and the `updated_at_ms > since` boundary fires exactly once and
|
|
124
|
+
then falls behind forever. A goal blocked while the daemon was down is missed — the same
|
|
125
|
+
honest boundary `peer-mute` already has (§7).
|
|
126
|
+
|
|
69
127
|
## 3. What the owner is told — and what is deliberately withheld
|
|
70
128
|
|
|
71
129
|
A notice states who, which runtime, which error type, which model, and when the wall lifts.
|
|
@@ -108,15 +166,15 @@ resets, the plan) — data, never our interpretation of the cause.
|
|
|
108
166
|
| field | type | notes |
|
|
109
167
|
|---|---|---|
|
|
110
168
|
| `id` | string | `n1`, `n2`, … Unique while live; **not** stable across a daemon restart. |
|
|
111
|
-
| `personality` | string | the
|
|
169
|
+
| `personality` | string | the affected peer |
|
|
112
170
|
| `runtime` | string | `claude` \| `codex` |
|
|
113
|
-
| `kind` | string | `peer-mute`
|
|
114
|
-
| `errorType` | string | the runtime's OWN value (`rate_limit`, `overloaded`, `rate_limit_reached`,
|
|
115
|
-
| `model` | string? | **absent** when the runtime did not name it |
|
|
171
|
+
| `kind` | string | `peer-mute` \| `peer-goal-stalled` (codex only). **Treat unknown kinds as renderable** — new kinds ride this surface. |
|
|
172
|
+
| `errorType` | string | the runtime's OWN value (`rate_limit`, `overloaded`, `rate_limit_reached`, …; for `peer-goal-stalled` the goal status: `blocked` \| `usage_limited` \| `budget_limited`). Free-form: do not switch exhaustively. |
|
|
173
|
+
| `model` | string? | **absent** when the runtime did not name it (always absent on `peer-goal-stalled` — the goal store names no model) |
|
|
116
174
|
| `resetsAtMs` | number? | epoch-ms; **absent** when the runtime did not state it → render "unknown", never substitute |
|
|
117
|
-
| `summary` | string | one line, e.g. `boris · claude — rate_limit (Fable 5)` |
|
|
118
|
-
| `content` | string | verbatim (claude) / rendered from typed fields (codex) |
|
|
119
|
-
| `sessionId` | string? | correlates with the on-disk transcript |
|
|
175
|
+
| `summary` | string | one line, e.g. `boris · claude — rate_limit (Fable 5)`, `zapret2-oneclick · codex — goal blocked: <objective>` |
|
|
176
|
+
| `content` | string | verbatim (claude) / rendered from typed fields (codex, both kinds) |
|
|
177
|
+
| `sessionId` | string? | correlates with the on-disk transcript. On `peer-goal-stalled` this is the codex **thread id** — the rollout's `session_meta.session_id`. |
|
|
120
178
|
| `createdMs` | number | when the notice was raised (board clock — the TTL anchor) |
|
|
121
179
|
| `lastMs` | number | when the latest occurrence was folded in (board clock). A sweep that merely re-reads an already-counted event does not move it. |
|
|
122
180
|
| `expiresMs` | number | TTL boundary |
|
|
@@ -241,6 +299,18 @@ there is nothing to derive from. Reading the file's own statement works for both
|
|
|
241
299
|
- **The blocking-window heuristic is unverified.** Codex does not say WHICH window blocked;
|
|
242
300
|
the notice reports the **fullest** one. Both windows ride in `content`, so a human sees the
|
|
243
301
|
raw fact even if the heuristic picks wrong.
|
|
302
|
+
- **`peer-goal-stalled` — the state read is proven on real data; the CAUSE of one instance is
|
|
303
|
+
not.** The detector was run against the live goal store on 16.07.2026 and returned exactly
|
|
304
|
+
the real stall (`zapret2-oneclick`, `blocked`, goal `c25d2378…`, 09:54:32, attributed via the
|
|
305
|
+
real registry), and returned nothing with the boundary at `now` — both directions on real
|
|
306
|
+
bytes, not fixtures. What is **not** proven is which turn error blocked that particular goal:
|
|
307
|
+
codex logged nothing in the 28 s spanning the transition and the rollout carries no error
|
|
308
|
+
event. That gap does not weaken the detector, which watches the **state** and is deliberately
|
|
309
|
+
cause-agnostic — it is why the notice reports codex's status verbatim and never names a cause.
|
|
310
|
+
- **The `blocked` mechanism itself is proven by source + live state**, not inferred from
|
|
311
|
+
silence: `on_turn_error` → `TurnError` → `Blocked` is the only non-`update_goal` writer in
|
|
312
|
+
the upstream tree, the model called `update_goal` zero times in the captured rollout, and the
|
|
313
|
+
blocked row's counters stayed frozen through 17 further minutes of turns.
|
|
244
314
|
|
|
245
315
|
Forensic evidence for all of the above: `docs/internals/forensics/model-limit-2026-07-15/`.
|
|
246
316
|
|
package/package.json
CHANGED
package/src/daemon/fleet.ts
CHANGED
|
@@ -64,6 +64,8 @@ import { approvalsLogPath } from './approvalslog.ts'
|
|
|
64
64
|
import { ApprovalBroker } from './approvals.ts'
|
|
65
65
|
import { noticesLogPath } from './noticeslog.ts'
|
|
66
66
|
import { NoticeBoard } from './notices.ts'
|
|
67
|
+
import { turnsLogPath } from './turnslog.ts'
|
|
68
|
+
import { ActivityBoard, type TurnActivity } from './turnwatch.ts'
|
|
67
69
|
import { callTool, type CallToolDeps } from './index.ts'
|
|
68
70
|
|
|
69
71
|
/** The API generation baked into every snapshot (additive evolution within v1 —
|
|
@@ -79,6 +81,18 @@ export interface FleetRuntimeStatus {
|
|
|
79
81
|
status: 'live' | 'asleep' | 'stopped'
|
|
80
82
|
/** Present (true) iff a HUMAN operator is attached to this live hosted session. */
|
|
81
83
|
attached?: boolean
|
|
84
|
+
/** Whether this runtime is IN A TURN right now — a SEPARATE concept from `status`, which
|
|
85
|
+
* stays pure liveness (docs/15 §Activity). Read from the runtime's own turn marker
|
|
86
|
+
* (codex `task_started`/`task_complete`; claude assistant `stop_reason`), never from CPU or
|
|
87
|
+
* an mtime proxy.
|
|
88
|
+
*
|
|
89
|
+
* ABSENT unless `status === 'live'`: an asleep/stopped session has no turn to be in, and a
|
|
90
|
+
* fabricated `idle` would read as a claim we have not earned. Also absent for a live session
|
|
91
|
+
* the daemon has no evidence for yet.
|
|
92
|
+
*
|
|
93
|
+
* `unknown` = live, saw the turn start, never saw it finish, and the evidence has gone
|
|
94
|
+
* stale (TURN_STALE_MS) — an honest "no claim", never a phantom `working`. */
|
|
95
|
+
activity?: TurnActivity
|
|
82
96
|
}
|
|
83
97
|
|
|
84
98
|
export interface FleetPeer {
|
|
@@ -208,12 +222,25 @@ export function buildFleetSnapshot(
|
|
|
208
222
|
nowMs: number = Date.now(),
|
|
209
223
|
approvals: FleetApproval[] = [],
|
|
210
224
|
notices: FleetNotice[] = [],
|
|
225
|
+
/** Per-runtime turn activity (docs/15 §Activity). Injected — the snapshot never scans
|
|
226
|
+
* session files itself; the turn-watch owns that and keeps an in-memory board, so building
|
|
227
|
+
* a snapshot stays O(peers). Absent lookup (library/test daemons) → the field is simply
|
|
228
|
+
* never emitted, which is exactly its "no claim" semantics. */
|
|
229
|
+
activityOf?: (personality: string, runtime: string) => TurnActivity | undefined,
|
|
211
230
|
): FleetSnapshot {
|
|
212
231
|
const peers: FleetPeer[] = ops.listPeers({ env }).map(r => {
|
|
213
232
|
const runtimes: FleetRuntimeStatus[] = r.runtimes.map(s => {
|
|
214
233
|
const attached =
|
|
215
234
|
s.status === 'live' && hasAttachedSupervisorClient(buildProcessAddress(s.runtime, r.personality), env)
|
|
216
|
-
|
|
235
|
+
// Live-only, by design: see FleetRuntimeStatus.activity on why a dead session gets no
|
|
236
|
+
// claim rather than a fabricated `idle`.
|
|
237
|
+
const activity = s.status === 'live' ? activityOf?.(r.personality, s.runtime) : undefined
|
|
238
|
+
return {
|
|
239
|
+
runtime: s.runtime,
|
|
240
|
+
status: s.status,
|
|
241
|
+
...(attached ? { attached: true } : {}),
|
|
242
|
+
...(activity ? { activity } : {}),
|
|
243
|
+
}
|
|
217
244
|
})
|
|
218
245
|
let ephemeral = false
|
|
219
246
|
try {
|
|
@@ -296,6 +323,10 @@ export function fleetEventFiles(cfg: LifecycleConfig): Array<{ path: string; src
|
|
|
296
323
|
// path — a face subscribed to /fleet/v1/events receives owner notices with no
|
|
297
324
|
// further wiring, exactly as it already receives approvals.
|
|
298
325
|
{ path: noticesLogPath(cfg.eventLogDir), src: 'notices' },
|
|
326
|
+
// Turn-activity events (docs/15 §Activity): turn-started / turn-ended. Same trick — the
|
|
327
|
+
// log IS the push path. Written on a real state CHANGE only, so this is two lines per turn
|
|
328
|
+
// per runtime, not one per sweep.
|
|
329
|
+
{ path: turnsLogPath(cfg.eventLogDir), src: 'turns' },
|
|
299
330
|
]
|
|
300
331
|
}
|
|
301
332
|
|
|
@@ -517,6 +548,11 @@ export interface FleetHandlerOptions {
|
|
|
517
548
|
* mute-watch raises into — that shared instance IS the wiring: the watch writes, this
|
|
518
549
|
* handler reads. Default: a fresh one (a handler with no watch simply stays empty). */
|
|
519
550
|
board?: NoticeBoard
|
|
551
|
+
/** The turn-activity board (docs/15 §Activity). Same sharing contract as `board`: the
|
|
552
|
+
* composition point passes the SAME instance the turn-watch folds observations into.
|
|
553
|
+
* Default: a fresh one — a handler with no watch reports no activity at all, which is the
|
|
554
|
+
* correct "no claim" rather than a fabricated idle. */
|
|
555
|
+
activity?: ActivityBoard
|
|
520
556
|
}
|
|
521
557
|
|
|
522
558
|
export type FleetHandler = (
|
|
@@ -542,6 +578,11 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
542
578
|
// The notice board — one per daemon; the mute-watch RAISES into it and the /notices reads
|
|
543
579
|
// below serve it. Same shape as the broker above: one in-memory store, many interfaces.
|
|
544
580
|
const board = opts.board ?? new NoticeBoard({ logDir: cfg.eventLogDir, env })
|
|
581
|
+
// The turn-activity board — same one-store-many-interfaces shape. The turn-watch folds
|
|
582
|
+
// observations in; the snapshot below reads it. Reading is O(1): no file touched per request.
|
|
583
|
+
const activity = opts.activity ?? new ActivityBoard()
|
|
584
|
+
const activityOf = (personality: string, runtime: string): TurnActivity | undefined =>
|
|
585
|
+
activity.get(personality, runtime, Date.now())
|
|
545
586
|
let opsPromise: Promise<FleetOps> | undefined
|
|
546
587
|
const getOps = (): Promise<FleetOps> => {
|
|
547
588
|
if (opts.ops) return Promise.resolve(opts.ops)
|
|
@@ -564,7 +605,7 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
564
605
|
return sendJson(
|
|
565
606
|
res,
|
|
566
607
|
200,
|
|
567
|
-
buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list()),
|
|
608
|
+
buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list(), activityOf),
|
|
568
609
|
)
|
|
569
610
|
}
|
|
570
611
|
if (path === '/fleet/v1/events') {
|
|
@@ -576,7 +617,17 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
576
617
|
if (peerMatch) {
|
|
577
618
|
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/peers/<peer>' })
|
|
578
619
|
const personality = decodeURIComponent(peerMatch[1]!)
|
|
579
|
-
|
|
620
|
+
// activityOf passed here too: peer-detail must not disagree with the list it came from.
|
|
621
|
+
const snapshot = buildFleetSnapshot(
|
|
622
|
+
env,
|
|
623
|
+
cfg,
|
|
624
|
+
await getOps(),
|
|
625
|
+
startedAtMs,
|
|
626
|
+
undefined,
|
|
627
|
+
undefined,
|
|
628
|
+
undefined,
|
|
629
|
+
activityOf,
|
|
630
|
+
)
|
|
580
631
|
const peer = snapshot.peers.find(p => p.personality === personality)
|
|
581
632
|
if (!peer) return sendJson(res, 404, { error: `unknown peer "${personality}"` })
|
|
582
633
|
const events = readRecentEvents(files, 50, personality)
|
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
// Goal-watch — the daemon's detector for a codex peer whose OBJECTIVE died while the peer
|
|
2
|
+
// itself stayed perfectly healthy (docs/19). Sibling of mutewatch.ts: same class of failure
|
|
3
|
+
// (peer alive, every health signal green, nothing anywhere saying why), different evidence.
|
|
4
|
+
//
|
|
5
|
+
// WHAT CODEX'S GOAL HARNESS IS. Codex 0.144.x ships a "thread goal": an objective pinned to a
|
|
6
|
+
// thread (`create_goal`), after which codex AUTOMATICALLY continues working it — when the
|
|
7
|
+
// thread goes idle the TUI injects a synthetic `<codex_internal_context source="goal">` user
|
|
8
|
+
// message and runs another turn. That loop is the harness's own; iapeer owns NO part of it and
|
|
9
|
+
// this module WRITES nothing — it only reads what codex already wrote about itself.
|
|
10
|
+
//
|
|
11
|
+
// THE FAILURE. `ext/goal/src/extension.rs::on_turn_error` maps EVERY terminal turn error that
|
|
12
|
+
// is not a usage limit to `ActiveGoalStopReason::TurnError`, and `runtime.rs` turns that into
|
|
13
|
+
// status `blocked`. Upstream calls this deliberate ("Block the goal to prevent automatic
|
|
14
|
+
// continuation from looping and consuming tokens, as can happen with compaction errors"), and
|
|
15
|
+
// it is the ONLY path that writes `blocked` without the model calling `update_goal`. Once the
|
|
16
|
+
// status leaves `active`, `clear_active_goal()` runs and automatic continuation STOPS FOREVER.
|
|
17
|
+
// The objective is abandoned. Nothing dies, nothing errors, no exit code, no IAP message —
|
|
18
|
+
// the peer just never works the objective again. Measured live on this host 16.07.2026
|
|
19
|
+
// (zapret2-oneclick, thread 019f6709…, goal c25d2378…): status `blocked` at 09:54:32 with no
|
|
20
|
+
// `update_goal` call anywhere in the rollout, while the peer went on running unrelated turns
|
|
21
|
+
// for another 17 minutes with its goal counters FROZEN at the block instant.
|
|
22
|
+
//
|
|
23
|
+
// WHY SQLITE AND NOT THE ROLLOUT. mutewatch reads the rollout because the rate-limit fact is
|
|
24
|
+
// written there. The block is NOT: the same live capture shows exactly ONE `thread_goal_updated`
|
|
25
|
+
// event in 1971 rollout lines (the `active` one at resume) and NONE for the transition to
|
|
26
|
+
// `blocked`. The transition is written ONLY to codex's goals DB. Read the state or miss it.
|
|
27
|
+
//
|
|
28
|
+
// WHY TWO DATABASES. The goals DB keys on thread_id and carries no cwd, so a goal row alone
|
|
29
|
+
// cannot be attributed to a peer. Codex's state DB (`threads`) carries `id` + `cwd`. Joining
|
|
30
|
+
// the two yields cwd, and cwd → personality is the SAME attribution rule mutewatch already
|
|
31
|
+
// uses: a thread whose cwd is no peer's (a human's own codex) is ignored, never notified.
|
|
32
|
+
//
|
|
33
|
+
// WHY THE FILENAMES ARE GLOBBED. These are codex's private files and their names carry a
|
|
34
|
+
// schema generation (`goals_1.sqlite`, `state_5.sqlite` — the 5 says it has rolled before).
|
|
35
|
+
// Pinning the literal name would silently read a stale DB the day codex rolls it, so the
|
|
36
|
+
// highest generation present wins. Absent/unreadable/renamed → the detector no-ops. Reading
|
|
37
|
+
// another product's internal store is a real coupling; it is accepted for the same reason
|
|
38
|
+
// mutewatch parses codex's rollout — there is no other place the fact exists — and it is
|
|
39
|
+
// confined to this module, guarded, and READ-ONLY.
|
|
40
|
+
//
|
|
41
|
+
// ONE-SHOT, WITHOUT A STATE FILE. A blocked goal's row is never touched again (its accounting
|
|
42
|
+
// is ActiveOnly — proven live: counters frozen through 17 minutes of later turns), so its
|
|
43
|
+
// `updated_at_ms` is a STABLE transition clock. Filtering on `updated_at_ms > sinceMs` there-
|
|
44
|
+
// fore fires exactly once per transition and then falls behind the boundary forever — no
|
|
45
|
+
// repeat, no TTL re-raise, no persisted "already told him" set. A goal blocked while the
|
|
46
|
+
// daemon was down is missed, exactly as mutewatch misses an error it was not up for; the
|
|
47
|
+
// boundary is the contour's existing honesty, not a new compromise.
|
|
48
|
+
|
|
49
|
+
import { existsSync, readdirSync } from 'node:fs'
|
|
50
|
+
import { join } from 'node:path'
|
|
51
|
+
import type { LifecycleConfig } from '../lifecycle/index.ts'
|
|
52
|
+
import type { NoticeBoard } from './notices.ts'
|
|
53
|
+
|
|
54
|
+
/** Overlap on the since-boundary so a transition landing between two sweeps is never missed.
|
|
55
|
+
* Re-reads are free: the board folds them by `eventAtMs` (see notices.ts). */
|
|
56
|
+
const SCAN_OVERLAP_MS = 15_000
|
|
57
|
+
/** Objective text is user-authored and unbounded — bound it before it reaches a notice. */
|
|
58
|
+
const OBJECTIVE_MAX = 300
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The goal statuses that mean "this objective is not being worked and will not resume on its
|
|
62
|
+
* own". CODEX'S OWN taxonomy values, never our invention (docs/19 §2).
|
|
63
|
+
*
|
|
64
|
+
* Deliberately excluded:
|
|
65
|
+
* • `active` — working; the healthy case.
|
|
66
|
+
* • `complete` — the model proved the objective done. Success is not news.
|
|
67
|
+
* • `paused` — a human's own choice. Notifying him of his own click is noise.
|
|
68
|
+
* `usage_limited` / `budget_limited` ride along because they are the same shape: continuation
|
|
69
|
+
* stopped, objective unfinished, nobody told. They are NOT the same CAUSE as `blocked`, which
|
|
70
|
+
* is why the status travels verbatim as the notice's errorType instead of being flattened.
|
|
71
|
+
*/
|
|
72
|
+
export const STALLED_GOAL_STATUSES = new Set(['blocked', 'usage_limited', 'budget_limited'])
|
|
73
|
+
|
|
74
|
+
/** A row of codex's `thread_goals`, as codex wrote it. */
|
|
75
|
+
export interface GoalRow {
|
|
76
|
+
threadId: string
|
|
77
|
+
goalId: string
|
|
78
|
+
objective: string
|
|
79
|
+
status: string
|
|
80
|
+
tokensUsed: number
|
|
81
|
+
timeUsedSeconds: number
|
|
82
|
+
updatedAtMs: number
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface GoalDetection {
|
|
86
|
+
personality: string
|
|
87
|
+
runtime: 'codex'
|
|
88
|
+
/** Codex's own status value — `blocked` | `usage_limited` | `budget_limited`. */
|
|
89
|
+
status: string
|
|
90
|
+
goalId: string
|
|
91
|
+
threadId: string
|
|
92
|
+
objective: string
|
|
93
|
+
tokensUsed: number
|
|
94
|
+
timeUsedSeconds: number
|
|
95
|
+
/** The row's `updated_at_ms` — the TRANSITION's own clock, not our sweep's. */
|
|
96
|
+
atMs: number
|
|
97
|
+
cwd: string
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
101
|
+
// Pure selection — rows in, detections out (fs/sqlite-free: tests need no DB)
|
|
102
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
function clip(s: string, max: number): string {
|
|
105
|
+
const t = s.replace(/\s+/g, ' ').trim()
|
|
106
|
+
return t.length <= max ? t : `${t.slice(0, max - 1)}…`
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The human-readable line, rendered from codex's TYPED fields — codex writes no prose for a
|
|
110
|
+
* goal transition, the same situation as its rate-limit snapshot, and the same answer:
|
|
111
|
+
* render the facts, never our reading of the cause. */
|
|
112
|
+
export function renderGoalContent(d: Omit<GoalDetection, 'personality' | 'runtime'>): string {
|
|
113
|
+
return [
|
|
114
|
+
`Codex goal ${d.status} — automatic goal continuation has stopped; this objective will not`,
|
|
115
|
+
`resume without an explicit resume. Objective: "${d.objective}".`,
|
|
116
|
+
`goal=${d.goalId} thread=${d.threadId} accounted=${d.tokensUsed} tokens / ${d.timeUsedSeconds}s.`,
|
|
117
|
+
].join(' ')
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Every peer-attributable goal that stalled STRICTLY after `sinceMs`.
|
|
122
|
+
*
|
|
123
|
+
* `cwdByThread` and `peerByCwd` are injected so this stays pure: the whole selection rule is
|
|
124
|
+
* testable without a codex install.
|
|
125
|
+
*/
|
|
126
|
+
export function selectStalledGoals(
|
|
127
|
+
goals: GoalRow[],
|
|
128
|
+
cwdByThread: Map<string, string>,
|
|
129
|
+
peerByCwd: (cwd: string) => string | undefined,
|
|
130
|
+
sinceMs: number,
|
|
131
|
+
): GoalDetection[] {
|
|
132
|
+
const out: GoalDetection[] = []
|
|
133
|
+
for (const g of goals) {
|
|
134
|
+
if (!STALLED_GOAL_STATUSES.has(g.status)) continue
|
|
135
|
+
if (!Number.isFinite(g.updatedAtMs) || g.updatedAtMs <= sinceMs) continue
|
|
136
|
+
const cwd = cwdByThread.get(g.threadId)
|
|
137
|
+
if (!cwd) continue // thread not in the state DB → unattributable; ignore rather than guess
|
|
138
|
+
const personality = peerByCwd(cwd)
|
|
139
|
+
if (!personality) continue // a human's own codex thread — never notify
|
|
140
|
+
out.push({
|
|
141
|
+
personality,
|
|
142
|
+
runtime: 'codex',
|
|
143
|
+
status: g.status,
|
|
144
|
+
goalId: g.goalId,
|
|
145
|
+
threadId: g.threadId,
|
|
146
|
+
objective: clip(g.objective, OBJECTIVE_MAX),
|
|
147
|
+
tokensUsed: g.tokensUsed,
|
|
148
|
+
timeUsedSeconds: g.timeUsedSeconds,
|
|
149
|
+
atMs: g.updatedAtMs,
|
|
150
|
+
cwd,
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
return out
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
157
|
+
// The codex-state reader — the only part that touches another product's files
|
|
158
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
159
|
+
|
|
160
|
+
export interface GoalWatchPaths {
|
|
161
|
+
/** ~/.codex — INJECTED, never re-resolved from env inside a sweep. */
|
|
162
|
+
codexHome: string
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function resolveGoalWatchPaths(env: NodeJS.ProcessEnv): GoalWatchPaths {
|
|
166
|
+
const home = env.HOME?.trim() || ''
|
|
167
|
+
return { codexHome: env.IAPEER_CODEX_HOME?.trim() || join(home, '.codex') }
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The highest-generation `<prefix>_<n>.sqlite` in `dir`, or null.
|
|
172
|
+
* Codex rolls the generation when it changes schema; the newest is the live one.
|
|
173
|
+
*/
|
|
174
|
+
export function findVersionedDb(dir: string, prefix: string): string | null {
|
|
175
|
+
if (!existsSync(dir)) return null
|
|
176
|
+
let best: { n: number; name: string } | null = null
|
|
177
|
+
let names: string[]
|
|
178
|
+
try {
|
|
179
|
+
names = readdirSync(dir)
|
|
180
|
+
} catch {
|
|
181
|
+
return null
|
|
182
|
+
}
|
|
183
|
+
const re = new RegExp(`^${prefix}_(\\d+)\\.sqlite$`)
|
|
184
|
+
for (const name of names) {
|
|
185
|
+
const m = re.exec(name)
|
|
186
|
+
if (!m) continue
|
|
187
|
+
const n = parseInt(m[1] as string, 10)
|
|
188
|
+
if (!Number.isFinite(n)) continue
|
|
189
|
+
if (!best || n > best.n) best = { n, name }
|
|
190
|
+
}
|
|
191
|
+
return best ? join(dir, best.name) : null
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Open a codex DB READ-ONLY. Never creates, never migrates, never writes. */
|
|
195
|
+
async function openReadOnly(path: string): Promise<{ query: (sql: string) => { all: (...a: unknown[]) => unknown[] }; close: () => void } | null> {
|
|
196
|
+
try {
|
|
197
|
+
const { Database } = (await import('bun:sqlite')) as typeof import('bun:sqlite')
|
|
198
|
+
return new Database(path, { readonly: true }) as unknown as {
|
|
199
|
+
query: (sql: string) => { all: (...a: unknown[]) => unknown[] }
|
|
200
|
+
close: () => void
|
|
201
|
+
}
|
|
202
|
+
} catch {
|
|
203
|
+
return null // absent / locked / rolled schema → the detector no-ops, never throws
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function num(v: unknown): number {
|
|
208
|
+
return typeof v === 'number' && Number.isFinite(v) ? v : 0
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function str(v: unknown): string {
|
|
212
|
+
return typeof v === 'string' ? v : ''
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Codex's goal rows. Empty on any shape we do not recognise — never a throw. */
|
|
216
|
+
export async function readGoalRows(codexHome: string): Promise<GoalRow[]> {
|
|
217
|
+
const path = findVersionedDb(codexHome, 'goals')
|
|
218
|
+
if (!path) return []
|
|
219
|
+
const db = await openReadOnly(path)
|
|
220
|
+
if (!db) return []
|
|
221
|
+
try {
|
|
222
|
+
const rows = db
|
|
223
|
+
.query('SELECT thread_id, goal_id, objective, status, tokens_used, time_used_seconds, updated_at_ms FROM thread_goals')
|
|
224
|
+
.all() as Array<Record<string, unknown>>
|
|
225
|
+
return rows.map(r => ({
|
|
226
|
+
threadId: str(r.thread_id),
|
|
227
|
+
goalId: str(r.goal_id),
|
|
228
|
+
objective: str(r.objective),
|
|
229
|
+
status: str(r.status),
|
|
230
|
+
tokensUsed: num(r.tokens_used),
|
|
231
|
+
timeUsedSeconds: num(r.time_used_seconds),
|
|
232
|
+
updatedAtMs: num(r.updated_at_ms),
|
|
233
|
+
}))
|
|
234
|
+
} catch {
|
|
235
|
+
return []
|
|
236
|
+
} finally {
|
|
237
|
+
try {
|
|
238
|
+
db.close()
|
|
239
|
+
} catch {
|
|
240
|
+
/* best-effort */
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** thread_id → cwd, for the given threads only. Empty on any unrecognised shape. */
|
|
246
|
+
export async function readThreadCwds(codexHome: string, threadIds: string[]): Promise<Map<string, string>> {
|
|
247
|
+
const out = new Map<string, string>()
|
|
248
|
+
if (threadIds.length === 0) return out
|
|
249
|
+
const path = findVersionedDb(codexHome, 'state')
|
|
250
|
+
if (!path) return out
|
|
251
|
+
const db = await openReadOnly(path)
|
|
252
|
+
if (!db) return out
|
|
253
|
+
try {
|
|
254
|
+
const holes = threadIds.map(() => '?').join(',')
|
|
255
|
+
const rows = db.query(`SELECT id, cwd FROM threads WHERE id IN (${holes})`).all(...threadIds) as Array<Record<string, unknown>>
|
|
256
|
+
for (const r of rows) {
|
|
257
|
+
const id = str(r.id)
|
|
258
|
+
const cwd = str(r.cwd)
|
|
259
|
+
if (id && cwd) out.set(id, cwd)
|
|
260
|
+
}
|
|
261
|
+
} catch {
|
|
262
|
+
/* unrecognised schema → no attribution, no notice */
|
|
263
|
+
} finally {
|
|
264
|
+
try {
|
|
265
|
+
db.close()
|
|
266
|
+
} catch {
|
|
267
|
+
/* best-effort */
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return out
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export interface GoalScanDeps extends GoalWatchPaths {
|
|
274
|
+
/** personality ← cwd. Built from the registry by the composition point. */
|
|
275
|
+
peerByCwd: (cwd: string) => string | undefined
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** One sweep: every peer-attributable goal that stalled since `sinceMs`. */
|
|
279
|
+
export async function scanStalledGoals(deps: GoalScanDeps, sinceMs: number): Promise<GoalDetection[]> {
|
|
280
|
+
const floor = sinceMs - SCAN_OVERLAP_MS
|
|
281
|
+
const goals = await readGoalRows(deps.codexHome)
|
|
282
|
+
// Only the stalled rows are worth a threads lookup — an active goal is the common case.
|
|
283
|
+
const candidates = goals.filter(g => STALLED_GOAL_STATUSES.has(g.status) && g.updatedAtMs > floor)
|
|
284
|
+
if (candidates.length === 0) return []
|
|
285
|
+
const cwds = await readThreadCwds(
|
|
286
|
+
deps.codexHome,
|
|
287
|
+
candidates.map(g => g.threadId).filter(Boolean),
|
|
288
|
+
)
|
|
289
|
+
return selectStalledGoals(candidates, cwds, deps.peerByCwd, floor)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
293
|
+
// The tick
|
|
294
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
295
|
+
|
|
296
|
+
export interface GoalWatchDeps extends GoalScanDeps {
|
|
297
|
+
board: NoticeBoard
|
|
298
|
+
now?: () => number
|
|
299
|
+
/** Reported, never thrown — a detector must never take the daemon down. */
|
|
300
|
+
onError?: (err: unknown) => void
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** Run one watch tick: sweep, raise. Returns notices actually RAISED + the new boundary. */
|
|
304
|
+
export async function goalWatchTick(deps: GoalWatchDeps, sinceMs: number): Promise<{ raised: number; sinceMs: number }> {
|
|
305
|
+
const now = deps.now ?? Date.now
|
|
306
|
+
const startedMs = now()
|
|
307
|
+
let raised = 0
|
|
308
|
+
try {
|
|
309
|
+
for (const d of await scanStalledGoals(deps, sinceMs)) {
|
|
310
|
+
const { deduped } = deps.board.raise({
|
|
311
|
+
personality: d.personality,
|
|
312
|
+
runtime: d.runtime,
|
|
313
|
+
kind: 'peer-goal-stalled',
|
|
314
|
+
// Codex's own status is the taxonomy value — `blocked` and `usage_limited` are
|
|
315
|
+
// different facts for the owner and must not collapse into one dedup identity.
|
|
316
|
+
errorType: d.status,
|
|
317
|
+
content: renderGoalContent(d),
|
|
318
|
+
// The thread IS the session: it correlates the notice with the rollout on disk.
|
|
319
|
+
sessionId: d.threadId,
|
|
320
|
+
// The TRANSITION's own clock — see notices.ts on why this is not the sweep's.
|
|
321
|
+
eventAtMs: d.atMs,
|
|
322
|
+
summary: `${d.personality} · codex — goal ${d.status}: ${clip(d.objective, 80)}`,
|
|
323
|
+
})
|
|
324
|
+
if (!deduped) raised += 1
|
|
325
|
+
}
|
|
326
|
+
} catch (e) {
|
|
327
|
+
deps.onError?.(e)
|
|
328
|
+
}
|
|
329
|
+
return { raised, sinceMs: startedMs }
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
export interface StartGoalWatchOptions {
|
|
333
|
+
env?: NodeJS.ProcessEnv
|
|
334
|
+
intervalMs?: number
|
|
335
|
+
paths?: GoalWatchPaths
|
|
336
|
+
/** Override the registry read (tests). Default: a lazy import of the CLI's listPeers. */
|
|
337
|
+
listPeers?: () => Array<{ personality: string; cwd: string }>
|
|
338
|
+
now?: () => number
|
|
339
|
+
onError?: (err: unknown) => void
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Start the goal-watch timer. Returns its stop function — the caller OWNS teardown.
|
|
344
|
+
* Mirrors startMuteWatch: registry re-read every sweep, `listPeers` imported lazily.
|
|
345
|
+
*/
|
|
346
|
+
export function startGoalWatch(_cfg: LifecycleConfig, board: NoticeBoard, opts: StartGoalWatchOptions = {}): () => void {
|
|
347
|
+
const env = opts.env ?? process.env
|
|
348
|
+
const paths = opts.paths ?? resolveGoalWatchPaths(env)
|
|
349
|
+
const now = opts.now ?? Date.now
|
|
350
|
+
let listPeers = opts.listPeers
|
|
351
|
+
let since = now()
|
|
352
|
+
let running = false
|
|
353
|
+
|
|
354
|
+
const sweep = async (): Promise<void> => {
|
|
355
|
+
if (running) return // a slow sweep must never overlap itself into a stampede
|
|
356
|
+
running = true
|
|
357
|
+
try {
|
|
358
|
+
if (!listPeers) {
|
|
359
|
+
const cli = await import('../cli/index.ts')
|
|
360
|
+
listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
|
|
361
|
+
}
|
|
362
|
+
const rows = listPeers()
|
|
363
|
+
// Case-insensitive, trailing-slash-tolerant — same rule and same reason as mutewatch:
|
|
364
|
+
// macOS paths are case-insensitive and codex records the cwd STRING as launched.
|
|
365
|
+
const byCwd = new Map<string, string>()
|
|
366
|
+
for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
|
|
367
|
+
const deps: GoalWatchDeps = {
|
|
368
|
+
...paths,
|
|
369
|
+
board,
|
|
370
|
+
now,
|
|
371
|
+
onError: opts.onError,
|
|
372
|
+
peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
|
|
373
|
+
}
|
|
374
|
+
since = (await goalWatchTick(deps, since)).sinceMs
|
|
375
|
+
} catch (e) {
|
|
376
|
+
opts.onError?.(e)
|
|
377
|
+
} finally {
|
|
378
|
+
running = false
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const timer = setInterval(() => void sweep(), opts.intervalMs ?? 20_000)
|
|
383
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
384
|
+
return () => clearInterval(timer)
|
|
385
|
+
}
|
package/src/daemon/main.ts
CHANGED
|
@@ -69,6 +69,8 @@ import { appendDeliveryEvent } from './deliverylog.ts'
|
|
|
69
69
|
import { buildFleetHandler } from './fleet.ts'
|
|
70
70
|
import { NoticeBoard } from './notices.ts'
|
|
71
71
|
import { startMuteWatch } from './mutewatch.ts'
|
|
72
|
+
import { startGoalWatch } from './goalwatch.ts'
|
|
73
|
+
import { ActivityBoard, startTurnWatch } from './turnwatch.ts'
|
|
72
74
|
import { defaultDaemonSocketPath, startDaemon, type DaemonHandle } from './index.ts'
|
|
73
75
|
|
|
74
76
|
/** Default TCP loopback port for the always-on router. Real http MCP clients
|
|
@@ -100,6 +102,18 @@ export const DEFAULT_SUPERVISE_INTERVAL_MS = 60_000
|
|
|
100
102
|
* to spare. The sweep is cheap — it stats only files that CHANGED since the last pass. */
|
|
101
103
|
export const DEFAULT_MUTEWATCH_INTERVAL_MS = 20_000
|
|
102
104
|
|
|
105
|
+
/** Goal-watch cadence (docs/19) — how often the daemon reads codex's goal store for an
|
|
106
|
+
* objective that stopped continuing. Same 20 s and the same reason as the mute-watch: the
|
|
107
|
+
* owner must learn within a minute. Cheaper than that sweep — two indexed reads of a small
|
|
108
|
+
* DB, and the threads join runs ONLY when a stalled row is actually present. */
|
|
109
|
+
export const DEFAULT_GOALWATCH_INTERVAL_MS = 20_000
|
|
110
|
+
|
|
111
|
+
/** Turn-watch cadence (docs/15 §Activity) — how often the daemon re-reads the turn markers of
|
|
112
|
+
* session files that CHANGED. Much faster than the notice watches because the fact is
|
|
113
|
+
* short-lived: a turn can be shorter than 20 s, so a 20 s sweep would report `idle` straight
|
|
114
|
+
* through one. This bounds the snapshot's idle↔working lag to roughly one interval. */
|
|
115
|
+
export const DEFAULT_TURNWATCH_INTERVAL_MS = 3_000
|
|
116
|
+
|
|
103
117
|
// This module's own path — the launchd plist runs `bun <this>` as the daemon.
|
|
104
118
|
const DAEMON_MAIN_PATH = fileURLToPath(import.meta.url)
|
|
105
119
|
|
|
@@ -330,6 +344,10 @@ export interface ConfiguredDaemonOptions {
|
|
|
330
344
|
superviseIntervalMs?: number
|
|
331
345
|
/** Mute-watch sweep cadence (default DEFAULT_MUTEWATCH_INTERVAL_MS). */
|
|
332
346
|
muteWatchIntervalMs?: number
|
|
347
|
+
/** Goal-watch sweep cadence (default DEFAULT_GOALWATCH_INTERVAL_MS). */
|
|
348
|
+
goalWatchIntervalMs?: number
|
|
349
|
+
/** Turn-watch sweep cadence (default DEFAULT_TURNWATCH_INTERVAL_MS). */
|
|
350
|
+
turnWatchIntervalMs?: number
|
|
333
351
|
/** Write the router.json discovery file (default true for production). */
|
|
334
352
|
discovery?: boolean
|
|
335
353
|
rootDir?: string
|
|
@@ -369,6 +387,36 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
369
387
|
} catch { /* best-effort */ }
|
|
370
388
|
},
|
|
371
389
|
})
|
|
390
|
+
// Goal-watch (docs/19) — the SAME board and the same class as the mute-watch: a peer left
|
|
391
|
+
// alive and green while the thing it was told to do quietly stopped. Separate detector
|
|
392
|
+
// because the evidence is elsewhere (codex's goal store, not a transcript) and it is
|
|
393
|
+
// codex-only; sharing the board is what makes it one owner-facing surface.
|
|
394
|
+
const stopGoalWatch = startGoalWatch(cfg, board, {
|
|
395
|
+
env,
|
|
396
|
+
intervalMs: opts.goalWatchIntervalMs ?? DEFAULT_GOALWATCH_INTERVAL_MS,
|
|
397
|
+
onError: (err: unknown) => {
|
|
398
|
+
try {
|
|
399
|
+
const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
400
|
+
appendLifecycleEvent(cfg.eventLogDir, { ev: 'goalwatch-error', error: detail.replace(/\s+/g, ' ').slice(0, 600) }, { env })
|
|
401
|
+
} catch { /* best-effort */ }
|
|
402
|
+
},
|
|
403
|
+
})
|
|
404
|
+
// Turn-activity board + watch (docs/15 §Activity). Constructed HERE for the same reason as
|
|
405
|
+
// the notice board: the watch WRITES it and the fleet handler READS it, and that sharing is
|
|
406
|
+
// the whole wiring. Faster cadence than the notice watches (3 s vs 20 s) because a turn is a
|
|
407
|
+
// short-lived fact — a 20 s sweep would miss whole turns. Affordable: the sweep stats only
|
|
408
|
+
// files CHANGED since the last pass (measured on this host: ~2 ms across both runtime trees).
|
|
409
|
+
const activity = new ActivityBoard()
|
|
410
|
+
const stopTurnWatch = startTurnWatch(cfg, activity, {
|
|
411
|
+
env,
|
|
412
|
+
intervalMs: opts.turnWatchIntervalMs ?? DEFAULT_TURNWATCH_INTERVAL_MS,
|
|
413
|
+
onError: (err: unknown) => {
|
|
414
|
+
try {
|
|
415
|
+
const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
416
|
+
appendLifecycleEvent(cfg.eventLogDir, { ev: 'turnwatch-error', error: detail.replace(/\s+/g, ' ').slice(0, 600) }, { env })
|
|
417
|
+
} catch { /* best-effort */ }
|
|
418
|
+
},
|
|
419
|
+
})
|
|
372
420
|
const handle = await startDaemon({
|
|
373
421
|
wake: makeWakeFn(cfg, env),
|
|
374
422
|
// M2 arm-on-outbound (see makeArmEphemeralOnDelivered): ephemeral caller's ok
|
|
@@ -395,7 +443,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
395
443
|
// (the same listPeers truth as `iapeer list`), SSE events (tail of the durable
|
|
396
444
|
// logs), commands over the existing verb functions. Same listeners, same bearer
|
|
397
445
|
// gate; advertised in router.json as fleet:1. Contract: docs/15-fleet-api.md.
|
|
398
|
-
fleet: buildFleetHandler({ env, board }),
|
|
446
|
+
fleet: buildFleetHandler({ env, board, activity }),
|
|
399
447
|
supervise: {
|
|
400
448
|
intervalMs: opts.superviseIntervalMs ?? DEFAULT_SUPERVISE_INTERVAL_MS,
|
|
401
449
|
// idle-reap / zombie-sweep, THEN the eager fresh re-launch for any peer whose
|
|
@@ -443,7 +491,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
443
491
|
// The mute-watch timer is OURS, not startDaemon's — so its teardown must hang off the
|
|
444
492
|
// handle we return, or a closed daemon would leave a live timer sweeping the disk (and
|
|
445
493
|
// every test that starts a daemon would hang on exit).
|
|
446
|
-
return { ...handle, close: async () => { stopMuteWatch(); await handle.close() } }
|
|
494
|
+
return { ...handle, close: async () => { stopMuteWatch(); stopGoalWatch(); stopTurnWatch(); await handle.close() } }
|
|
447
495
|
}
|
|
448
496
|
|
|
449
497
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Turn-activity durable log — the daemon's DURABLE trace of every per-runtime turn boundary
|
|
2
|
+
// it observes (docs/15 §Activity). Sibling to lifecycle.log / delivery.log / exits.log /
|
|
3
|
+
// approvals.log / notices.log under cfg.eventLogDir; tailed by the fleet-API SSE stream
|
|
4
|
+
// (fleetEventFiles), so `turn-started` / `turn-ended` reach every operator client with
|
|
5
|
+
// at-least-once semantics — the same "the log IS the push mechanism" trick notices.log uses.
|
|
6
|
+
//
|
|
7
|
+
// Volume is bounded by DESIGN, not by luck: turnWatchTick writes only on a real state CHANGE,
|
|
8
|
+
// never once per sweep. The watch polls every ~3 s; without that guard it would emit a line per
|
|
9
|
+
// peer per pass and drown the very log meant to make turn history readable. Two lines per turn
|
|
10
|
+
// per runtime is the ceiling.
|
|
11
|
+
//
|
|
12
|
+
// Same rotation class as its siblings: written by OUR code → self-rotates via the shared
|
|
13
|
+
// storage/rotatelog primitive. The dir is passed IN by the composition point, never re-resolved
|
|
14
|
+
// from env — a falsy dir → no-op (library/test daemons stay hermetic).
|
|
15
|
+
|
|
16
|
+
import { join } from 'path'
|
|
17
|
+
import { DEFAULT_LOG_KEEP, DEFAULT_LOG_MAX_BYTES, appendRotatedEvent } from '../storage/rotatelog.ts'
|
|
18
|
+
|
|
19
|
+
/** The turn-activity log inside `logDir` (sibling to notices.log). */
|
|
20
|
+
export function turnsLogPath(logDir: string): string {
|
|
21
|
+
return join(logDir, 'turns.log')
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
25
|
+
const n = parseInt(raw ?? '', 10)
|
|
26
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface AppendTurnOptions {
|
|
30
|
+
/** Reads the rotation knobs IAPEER_TURNS_LOG_MAX_BYTES / _KEEP. */
|
|
31
|
+
env?: NodeJS.ProcessEnv
|
|
32
|
+
/** Stamp the line with this epoch-ms. Default Date.now(). */
|
|
33
|
+
nowMs?: number
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Append one turn event line into `logDir`/turns.log. A falsy `logDir` is a no-op. Fully
|
|
38
|
+
* best-effort — never throws; logging must never fail a detection sweep.
|
|
39
|
+
*/
|
|
40
|
+
export function appendTurnEvent(
|
|
41
|
+
logDir: string | undefined,
|
|
42
|
+
fields: Record<string, string | number | undefined>,
|
|
43
|
+
opts: AppendTurnOptions = {},
|
|
44
|
+
): void {
|
|
45
|
+
if (!logDir) return
|
|
46
|
+
const env = opts.env ?? process.env
|
|
47
|
+
appendRotatedEvent(turnsLogPath(logDir), fields, {
|
|
48
|
+
nowMs: opts.nowMs,
|
|
49
|
+
maxBytes: envPosInt(env.IAPEER_TURNS_LOG_MAX_BYTES, DEFAULT_LOG_MAX_BYTES),
|
|
50
|
+
keep: envPosInt(env.IAPEER_TURNS_LOG_KEEP, DEFAULT_LOG_KEEP),
|
|
51
|
+
})
|
|
52
|
+
}
|
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
// Turn-watch — per-runtime TURN ACTIVITY for the fleet surface (docs/15 §Activity).
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS. The snapshot answered exactly one question about a session: is the process
|
|
4
|
+
// alive (`status: live | asleep | stopped`). That is LIVENESS, and it is not the operator's
|
|
5
|
+
// question. A peer sitting idle with an unfinished job and a peer grinding through a turn are
|
|
6
|
+
// both `live` — indistinguishable. The owner spotted a stalled peer before the PM twice
|
|
7
|
+
// because the surface could not tell "working" from "merely alive".
|
|
8
|
+
//
|
|
9
|
+
// ACTIVITY IS NOT LIVENESS, AND THEY STAY SEPARATE. `status` keeps its exact meaning; the new
|
|
10
|
+
// `activity` field is additive and rides ALONGSIDE it on the same per-runtime object. A client
|
|
11
|
+
// that ignores `activity` is unaffected. Activity is reported ONLY for a `live` runtime — for
|
|
12
|
+
// an asleep/stopped session there is no turn to be in, and the field is OMITTED rather than
|
|
13
|
+
// filled with a fake `idle`: absent means "no claim", which a dead session deserves.
|
|
14
|
+
//
|
|
15
|
+
// THE EVIDENCE IS THE RUNTIME'S OWN TURN MARKER — mechanical, never a CPU/mtime reconstruction:
|
|
16
|
+
//
|
|
17
|
+
// • codex → the rollout's `event_msg` payloads `task_started` / `task_complete`. Explicit
|
|
18
|
+
// turn boundaries; the last one wins. Measured across 3 days of real rollouts: 156 started
|
|
19
|
+
// / 147 complete — the 9-line gap is precisely the dangling-turn risk the stale bound below
|
|
20
|
+
// exists for. Codex emits `task_complete` even for a turn that ended on an error (measured:
|
|
21
|
+
// the 16.07 goal-blocking turn ends token_count → task_complete, no error event), so the
|
|
22
|
+
// terminal error path needs no separate marker.
|
|
23
|
+
//
|
|
24
|
+
// • claude → the transcript's assistant `stop_reason`. `tool_use` means the turn CONTINUES;
|
|
25
|
+
// every other terminal reason means it ended. Measured over every transcript this peer has:
|
|
26
|
+
// tool_use 18317, end_turn 663, stop_sequence 16, null 4. So the rule is deliberately
|
|
27
|
+
// ASYMMETRIC — only `tool_use` reads as working, and anything else (end_turn, stop_sequence,
|
|
28
|
+
// max_tokens, refusal, an unknown future value, null) reads as IDLE. An unknown reason must
|
|
29
|
+
// never invent work; the polarity always fails toward idle.
|
|
30
|
+
//
|
|
31
|
+
// THE THREE FALSE-WORKING TRAPS, each measured on real transcripts rather than assumed:
|
|
32
|
+
//
|
|
33
|
+
// 1. **A `user` line means working** (the model is about to answer, or a tool_result just
|
|
34
|
+
// landed) — but claude writes NON-model user lines too. A compaction summary is
|
|
35
|
+
// `type:"user"` with `isCompactSummary:true` (12 found here), and a `/compact` run while
|
|
36
|
+
// IDLE would leave one as the last line → a permanent phantom "working". Excluded. The
|
|
37
|
+
// `compact_boundary` line is `type:"system"` and never qualifies anyway. Excluding the
|
|
38
|
+
// summary is also CORRECT for mid-turn compaction: the prior assistant `tool_use` line
|
|
39
|
+
// then decides, and the turn is genuinely still running.
|
|
40
|
+
// 2. **Sub-agent lines.** A Task sidechain writes assistant/user lines into the SAME
|
|
41
|
+
// transcript with `isSidechain:true`. They belong to a different chain and must not decide
|
|
42
|
+
// the peer's turn state. Excluded.
|
|
43
|
+
// 3. **Injected meta lines** (`isMeta:true`, 13 found here) are not the model being asked
|
|
44
|
+
// anything. Excluded.
|
|
45
|
+
//
|
|
46
|
+
// THE STALE BOUND. A `working` claim rests on evidence that stops being written the moment a
|
|
47
|
+
// session hangs mid-turn — the transcript simply goes quiet. Liveness already removes the dead
|
|
48
|
+
// session (activity is live-only), so what remains is a LIVE session whose last recorded act
|
|
49
|
+
// was starting work and which has since written nothing. Past TURN_STALE_MS the claim decays to
|
|
50
|
+
// `unknown` rather than standing as `working` forever: unknown is the honest word for "we saw
|
|
51
|
+
// it start and never saw it finish". `idle` never decays — a session that ended its turn stays
|
|
52
|
+
// idle until something pokes it, and age proves nothing against that.
|
|
53
|
+
//
|
|
54
|
+
// Deliberately NOT the pane-log mtime, which looks like a freshness clock and is not: measured
|
|
55
|
+
// live 16.07, an IDLE claude pane-log advances every second (the statusline repaints) while an
|
|
56
|
+
// IDLE codex pane-log freezes (39s→46s). The signal means opposite things per runtime; one
|
|
57
|
+
// clock that lies differently on each runtime is worse than no clock.
|
|
58
|
+
|
|
59
|
+
import { existsSync, openSync, closeSync, fstatSync, readSync, readdirSync, statSync } from 'node:fs'
|
|
60
|
+
import { join } from 'node:path'
|
|
61
|
+
import type { LifecycleConfig } from '../lifecycle/index.ts'
|
|
62
|
+
import { appendTurnEvent } from './turnslog.ts'
|
|
63
|
+
|
|
64
|
+
/** Tail parsed per session file. The turn marker is always at the very end. */
|
|
65
|
+
const TAIL_BYTES = 128 * 1024
|
|
66
|
+
/** Head slice for codex's `session_meta` (line 1 — a long rollout pushes it out of the tail). */
|
|
67
|
+
const HEAD_BYTES = 64 * 1024
|
|
68
|
+
/** Walk depth for the codex YYYY/MM/DD tree. */
|
|
69
|
+
const CODEX_WALK_DEPTH = 3
|
|
70
|
+
/** How far back the FIRST sweep looks, to seed state for sessions already running when the
|
|
71
|
+
* daemon started. Bounded on purpose: a full-history pass would read every transcript ever.
|
|
72
|
+
* Measured cost of a 48h seed on this host: ~2 ms of walk, 97 files. */
|
|
73
|
+
const SEED_LOOKBACK_MS = 48 * 3_600_000
|
|
74
|
+
/** Overlap on the since-boundary so a write landing between sweeps is never missed. */
|
|
75
|
+
const SCAN_OVERLAP_MS = 2_000
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* How long a `working` claim outlives its last evidence before decaying to `unknown`.
|
|
79
|
+
* Generous on purpose: a turn legitimately writes NOTHING while one long tool runs (a deploy
|
|
80
|
+
* here takes minutes), and a false `unknown` on a genuinely working peer is a worse lie than a
|
|
81
|
+
* late one. Ten minutes is far past any tool this fleet runs and still bounds the phantom.
|
|
82
|
+
*/
|
|
83
|
+
export const TURN_STALE_MS = 10 * 60_000
|
|
84
|
+
|
|
85
|
+
/** The operator-facing verdict. `unknown` = live, but the evidence cannot support a claim. */
|
|
86
|
+
export type TurnActivity = 'working' | 'idle' | 'unknown'
|
|
87
|
+
|
|
88
|
+
export interface TurnState {
|
|
89
|
+
state: 'working' | 'idle'
|
|
90
|
+
/** The EVIDENCE's own timestamp (the transcript line's), never our sweep clock. */
|
|
91
|
+
atMs: number
|
|
92
|
+
cwd: string
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
96
|
+
// Pure parsers — text in, state out (fs-free: tests need no session tree)
|
|
97
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
98
|
+
|
|
99
|
+
function toMs(ts: unknown): number | null {
|
|
100
|
+
if (typeof ts !== 'string') return null
|
|
101
|
+
const ms = Date.parse(ts)
|
|
102
|
+
return Number.isFinite(ms) ? ms : null
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The claude transcript's turn state, or null if the file states nothing usable.
|
|
107
|
+
*
|
|
108
|
+
* Only `tool_use` continues a turn; every other stop_reason ends it (see the header on why the
|
|
109
|
+
* polarity is asymmetric). Compaction summaries, sidechain lines and meta lines are excluded —
|
|
110
|
+
* each is a measured false-working vector.
|
|
111
|
+
*/
|
|
112
|
+
export function parseClaudeTurn(text: string): TurnState | null {
|
|
113
|
+
let found: TurnState | null = null
|
|
114
|
+
for (const line of text.split('\n')) {
|
|
115
|
+
const s = line.trim()
|
|
116
|
+
if (!s.startsWith('{')) continue // a partial first line from the tail cut
|
|
117
|
+
let d: Record<string, unknown>
|
|
118
|
+
try {
|
|
119
|
+
d = JSON.parse(s) as Record<string, unknown>
|
|
120
|
+
} catch {
|
|
121
|
+
continue // never fail a sweep over one garbled line
|
|
122
|
+
}
|
|
123
|
+
const type = d.type
|
|
124
|
+
if (type !== 'assistant' && type !== 'user') continue
|
|
125
|
+
if (d.isSidechain === true) continue // a sub-agent's chain, not the peer's turn
|
|
126
|
+
if (d.isCompactSummary === true) continue // compaction artifact, not the model being asked
|
|
127
|
+
if (d.isMeta === true) continue // injected, not a real prompt
|
|
128
|
+
const atMs = toMs(d.timestamp)
|
|
129
|
+
if (atMs === null) continue
|
|
130
|
+
const cwd = typeof d.cwd === 'string' ? d.cwd : ''
|
|
131
|
+
if (!cwd) continue // unattributable to a peer → ignore rather than guess
|
|
132
|
+
let state: 'working' | 'idle'
|
|
133
|
+
if (type === 'assistant') {
|
|
134
|
+
const msg = d.message as { stop_reason?: unknown } | undefined
|
|
135
|
+
// ONLY tool_use continues. end_turn / stop_sequence / max_tokens / refusal / an unknown
|
|
136
|
+
// future value / null all mean the assistant stopped.
|
|
137
|
+
state = msg?.stop_reason === 'tool_use' ? 'working' : 'idle'
|
|
138
|
+
} else {
|
|
139
|
+
// A real user line: a prompt to answer or a tool_result to consume — either way the
|
|
140
|
+
// model is on the hook. (The non-model user lines were filtered above.)
|
|
141
|
+
state = 'working'
|
|
142
|
+
}
|
|
143
|
+
found = { state, atMs, cwd }
|
|
144
|
+
}
|
|
145
|
+
return found
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The codex rollout's turn state, or null. `task_started` / `task_complete` are explicit
|
|
150
|
+
* boundaries — the last one wins. `cwd` comes from `session_meta` (line 1), so pass the file
|
|
151
|
+
* HEAD when the tail may have cut it away.
|
|
152
|
+
*/
|
|
153
|
+
export function parseCodexTurn(text: string, head = ''): TurnState | null {
|
|
154
|
+
let cwd = ''
|
|
155
|
+
const scanMeta = (src: string): void => {
|
|
156
|
+
for (const line of src.split('\n')) {
|
|
157
|
+
const s = line.trim()
|
|
158
|
+
if (!s.startsWith('{') || !s.includes('session_meta')) continue
|
|
159
|
+
try {
|
|
160
|
+
const d = JSON.parse(s) as { type?: string; payload?: { cwd?: unknown } }
|
|
161
|
+
if (d.type !== 'session_meta' || !d.payload) continue
|
|
162
|
+
if (typeof d.payload.cwd === 'string') cwd = d.payload.cwd
|
|
163
|
+
} catch {
|
|
164
|
+
/* ignore */
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
scanMeta(head)
|
|
169
|
+
if (!cwd) scanMeta(text)
|
|
170
|
+
if (!cwd) return null // unattributable to a peer
|
|
171
|
+
|
|
172
|
+
let found: TurnState | null = null
|
|
173
|
+
for (const line of text.split('\n')) {
|
|
174
|
+
const s = line.trim()
|
|
175
|
+
if (!s.startsWith('{') || !s.includes('task_')) continue
|
|
176
|
+
let d: { timestamp?: unknown; type?: unknown; payload?: { type?: unknown } }
|
|
177
|
+
try {
|
|
178
|
+
d = JSON.parse(s)
|
|
179
|
+
} catch {
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
if (d.type !== 'event_msg') continue
|
|
183
|
+
const p = d.payload?.type
|
|
184
|
+
if (p !== 'task_started' && p !== 'task_complete') continue
|
|
185
|
+
const atMs = toMs(d.timestamp)
|
|
186
|
+
if (atMs === null) continue
|
|
187
|
+
found = { state: p === 'task_started' ? 'working' : 'idle', atMs, cwd }
|
|
188
|
+
}
|
|
189
|
+
return found
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
193
|
+
// The board — in-memory (personality, runtime) → last known turn state
|
|
194
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
195
|
+
|
|
196
|
+
function key(personality: string, runtime: string): string {
|
|
197
|
+
return `${runtime}-${personality}` // the identity scheme (docs/03)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Last known turn state per (personality, runtime). In-memory and ephemeral, exactly like the
|
|
202
|
+
* notice board: a restart re-seeds from the session files, which are the durable truth.
|
|
203
|
+
*/
|
|
204
|
+
export class ActivityBoard {
|
|
205
|
+
private readonly states = new Map<string, TurnState>()
|
|
206
|
+
|
|
207
|
+
/** Record an observation. Returns the PREVIOUS state, so the caller can log only real
|
|
208
|
+
* transitions rather than one line per sweep. */
|
|
209
|
+
set(personality: string, runtime: string, next: TurnState): TurnState | undefined {
|
|
210
|
+
const k = key(personality, runtime)
|
|
211
|
+
const prev = this.states.get(k)
|
|
212
|
+
// Evidence older than what we already hold is a stale re-read — never let it rewind state.
|
|
213
|
+
if (prev && next.atMs < prev.atMs) return prev
|
|
214
|
+
this.states.set(k, next)
|
|
215
|
+
return prev
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** The operator-facing verdict, with the stale bound applied at READ time (so a claim decays
|
|
219
|
+
* even while nothing new is swept). `undefined` → we have no evidence at all. */
|
|
220
|
+
get(personality: string, runtime: string, nowMs: number): TurnActivity | undefined {
|
|
221
|
+
const st = this.states.get(key(personality, runtime))
|
|
222
|
+
if (!st) return undefined
|
|
223
|
+
if (st.state === 'idle') return 'idle' // idle never decays — age proves nothing against it
|
|
224
|
+
return nowMs - st.atMs > TURN_STALE_MS ? 'unknown' : 'working'
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** The raw record (peer-detail / tests). */
|
|
228
|
+
raw(personality: string, runtime: string): TurnState | undefined {
|
|
229
|
+
return this.states.get(key(personality, runtime))
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
size(): number {
|
|
233
|
+
return this.states.size
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
238
|
+
// The sweep
|
|
239
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
240
|
+
|
|
241
|
+
function readTail(path: string): { tail: string; head: string } | null {
|
|
242
|
+
try {
|
|
243
|
+
const fd = openSync(path, 'r')
|
|
244
|
+
try {
|
|
245
|
+
const size = fstatSync(fd).size
|
|
246
|
+
if (size <= 0) return null
|
|
247
|
+
const off = Math.max(0, size - TAIL_BYTES)
|
|
248
|
+
const buf = Buffer.alloc(size - off)
|
|
249
|
+
readSync(fd, buf, 0, buf.length, off)
|
|
250
|
+
let head = ''
|
|
251
|
+
if (off > 0) {
|
|
252
|
+
const hb = Buffer.alloc(Math.min(HEAD_BYTES, off))
|
|
253
|
+
readSync(fd, hb, 0, hb.length, 0)
|
|
254
|
+
head = hb.toString('utf8')
|
|
255
|
+
}
|
|
256
|
+
return { tail: buf.toString('utf8'), head }
|
|
257
|
+
} finally {
|
|
258
|
+
closeSync(fd)
|
|
259
|
+
}
|
|
260
|
+
} catch {
|
|
261
|
+
return null
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function safeReaddir(dir: string): string[] {
|
|
266
|
+
try {
|
|
267
|
+
return readdirSync(dir)
|
|
268
|
+
} catch {
|
|
269
|
+
return []
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** *.jsonl under `root` (depth-bounded) whose mtime is at/after `sinceMs`. */
|
|
274
|
+
function changedJsonl(root: string, sinceMs: number, depth: number): string[] {
|
|
275
|
+
if (!existsSync(root)) return []
|
|
276
|
+
const out: string[] = []
|
|
277
|
+
const walk = (dir: string, left: number): void => {
|
|
278
|
+
for (const name of safeReaddir(dir)) {
|
|
279
|
+
const p = join(dir, name)
|
|
280
|
+
if (name.endsWith('.jsonl')) {
|
|
281
|
+
let m: number | null = null
|
|
282
|
+
try {
|
|
283
|
+
m = statSync(p).mtimeMs
|
|
284
|
+
} catch {
|
|
285
|
+
continue
|
|
286
|
+
}
|
|
287
|
+
if (m >= sinceMs) out.push(p)
|
|
288
|
+
continue
|
|
289
|
+
}
|
|
290
|
+
if (left <= 0 || name.startsWith('.')) continue
|
|
291
|
+
try {
|
|
292
|
+
if (statSync(p).isDirectory()) walk(p, left - 1)
|
|
293
|
+
} catch {
|
|
294
|
+
continue
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
walk(root, depth)
|
|
299
|
+
return out
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export interface TurnWatchPaths {
|
|
303
|
+
/** ~/.claude/projects — INJECTED, never re-resolved from env inside a sweep. */
|
|
304
|
+
claudeProjectsDir: string
|
|
305
|
+
/** ~/.codex/sessions — same rule. */
|
|
306
|
+
codexSessionsDir: string
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export function resolveTurnWatchPaths(env: NodeJS.ProcessEnv): TurnWatchPaths {
|
|
310
|
+
const home = env.HOME?.trim() || ''
|
|
311
|
+
return {
|
|
312
|
+
claudeProjectsDir: env.IAPEER_CLAUDE_PROJECTS_DIR?.trim() || join(home, '.claude', 'projects'),
|
|
313
|
+
codexSessionsDir: env.IAPEER_CODEX_SESSIONS_DIR?.trim() || join(home, '.codex', 'sessions'),
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export interface TurnScanDeps extends TurnWatchPaths {
|
|
318
|
+
/** personality ← cwd. Built from the registry by the composition point. */
|
|
319
|
+
peerByCwd: (cwd: string) => string | undefined
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
export interface TurnObservation extends TurnState {
|
|
323
|
+
personality: string
|
|
324
|
+
runtime: 'claude' | 'codex'
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** One sweep: the turn state of every peer session file touched since `sinceMs`. */
|
|
328
|
+
export function scanTurnStates(deps: TurnScanDeps, sinceMs: number): TurnObservation[] {
|
|
329
|
+
const floor = sinceMs - SCAN_OVERLAP_MS
|
|
330
|
+
const out: TurnObservation[] = []
|
|
331
|
+
const push = (st: TurnState | null, runtime: 'claude' | 'codex'): void => {
|
|
332
|
+
if (!st) return
|
|
333
|
+
const personality = deps.peerByCwd(st.cwd)
|
|
334
|
+
if (!personality) return // a human's own session — never reported
|
|
335
|
+
out.push({ ...st, personality, runtime })
|
|
336
|
+
}
|
|
337
|
+
for (const file of changedJsonl(deps.claudeProjectsDir, floor, 1)) {
|
|
338
|
+
const t = readTail(file)
|
|
339
|
+
if (t) push(parseClaudeTurn(t.tail), 'claude')
|
|
340
|
+
}
|
|
341
|
+
for (const file of changedJsonl(deps.codexSessionsDir, floor, CODEX_WALK_DEPTH)) {
|
|
342
|
+
const t = readTail(file)
|
|
343
|
+
if (t) push(parseCodexTurn(t.tail, t.head), 'codex')
|
|
344
|
+
}
|
|
345
|
+
return out
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export interface TurnWatchDeps extends TurnScanDeps {
|
|
349
|
+
board: ActivityBoard
|
|
350
|
+
/** cfg.eventLogDir — turns.log (falsy → no durable log, hermetic). */
|
|
351
|
+
logDir?: string
|
|
352
|
+
env?: NodeJS.ProcessEnv
|
|
353
|
+
now?: () => number
|
|
354
|
+
onError?: (err: unknown) => void
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Run one watch tick: sweep, fold into the board, log REAL transitions only.
|
|
359
|
+
* Returns how many transitions were logged plus the new since-boundary.
|
|
360
|
+
*/
|
|
361
|
+
export function turnWatchTick(deps: TurnWatchDeps, sinceMs: number): { transitions: number; sinceMs: number } {
|
|
362
|
+
const now = deps.now ?? Date.now
|
|
363
|
+
const startedMs = now()
|
|
364
|
+
let transitions = 0
|
|
365
|
+
try {
|
|
366
|
+
for (const o of scanTurnStates(deps, sinceMs)) {
|
|
367
|
+
const prev = deps.board.set(o.personality, o.runtime, { state: o.state, atMs: o.atMs, cwd: o.cwd })
|
|
368
|
+
// Only a genuine change is an event. Without this the 3 s sweep would write a line per
|
|
369
|
+
// pass per peer and bury the log it is supposed to make readable.
|
|
370
|
+
if (prev?.state === o.state) continue
|
|
371
|
+
// A rewound observation (older evidence than held) is not a transition either.
|
|
372
|
+
if (prev && o.atMs < prev.atMs) continue
|
|
373
|
+
transitions += 1
|
|
374
|
+
appendTurnEvent(
|
|
375
|
+
deps.logDir,
|
|
376
|
+
{
|
|
377
|
+
ev: o.state === 'working' ? 'turn-started' : 'turn-ended',
|
|
378
|
+
personality: o.personality,
|
|
379
|
+
runtime: o.runtime,
|
|
380
|
+
from: prev?.state,
|
|
381
|
+
at: o.atMs,
|
|
382
|
+
at_iso: new Date(o.atMs).toISOString(),
|
|
383
|
+
},
|
|
384
|
+
{ env: deps.env, nowMs: startedMs },
|
|
385
|
+
)
|
|
386
|
+
}
|
|
387
|
+
} catch (e) {
|
|
388
|
+
deps.onError?.(e)
|
|
389
|
+
}
|
|
390
|
+
return { transitions, sinceMs: startedMs }
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export interface StartTurnWatchOptions {
|
|
394
|
+
env?: NodeJS.ProcessEnv
|
|
395
|
+
intervalMs?: number
|
|
396
|
+
paths?: TurnWatchPaths
|
|
397
|
+
listPeers?: () => Array<{ personality: string; cwd: string }>
|
|
398
|
+
now?: () => number
|
|
399
|
+
onError?: (err: unknown) => void
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* Start the turn-watch timer. Returns its stop function — the caller OWNS teardown.
|
|
404
|
+
* The FIRST sweep reaches back SEED_LOOKBACK_MS so sessions already mid-turn when the daemon
|
|
405
|
+
* started are known immediately; every later sweep is incremental (changed files only).
|
|
406
|
+
*/
|
|
407
|
+
export function startTurnWatch(cfg: LifecycleConfig, board: ActivityBoard, opts: StartTurnWatchOptions = {}): () => void {
|
|
408
|
+
const env = opts.env ?? process.env
|
|
409
|
+
const paths = opts.paths ?? resolveTurnWatchPaths(env)
|
|
410
|
+
const now = opts.now ?? Date.now
|
|
411
|
+
let listPeers = opts.listPeers
|
|
412
|
+
let since = now() - SEED_LOOKBACK_MS
|
|
413
|
+
let running = false
|
|
414
|
+
|
|
415
|
+
const sweep = async (): Promise<void> => {
|
|
416
|
+
if (running) return // a slow sweep must never overlap itself into a stampede
|
|
417
|
+
running = true
|
|
418
|
+
try {
|
|
419
|
+
if (!listPeers) {
|
|
420
|
+
const cli = await import('../cli/index.ts')
|
|
421
|
+
listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
|
|
422
|
+
}
|
|
423
|
+
const rows = listPeers()
|
|
424
|
+
// Case-insensitive, trailing-slash-tolerant — same rule and reason as mutewatch.
|
|
425
|
+
const byCwd = new Map<string, string>()
|
|
426
|
+
for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
|
|
427
|
+
since = turnWatchTick(
|
|
428
|
+
{
|
|
429
|
+
...paths,
|
|
430
|
+
board,
|
|
431
|
+
logDir: cfg.eventLogDir,
|
|
432
|
+
env,
|
|
433
|
+
now,
|
|
434
|
+
onError: opts.onError,
|
|
435
|
+
peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
|
|
436
|
+
},
|
|
437
|
+
since,
|
|
438
|
+
).sinceMs
|
|
439
|
+
} catch (e) {
|
|
440
|
+
opts.onError?.(e)
|
|
441
|
+
} finally {
|
|
442
|
+
running = false
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
void sweep() // seed immediately — do not make the operator wait one interval for the truth
|
|
447
|
+
const timer = setInterval(() => void sweep(), opts.intervalMs ?? 3_000)
|
|
448
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
449
|
+
return () => clearInterval(timer)
|
|
450
|
+
}
|