@agfpd/iapeer 0.4.92 → 0.4.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/15-fleet-api.md +12 -4
- package/docs/19-notices.md +201 -0
- package/docs/README.md +1 -0
- package/package.json +1 -1
- package/src/daemon/fleet.ts +77 -1
- package/src/daemon/main.ts +34 -2
- package/src/daemon/mutewatch.ts +441 -0
- package/src/daemon/notices.ts +208 -0
- package/src/daemon/noticeslog.ts +54 -0
package/docs/15-fleet-api.md
CHANGED
|
@@ -63,6 +63,13 @@ The full fleet state. The peer rows are produced by the **same in-process functi
|
|
|
63
63
|
{ "id": "a1", "personality": "boris", "runtime": "claude", "kind": "circuit-breaker",
|
|
64
64
|
"tool": "dangerous-rm", "summary": "rm -rf /tmp/x",
|
|
65
65
|
"content": "cmd=\"rm -rf /tmp/x\" target=\"/tmp/x\"", "createdMs": 1751731100000, "expiresMs": 1751731400000 }
|
|
66
|
+
],
|
|
67
|
+
"notices": [
|
|
68
|
+
{ "id": "n1", "personality": "boris", "runtime": "claude", "kind": "peer-mute",
|
|
69
|
+
"errorType": "rate_limit", "model": "Fable 5", "summary": "boris · claude — rate_limit (Fable 5)",
|
|
70
|
+
"content": "You've reached your Fable 5 limit. Run /usage-credits to continue or switch models with /model.",
|
|
71
|
+
"sessionId": "85ea79c9-…", "createdMs": 1751731100000, "lastMs": 1751731160000,
|
|
72
|
+
"expiresMs": 1751734700000, "count": 3 }
|
|
66
73
|
]
|
|
67
74
|
}
|
|
68
75
|
```
|
|
@@ -81,6 +88,7 @@ Field notes:
|
|
|
81
88
|
- The daemon answering **is** the health signal: there is no `healthy` boolean. A dashboard that cannot reach any advertised address should render the daemon as down (this is also why a purely API-fed dashboard is insufficient when the daemon itself is broken — the built-in TUI keeps its direct in-process read path for exactly that case).
|
|
82
89
|
- The peer's **LLM model is deliberately absent**: the effective model of a live session is not an observable registry fact (static launch pins exist, but reporting a pin as «the model» would lie under runtime auto-switching). If it ever becomes observable, it will appear as an additive field.
|
|
83
90
|
- `approvals` (top-level, docs/17-approval) — the pending human-approval queue, the SAME items `GET /fleet/v1/approvals` returns, carried in the snapshot so a client that already re-fetches `/snapshot` on every event renders the queue with no extra call. **ADDITIVE + OMITTED when the queue is empty** — a client MUST treat its absence as an empty queue (the same rule as `approval_mode`). Each item: `id` (broker id, the target of `POST /fleet/v1/approvals/<id>/(approve|deny)`), `personality` + `runtime` (whose action), `kind` (`tool`|`plan`|`question`|`circuit-breaker`), `tool` (the tool / breaker name), `summary` (a one-line badge string), `content` (the FULL verbatim action — command / diff / plan; criterion #7), `createdMs` / `expiresMs`. The broker's item is a superset — unknown extra fields (`title`, `approvers`) MUST be ignored (obligation 2).
|
|
91
|
+
- `notices` (top-level, docs/19-notices) — live OWNER-facing notices: things the daemon noticed that the peer itself could not report, the v1 case being a structural API error that left it MUTE (`kind: peer-mute`). The SAME items `GET /fleet/v1/notices` returns. **ADDITIVE + OMITTED when the board is empty** — absence ⇒ empty board (the same rule as `approvals`). **READ-ONLY**: a notice is one-way, there is nothing to approve or deny. Each item: `id`, `personality` + `runtime` (who went mute), `kind` (`peer-mute`; treat unknown kinds as renderable), `errorType` (the RUNTIME's own value — `rate_limit`, `overloaded`, `rate_limit_reached`, … — free-form, do not switch exhaustively), `model?`, `resetsAtMs?`, `summary`, `content` (verbatim for claude), `sessionId?`, `createdMs` / `lastMs` / `expiresMs`, `count` (detections folded in — render as `×N`). **`model` and `resetsAtMs` are ABSENT when the runtime did not state them** — absence means *the runtime did not say*, NOT *there is no limit*; a client MUST render unknown and MUST NOT substitute the 5h/7d reset, which belongs to a different limit (docs/19 §3).
|
|
84
92
|
|
|
85
93
|
## GET /fleet/v1/peers/<personality>
|
|
86
94
|
|
|
@@ -95,7 +103,7 @@ One peer's card: the same object as the snapshot row, plus its recent history.
|
|
|
95
103
|
|
|
96
104
|
## GET /fleet/v1/events — SSE stream
|
|
97
105
|
|
|
98
|
-
`text/event-stream` of fleet events, sourced by tail-following the daemon's **durable logs** (`lifecycle.log`, `delivery.log`, `exits.log`). Those files are written by *every* participating process — the daemon, CLI verbs (attach/stop/wake), the pty supervisors — so the stream covers events an in-daemon bus would miss.
|
|
106
|
+
`text/event-stream` of fleet events, sourced by tail-following the daemon's **durable logs** (`lifecycle.log`, `delivery.log`, `exits.log`, `approvals.log`, `notices.log`). Those files are written by *every* participating process — the daemon, CLI verbs (attach/stop/wake), the pty supervisors — so the stream covers events an in-daemon bus would miss.
|
|
99
107
|
|
|
100
108
|
```
|
|
101
109
|
GET /fleet/v1/events?replay=50
|
|
@@ -110,11 +118,11 @@ GET /fleet/v1/events?replay=50
|
|
|
110
118
|
data: {"src":"lifecycle","ev":"wake","ts":"2026-07-05T18:00:00.123Z","personality":"boris","mode":"resume"}
|
|
111
119
|
```
|
|
112
120
|
|
|
113
|
-
`data` is the full parsed log line: `src` (`lifecycle` | `delivery` | `exits`) + `ev` + every logfmt field with quoted values decoded. `id` = epoch-ms of the event.
|
|
121
|
+
`data` is the full parsed log line: `src` (`lifecycle` | `delivery` | `exits` | `approvals` | `notices`) + `ev` + every logfmt field with quoted values decoded. `id` = epoch-ms of the event.
|
|
114
122
|
- A comment `: connected` marks the transition from replay to live; `: hb` comments every ~15 s keep the connection alive.
|
|
115
123
|
- Latency: live events surface within ~0.5 s (log-tail poll).
|
|
116
124
|
|
|
117
|
-
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving).
|
|
125
|
+
Current `ev` vocabulary (grows over time — see obligation 1): `wake`, `supervise` (with `action`: `reaped-idle` = parked, `reaped-ephemeral`, `dead`→fresh/resume classification, `eager-orphan-fresh`, `skipped-error`…), `stopped` / `started` (the stop/start verbs — an operator changed a peer's ✕/○ state; `action` is `stopped`/`started` for warm runtimes, `bootout`/`bootstrap` for always-on, with `reason` on failure), `delivery`, `topic-note`, `ephemeral-drain`, `attach` / `attach-end`, `hosted-deliver`, `pty-host-failed`, `supervise-error`, `composer-queue-failed-notify`, `memory-provision`, `session-exit` (from `exits.log`: `cause=child-exit` with `dead_status`/`dead_signal` when the session's process died on its own, `cause=shutdown-sigterm`/`shutdown-sigint` when the supervisor was torn down — reap/stop/new), `supervisor-uncaught` (a survivable supervisor fault left as forensics; the session keeps serving), `notice-raised` (docs/19 — the daemon told the OWNER something a peer could not tell him itself: a structural API error left it mute. One line per condition, not per occurrence — repeats fold into the live notice silently).
|
|
118
126
|
|
|
119
127
|
What the headline states look like on the stream: a peer **waking** is `ev=wake`; a peer **parked to sleep** is `ev=supervise action=reaped-idle`; an operator **stop/start** is `ev=stopped` / `ev=started`; a **death** is `ev=session-exit` (immediately, from the supervisor) followed by the daemon's classification on its next pass.
|
|
120
128
|
|
|
@@ -140,7 +148,7 @@ What the headline states look like on the stream: a peer **waking** is `ev=wake`
|
|
|
140
148
|
|
|
141
149
|
`from` is optional: a full `<runtime>-<personality>`, or a bare personality (its default runtime). Default: the **single natural-intelligence peer** of the registry — a GUI-originated message is the human's. With zero or several natural peers the request is refused (`400`) with an instruction to pass `from`.
|
|
142
150
|
|
|
143
|
-
Not part of the *command* API, deliberately: **attach** (a terminal handoff — a client opens the system terminal with `iapeer attach <peer>`), registry/host **admin verbs** (`create`/`remove`/`update`/`uninstall` — will be added when a client actually needs them). The **human-approval broker** now lives under `/fleet/v1/approvals*` (`GET` the queue / one item, `POST /<id>/(approve|deny)`) with the `approval-request` / `approval-resolved` SSE `ev` kinds and the `approvals` snapshot field above — the full contract is **docs/17-approval**.
|
|
151
|
+
Not part of the *command* API, deliberately: **attach** (a terminal handoff — a client opens the system terminal with `iapeer attach <peer>`), registry/host **admin verbs** (`create`/`remove`/`update`/`uninstall` — will be added when a client actually needs them). The **human-approval broker** now lives under `/fleet/v1/approvals*` (`GET` the queue / one item, `POST /<id>/(approve|deny)`) with the `approval-request` / `approval-resolved` SSE `ev` kinds and the `approvals` snapshot field above — the full contract is **docs/17-approval**. The **notice board** lives under `/fleet/v1/notices*` (`GET` the board / one item — **read-only**: a notice is one-way and has nothing to resolve) with the `notice-raised` SSE `ev` kind and the `notices` snapshot field above — the full contract is **docs/19-notices**.
|
|
144
152
|
|
|
145
153
|
Errors are always `{"error": "…"}` with an honest status: `400` malformed request, `401` bearer required, `404` unknown peer/endpoint, `405` wrong method, `502` the verb itself failed, `500` internal.
|
|
146
154
|
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# 19 — Notices: telling the owner a peer went mute
|
|
2
|
+
|
|
3
|
+
A peer can fail in a way that leaves it **alive, healthy by every signal the daemon has, and
|
|
4
|
+
unable to say a word**. Nobody inside that session can report it — that is precisely what
|
|
5
|
+
broke. So the daemon reports it, on the owner's own surfaces, by the same principle an
|
|
6
|
+
approval request travels.
|
|
7
|
+
|
|
8
|
+
This note is the normative contract for the daemon side and for any face that renders
|
|
9
|
+
notices. Client obligations are in §4.
|
|
10
|
+
|
|
11
|
+
## 1. Why a notice is not an approval
|
|
12
|
+
|
|
13
|
+
The approval broker (docs/17) and the notice board look alike and are not.
|
|
14
|
+
|
|
15
|
+
| | Approval | Notice |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| Direction | request → decision → unblock | one-way |
|
|
18
|
+
| Someone waits | yes — a hook long-polls, a tool is blocked | no |
|
|
19
|
+
| A human must act | yes, or a timeout default-denies | no — it is information |
|
|
20
|
+
| Faces can write | `POST …/approve` \| `…/deny` | nothing to write; **read-only** |
|
|
21
|
+
| Lifetime | until answered or timed out | until TTL |
|
|
22
|
+
|
|
23
|
+
Reusing the broker would drag a promise/timeout/fail-safe-deny machine into a problem with
|
|
24
|
+
none of those, and would invite a face to "resolve" something that has no resolution.
|
|
25
|
+
|
|
26
|
+
What the two DO share is the **surface**: a durable log tailed into the fleet SSE stream, an
|
|
27
|
+
additive snapshot field, and a GET endpoint for the verbatim content. A face that already
|
|
28
|
+
renders approvals renders notices with the same plumbing.
|
|
29
|
+
|
|
30
|
+
## 2. The signal: `peer-mute`
|
|
31
|
+
|
|
32
|
+
The v1 notice kind. It means: *a structural API error left this peer unable to answer.*
|
|
33
|
+
|
|
34
|
+
The daemon does not infer this from silence — silence is unfalsifiable. It reads the fact
|
|
35
|
+
**structurally** out of the runtime's own session file. Neither runtime is scraped from the
|
|
36
|
+
pane: the refusal there is prose that line-wraps, while both runtimes write the fact as data.
|
|
37
|
+
|
|
38
|
+
### claude — the transcript
|
|
39
|
+
|
|
40
|
+
`~/.claude/projects/<slug>/<sessionId>.jsonl`, on the line the runtime wrote when it refused:
|
|
41
|
+
|
|
42
|
+
```json
|
|
43
|
+
{ "isApiErrorMessage": true, "error": "rate_limit", "apiErrorStatus": 429,
|
|
44
|
+
"message": { "model": "<synthetic>", "content": [{ "type": "text",
|
|
45
|
+
"text": "You've reached your Fable 5 limit. Run /usage-credits to continue or switch models with /model." }] } }
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The detector keys on **`isApiErrorMessage: true`** — the CLASS — and reports `error`
|
|
49
|
+
verbatim as `errorType`. `rate_limit` is one value; `overloaded`, an expired auth and any
|
|
50
|
+
future member arrive on the same field and take the same path. `error` is a locale-stable
|
|
51
|
+
enum, so no prose is matched.
|
|
52
|
+
|
|
53
|
+
### codex — the rollout
|
|
54
|
+
|
|
55
|
+
`~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`, in an `event_msg` / `token_count` payload:
|
|
56
|
+
|
|
57
|
+
```json
|
|
58
|
+
{ "rate_limits": { "limit_id": "codex", "plan_type": "plus",
|
|
59
|
+
"primary": { "used_percent": 1.0, "window_minutes": 300, "resets_at": 1783114795 },
|
|
60
|
+
"secondary": { "used_percent": 17.0, "window_minutes": 10080, "resets_at": 1783412915 },
|
|
61
|
+
"rate_limit_reached_type": null } }
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`rate_limit_reached_type` is `null` while healthy and non-null once a wall is hit. The
|
|
65
|
+
detector keys on **non-null**, never on the variant strings (`rate_limit_reached`,
|
|
66
|
+
`workspace_owner_usage_limit_reached`, `workspace_member_credits_depleted`, … — read out of
|
|
67
|
+
the 0.144.1 binary), so a variant we have never seen still detects and reports itself.
|
|
68
|
+
|
|
69
|
+
## 3. What the owner is told — and what is deliberately withheld
|
|
70
|
+
|
|
71
|
+
A notice states who, which runtime, which error type, which model, and when the wall lifts.
|
|
72
|
+
Two of those are **absent by design rather than guessed**:
|
|
73
|
+
|
|
74
|
+
- **`resetsAtMs` on claude.** Claude does not say when a per-model bucket lifts. Its
|
|
75
|
+
`errorDetails` is a raw 429 reading *"Please try again later"* — no stamp. The 5h/7d
|
|
76
|
+
`resets_at` in the statusline blob belongs to a **different limit** and must never be
|
|
77
|
+
substituted: measured live on 15.07.2026, the 5h bucket sat at **11 %** and the 7d at
|
|
78
|
+
**66 %** while fable was fully exhausted. Codex does state its reset, so its notices carry
|
|
79
|
+
one. The asymmetry is real and stays visible.
|
|
80
|
+
- **`model` on codex.** The rollout snapshot names no model. Claude names it only in prose,
|
|
81
|
+
so it is lifted from the runtime's own sentence when no real model reply precedes the
|
|
82
|
+
error; a miss omits the field.
|
|
83
|
+
|
|
84
|
+
Absent field ⇒ *the runtime did not say*. Never ⇒ *there is no limit*.
|
|
85
|
+
|
|
86
|
+
`content` is the runtime's **verbatim** refusal for claude. For codex the rollout carries no
|
|
87
|
+
prose, so `content` is rendered from its typed fields (both windows, their percentages and
|
|
88
|
+
resets, the plan) — data, never our interpretation of the cause.
|
|
89
|
+
|
|
90
|
+
## 4. Fleet API
|
|
91
|
+
|
|
92
|
+
### `GET /fleet/v1/notices`
|
|
93
|
+
|
|
94
|
+
```json
|
|
95
|
+
{ "api": 1, "version": "0.4.93", "ts": "2026-07-15T21:10:00.000Z", "notices": [ { … } ] }
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### `GET /fleet/v1/notices/<id>`
|
|
99
|
+
|
|
100
|
+
```json
|
|
101
|
+
{ "api": 1, "version": "0.4.93", "ts": "…", "notice": { … } }
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
404 when the id is unknown or its TTL has passed. There is **no** POST on either route — see §1.
|
|
105
|
+
|
|
106
|
+
### The notice object
|
|
107
|
+
|
|
108
|
+
| field | type | notes |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `id` | string | `n1`, `n2`, … Unique while live; **not** stable across a daemon restart. |
|
|
111
|
+
| `personality` | string | the mute peer |
|
|
112
|
+
| `runtime` | string | `claude` \| `codex` |
|
|
113
|
+
| `kind` | string | `peer-mute` in v1. **Treat unknown kinds as renderable** — new kinds ride this surface. |
|
|
114
|
+
| `errorType` | string | the runtime's OWN value (`rate_limit`, `overloaded`, `rate_limit_reached`, …). Free-form: do not switch exhaustively. |
|
|
115
|
+
| `model` | string? | **absent** when the runtime did not name it |
|
|
116
|
+
| `resetsAtMs` | number? | epoch-ms; **absent** when the runtime did not state it → render "unknown", never substitute |
|
|
117
|
+
| `summary` | string | one line, e.g. `boris · claude — rate_limit (Fable 5)` |
|
|
118
|
+
| `content` | string | verbatim (claude) / rendered from typed fields (codex) |
|
|
119
|
+
| `sessionId` | string? | correlates with the on-disk transcript |
|
|
120
|
+
| `createdMs` | number | first occurrence |
|
|
121
|
+
| `lastMs` | number | latest OCCURRENCE (not the latest sweep that re-read it) |
|
|
122
|
+
| `expiresMs` | number | TTL boundary |
|
|
123
|
+
| `count` | number | distinct OCCURRENCES folded in (≥1) — how many times the peer actually hit the wall. Render as `×N`. The sweep window overlaps on purpose, so the same runtime event is re-read across passes; the board discriminates on the event's own timestamp, and a re-read never bumps this. |
|
|
124
|
+
|
|
125
|
+
### Snapshot
|
|
126
|
+
|
|
127
|
+
`GET /fleet/v1/snapshot` carries `notices?: FleetNotice[]` — the same items, so a client that
|
|
128
|
+
already fetches the snapshot on every event needs no extra call. **ADDITIVE and omitted when
|
|
129
|
+
the board is empty**: absence ⇒ empty board (the same rule as `approvals`).
|
|
130
|
+
|
|
131
|
+
### Events (SSE)
|
|
132
|
+
|
|
133
|
+
`notices.log` is one of the tailed durable logs, so `GET /fleet/v1/events` pushes:
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
ev=notice-raised id=n1 personality=boris runtime=claude kind=peer-mute error_type=rate_limit
|
|
137
|
+
model="Fable 5" session=… summary="…" created=… expires=…
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
`resets_at` + `resets_at_iso` appear **only** when the runtime stated a reset. At-least-once;
|
|
141
|
+
ignore unknown `ev` kinds as always.
|
|
142
|
+
|
|
143
|
+
### Client obligations
|
|
144
|
+
|
|
145
|
+
1. **Render, never resolve.** No approve/deny.
|
|
146
|
+
2. **Absent `resetsAtMs` ⇒ unknown.** Never substitute the 5h/7d reset — different limit.
|
|
147
|
+
3. **Absent `notices` ⇒ empty board.**
|
|
148
|
+
4. **Unknown `kind` / `errorType` ⇒ still render.** Both are growth seams.
|
|
149
|
+
5. **Do not key on `id` across daemon restarts** — the board is in-memory; a restart
|
|
150
|
+
re-detects from the on-disk evidence and issues new ids.
|
|
151
|
+
|
|
152
|
+
## 5. Dedup and TTL — one mechanism
|
|
153
|
+
|
|
154
|
+
A mute peer re-emits its error on **every** attempted turn. So a notice carries a dedup key —
|
|
155
|
+
`personality | runtime | kind | errorType | model` — and while a notice with that key is live,
|
|
156
|
+
a repeat detection only bumps `count` and `lastMs`: **no new id, no new log line, no new SSE
|
|
157
|
+
event**. The owner sees one card saying `×7`, not seven cards.
|
|
158
|
+
|
|
159
|
+
`count` means **occurrences, not sweeps**. The sweep window deliberately overlaps (so an event
|
|
160
|
+
landing between two passes is never missed), which means the same transcript line is re-read on
|
|
161
|
+
the next pass. The board therefore discriminates on the runtime event's OWN timestamp: a re-read
|
|
162
|
+
of an already-counted event folds silently and bumps nothing. Without that, two real refusals
|
|
163
|
+
would render as `×3` — a number the owner cannot check and that we would have made up.
|
|
164
|
+
|
|
165
|
+
When the TTL passes (`IAPEER_NOTICE_TTL_MS`, default 1 h) the notice expires silently. If the
|
|
166
|
+
peer is still broken, the next detection raises a **fresh** notice — a deliberate periodic
|
|
167
|
+
reminder rather than one card that scrolls away forever. Expiry is not an event: either the
|
|
168
|
+
peer recovered (nothing to say) or it is still broken and will re-raise.
|
|
169
|
+
|
|
170
|
+
The board is in-memory; `notices.log` is the audit trace, not a recovery store. A restart
|
|
171
|
+
re-detects from the transcript evidence still on disk.
|
|
172
|
+
|
|
173
|
+
## 6. Detection, and why paths are not derived from `cwd`
|
|
174
|
+
|
|
175
|
+
The daemon sweeps on its own timer (`DEFAULT_MUTEWATCH_INTERVAL_MS`, 20 s — detection latency
|
|
176
|
+
is the product here, and the 60 s supervise tick would put the worst case at the budget with
|
|
177
|
+
nothing to spare). A sweep only opens files whose mtime moved since the last pass.
|
|
178
|
+
|
|
179
|
+
The obvious route — slugify the peer's registry `cwd` into `~/.claude/projects/<slug>` — is a
|
|
180
|
+
**trap**. The slug mirrors the cwd *string as the process was launched*, not the real path.
|
|
181
|
+
This host carries BOTH `-Users-macmini-Projects-IAPeer` and `-Users-macmini-Projects-iapeer`
|
|
182
|
+
for one case-insensitive directory. So attribution runs the other way: read the cwd each file
|
|
183
|
+
states **about itself** (claude: the `cwd` field on every line; codex: `session_meta.payload.cwd`)
|
|
184
|
+
and match it against the registry, case-insensitively. A file whose cwd belongs to no peer is
|
|
185
|
+
ignored — a human's own session never notifies.
|
|
186
|
+
|
|
187
|
+
## 7. What is proven, and what is not
|
|
188
|
+
|
|
189
|
+
- **claude — proven live.** The transcript signal, the missing reset, the whole path
|
|
190
|
+
(detector → board → fleet surface) were exercised against a genuinely exhausted fable
|
|
191
|
+
bucket on 15.07.2026. Tests replay the captured line byte-for-byte.
|
|
192
|
+
- **codex — proven by replay, not live.** The rollout structure, windows, resets and the
|
|
193
|
+
healthy (`null`) path are real bytes from this host. Codex's quota never ran out here, so
|
|
194
|
+
**no real reached-bytes exist**: the tests flip that one field to a variant read out of the
|
|
195
|
+
binary. The detector keys on non-null precisely so this synthesis cannot flatter it. A real
|
|
196
|
+
codex reach should be captured opportunistically and folded in.
|
|
197
|
+
- **The blocking-window heuristic is unverified.** Codex does not say WHICH window blocked;
|
|
198
|
+
the notice reports the **fullest** one. Both windows ride in `content`, so a human sees the
|
|
199
|
+
raw fact even if the heuristic picks wrong.
|
|
200
|
+
|
|
201
|
+
Forensic evidence for all of the above: `docs/internals/forensics/model-limit-2026-07-15/`.
|
package/docs/README.md
CHANGED
|
@@ -80,6 +80,7 @@ The sections are layered: from "install and use" to "understand the architecture
|
|
|
80
80
|
- [16 — Tray](16-tray.md) — the macOS menu-bar fleet dashboard (SwiftBar plugin), the first external Fleet API client
|
|
81
81
|
- [17 — Human approval](17-approval.md) — per-peer gated/yolo mode: routing a runtime's blocking approvals to a human through the daemon broker (hook interception, the broker, Fleet API + CLI)
|
|
82
82
|
- [18 — Origin guard](18-origin-guard.md) — an agent's reply to a human must target the channel the human wrote from: the armed-pair model, the hold, `confirm-send`
|
|
83
|
+
- [19 — Notices](19-notices.md) — telling the owner a peer went mute: detecting the structural API error that leaves a session alive, green and speechless, and routing it to the owner's surfaces one-way (not an approval)
|
|
83
84
|
|
|
84
85
|
## Status
|
|
85
86
|
|
package/package.json
CHANGED
package/src/daemon/fleet.ts
CHANGED
|
@@ -62,6 +62,8 @@ import { heartbeatAgeSecs, probeFullDiskAccess, readMemoryProvider, readVoicePro
|
|
|
62
62
|
import { deliveryLogPath } from './deliverylog.ts'
|
|
63
63
|
import { approvalsLogPath } from './approvalslog.ts'
|
|
64
64
|
import { ApprovalBroker } from './approvals.ts'
|
|
65
|
+
import { noticesLogPath } from './noticeslog.ts'
|
|
66
|
+
import { NoticeBoard } from './notices.ts'
|
|
65
67
|
import { callTool, type CallToolDeps } from './index.ts'
|
|
66
68
|
|
|
67
69
|
/** The API generation baked into every snapshot (additive evolution within v1 —
|
|
@@ -128,6 +130,35 @@ export interface FleetApproval {
|
|
|
128
130
|
expiresMs: number
|
|
129
131
|
}
|
|
130
132
|
|
|
133
|
+
/** A live owner-facing notice as the snapshot exposes it (docs/19). The SAME items
|
|
134
|
+
* `GET /fleet/v1/notices` returns (board.list()) — carried in the snapshot so a client that
|
|
135
|
+
* already fetches `/snapshot` on every event renders the board with no extra call.
|
|
136
|
+
*
|
|
137
|
+
* ONE-WAY: there is nothing to answer. A face RENDERS a notice; it never resolves one (no
|
|
138
|
+
* approve/deny sibling exists on purpose — see docs/19 §1).
|
|
139
|
+
*
|
|
140
|
+
* `resetsAtMs` is present ONLY when the runtime itself stated when the wall lifts (codex
|
|
141
|
+
* does; claude does not for a per-model bucket). Its ABSENCE means "the runtime did not
|
|
142
|
+
* say" — a client MUST render that as unknown and MUST NOT substitute the 5h/7d reset,
|
|
143
|
+
* which belongs to a different limit. */
|
|
144
|
+
export interface FleetNotice {
|
|
145
|
+
id: string
|
|
146
|
+
personality: string
|
|
147
|
+
runtime: string
|
|
148
|
+
kind: string
|
|
149
|
+
errorType: string
|
|
150
|
+
model?: string
|
|
151
|
+
resetsAtMs?: number
|
|
152
|
+
summary: string
|
|
153
|
+
content: string
|
|
154
|
+
sessionId?: string
|
|
155
|
+
createdMs: number
|
|
156
|
+
lastMs: number
|
|
157
|
+
expiresMs: number
|
|
158
|
+
/** Detections folded into this notice while it stayed live (≥1) — render as "×N". */
|
|
159
|
+
count: number
|
|
160
|
+
}
|
|
161
|
+
|
|
131
162
|
export interface FleetSnapshot {
|
|
132
163
|
api: typeof FLEET_API_VERSION
|
|
133
164
|
version: string
|
|
@@ -137,6 +168,9 @@ export interface FleetSnapshot {
|
|
|
137
168
|
/** Pending human-approval requests (docs/17). ADDITIVE + omitted when the queue is empty —
|
|
138
169
|
* a client MUST treat its absence as an empty queue (same rule as `approval_mode`). */
|
|
139
170
|
approvals?: FleetApproval[]
|
|
171
|
+
/** Live owner-facing notices (docs/19). ADDITIVE + omitted when the board is empty — a
|
|
172
|
+
* client MUST treat its absence as an empty board (same rule as `approvals`). */
|
|
173
|
+
notices?: FleetNotice[]
|
|
140
174
|
}
|
|
141
175
|
|
|
142
176
|
// The CLI op functions, injectable for tests. Defaults LAZY-load cli/index.ts at
|
|
@@ -173,6 +207,7 @@ export function buildFleetSnapshot(
|
|
|
173
207
|
startedAtMs: number,
|
|
174
208
|
nowMs: number = Date.now(),
|
|
175
209
|
approvals: FleetApproval[] = [],
|
|
210
|
+
notices: FleetNotice[] = [],
|
|
176
211
|
): FleetSnapshot {
|
|
177
212
|
const peers: FleetPeer[] = ops.listPeers({ env }).map(r => {
|
|
178
213
|
const runtimes: FleetRuntimeStatus[] = r.runtimes.map(s => {
|
|
@@ -233,6 +268,8 @@ export function buildFleetSnapshot(
|
|
|
233
268
|
// superset of FleetApproval — the named fields are the contract; extras ride along and
|
|
234
269
|
// clients ignore them (obligation 2).
|
|
235
270
|
...(approvals.length ? { approvals } : {}),
|
|
271
|
+
// Same rule for the notice board (docs/19).
|
|
272
|
+
...(notices.length ? { notices } : {}),
|
|
236
273
|
}
|
|
237
274
|
}
|
|
238
275
|
|
|
@@ -255,6 +292,10 @@ export function fleetEventFiles(cfg: LifecycleConfig): Array<{ path: string; src
|
|
|
255
292
|
// Human-approval broker events (docs/17): approval-request / approval-resolved.
|
|
256
293
|
// A pre-approval daemon simply never writes this file (readLogTail tolerates absence).
|
|
257
294
|
{ path: approvalsLogPath(cfg.eventLogDir), src: 'approvals' },
|
|
295
|
+
// Notice-board events (docs/19): notice-raised. Tailing it here is the WHOLE push
|
|
296
|
+
// path — a face subscribed to /fleet/v1/events receives owner notices with no
|
|
297
|
+
// further wiring, exactly as it already receives approvals.
|
|
298
|
+
{ path: noticesLogPath(cfg.eventLogDir), src: 'notices' },
|
|
258
299
|
]
|
|
259
300
|
}
|
|
260
301
|
|
|
@@ -472,6 +513,10 @@ export interface FleetHandlerOptions {
|
|
|
472
513
|
pollMs?: number
|
|
473
514
|
/** Injectable approval broker (tests — e.g. a short timeout). Default: a fresh one. */
|
|
474
515
|
broker?: ApprovalBroker
|
|
516
|
+
/** The notice board (docs/19). The composition point passes the SAME instance the
|
|
517
|
+
* mute-watch raises into — that shared instance IS the wiring: the watch writes, this
|
|
518
|
+
* handler reads. Default: a fresh one (a handler with no watch simply stays empty). */
|
|
519
|
+
board?: NoticeBoard
|
|
475
520
|
}
|
|
476
521
|
|
|
477
522
|
export type FleetHandler = (
|
|
@@ -494,6 +539,9 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
494
539
|
// The approval broker — one per daemon; the ASK side (POST /approvals long-poll) and
|
|
495
540
|
// the ANSWER side (GET/POST /approvals/*) are all interfaces to this one queue.
|
|
496
541
|
const broker = opts.broker ?? new ApprovalBroker({ logDir: cfg.eventLogDir, env })
|
|
542
|
+
// The notice board — one per daemon; the mute-watch RAISES into it and the /notices reads
|
|
543
|
+
// below serve it. Same shape as the broker above: one in-memory store, many interfaces.
|
|
544
|
+
const board = opts.board ?? new NoticeBoard({ logDir: cfg.eventLogDir, env })
|
|
497
545
|
let opsPromise: Promise<FleetOps> | undefined
|
|
498
546
|
const getOps = (): Promise<FleetOps> => {
|
|
499
547
|
if (opts.ops) return Promise.resolve(opts.ops)
|
|
@@ -511,7 +559,13 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
511
559
|
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/snapshot' })
|
|
512
560
|
// broker.list() is PendingApproval[] — structurally a superset of FleetApproval, so it
|
|
513
561
|
// satisfies the snapshot contract (extra fields serialize + are ignored by clients).
|
|
514
|
-
|
|
562
|
+
// broker.list()/board.list() are supersets of FleetApproval/FleetNotice — the named
|
|
563
|
+
// fields are the contract; extras serialize and clients ignore them (obligation 2).
|
|
564
|
+
return sendJson(
|
|
565
|
+
res,
|
|
566
|
+
200,
|
|
567
|
+
buildFleetSnapshot(env, cfg, await getOps(), startedAtMs, undefined, broker.list(), board.list()),
|
|
568
|
+
)
|
|
515
569
|
}
|
|
516
570
|
if (path === '/fleet/v1/events') {
|
|
517
571
|
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/events' })
|
|
@@ -647,6 +701,26 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
647
701
|
if (!ok) return sendJson(res, 404, { error: `no pending approval "${id}" (already resolved, expired, or unknown)` })
|
|
648
702
|
return sendJson(res, 200, { id, action, ok: true })
|
|
649
703
|
}
|
|
704
|
+
|
|
705
|
+
// ── notices (docs/19) — READ-ONLY BY DESIGN ───────────────────────────
|
|
706
|
+
// No approve/deny sibling: a notice is one-way. Nothing is blocked on it and there
|
|
707
|
+
// is no decision to make, so there is nothing for a face to POST back.
|
|
708
|
+
if (path === '/fleet/v1/notices') {
|
|
709
|
+
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/notices' })
|
|
710
|
+
return sendJson(res, 200, {
|
|
711
|
+
api: FLEET_API_VERSION,
|
|
712
|
+
version: IAPEER_VERSION,
|
|
713
|
+
ts: new Date().toISOString(),
|
|
714
|
+
notices: board.list(),
|
|
715
|
+
})
|
|
716
|
+
}
|
|
717
|
+
const noticeMatch = /^\/fleet\/v1\/notices\/([^/]+)$/.exec(path)
|
|
718
|
+
if (noticeMatch) {
|
|
719
|
+
if (method !== 'GET') return sendJson(res, 405, { error: 'method not allowed — GET /fleet/v1/notices/<id>' })
|
|
720
|
+
const item = board.get(noticeMatch[1]!)
|
|
721
|
+
if (!item) return sendJson(res, 404, { error: `no live notice "${noticeMatch[1]}" (expired or unknown)` })
|
|
722
|
+
return sendJson(res, 200, { api: FLEET_API_VERSION, version: IAPEER_VERSION, ts: new Date().toISOString(), notice: item })
|
|
723
|
+
}
|
|
650
724
|
return sendJson(res, 404, {
|
|
651
725
|
error: `unknown fleet endpoint ${method} ${path}`,
|
|
652
726
|
endpoints: [
|
|
@@ -659,6 +733,8 @@ export function buildFleetHandler(opts: FleetHandlerOptions = {}): FleetHandler
|
|
|
659
733
|
'POST /fleet/v1/approvals (hook long-poll)',
|
|
660
734
|
'GET /fleet/v1/approvals/<id>',
|
|
661
735
|
'POST /fleet/v1/approvals/<id>/(approve|deny)',
|
|
736
|
+
'GET /fleet/v1/notices',
|
|
737
|
+
'GET /fleet/v1/notices/<id>',
|
|
662
738
|
],
|
|
663
739
|
})
|
|
664
740
|
} catch (e) {
|
package/src/daemon/main.ts
CHANGED
|
@@ -67,6 +67,8 @@ import {
|
|
|
67
67
|
import { iapeerBinPath } from '../install/index.ts'
|
|
68
68
|
import { appendDeliveryEvent } from './deliverylog.ts'
|
|
69
69
|
import { buildFleetHandler } from './fleet.ts'
|
|
70
|
+
import { NoticeBoard } from './notices.ts'
|
|
71
|
+
import { startMuteWatch } from './mutewatch.ts'
|
|
70
72
|
import { defaultDaemonSocketPath, startDaemon, type DaemonHandle } from './index.ts'
|
|
71
73
|
|
|
72
74
|
/** Default TCP loopback port for the always-on router. Real http MCP clients
|
|
@@ -91,6 +93,13 @@ export function parseDaemonPort(raw: string | undefined, dflt: number = DEFAULT_
|
|
|
91
93
|
* is the reap threshold; this is just how often the timer checks. */
|
|
92
94
|
export const DEFAULT_SUPERVISE_INTERVAL_MS = 60_000
|
|
93
95
|
|
|
96
|
+
/** Mute-watch cadence (docs/19) — how often the daemon sweeps the runtimes' session files
|
|
97
|
+
* for a structural API error that left a peer unable to answer. Its OWN timer rather than a
|
|
98
|
+
* ride on the 60 s supervise tick: detection latency IS the product here (the owner must
|
|
99
|
+
* learn within a minute), and a 60 s sweep puts the worst case AT the budget with nothing
|
|
100
|
+
* to spare. The sweep is cheap — it stats only files that CHANGED since the last pass. */
|
|
101
|
+
export const DEFAULT_MUTEWATCH_INTERVAL_MS = 20_000
|
|
102
|
+
|
|
94
103
|
// This module's own path — the launchd plist runs `bun <this>` as the daemon.
|
|
95
104
|
const DAEMON_MAIN_PATH = fileURLToPath(import.meta.url)
|
|
96
105
|
|
|
@@ -319,6 +328,8 @@ export interface ConfiguredDaemonOptions {
|
|
|
319
328
|
/** H8 bearer token; falls back to env.IAPEER_BEARER_TOKEN. Off when neither set. */
|
|
320
329
|
bearerToken?: string
|
|
321
330
|
superviseIntervalMs?: number
|
|
331
|
+
/** Mute-watch sweep cadence (default DEFAULT_MUTEWATCH_INTERVAL_MS). */
|
|
332
|
+
muteWatchIntervalMs?: number
|
|
322
333
|
/** Write the router.json discovery file (default true for production). */
|
|
323
334
|
discovery?: boolean
|
|
324
335
|
rootDir?: string
|
|
@@ -341,7 +352,24 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
341
352
|
const armEphemeralOnDelivered = makeArmEphemeralOnDelivered(cfg)
|
|
342
353
|
const ephemeral = makeEphemeralRouteDeps(cfg, env)
|
|
343
354
|
const composerQueue = makeComposerQueueRouteDeps(cfg, env)
|
|
344
|
-
|
|
355
|
+
// Notice board (docs/19) — ONE instance shared by the mute-watch (raises) and the
|
|
356
|
+
// fleet handler (serves /notices + the snapshot field). Constructed HERE, at the
|
|
357
|
+
// composition point, because that sharing is the whole wiring.
|
|
358
|
+
const board = new NoticeBoard({ logDir: cfg.eventLogDir, env })
|
|
359
|
+
const stopMuteWatch = startMuteWatch(cfg, board, {
|
|
360
|
+
env,
|
|
361
|
+
intervalMs: opts.muteWatchIntervalMs ?? DEFAULT_MUTEWATCH_INTERVAL_MS,
|
|
362
|
+
// Same stance as the supervise tick's onError: a detector throw must be VISIBLE, not
|
|
363
|
+
// swallowed into a silently-dead watch — the exact failure mode this feature exists to
|
|
364
|
+
// end. Best-effort; a reporter must never fail the daemon.
|
|
365
|
+
onError: (err: unknown) => {
|
|
366
|
+
try {
|
|
367
|
+
const detail = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
368
|
+
appendLifecycleEvent(cfg.eventLogDir, { ev: 'mutewatch-error', error: detail.replace(/\s+/g, ' ').slice(0, 600) }, { env })
|
|
369
|
+
} catch { /* best-effort */ }
|
|
370
|
+
},
|
|
371
|
+
})
|
|
372
|
+
const handle = await startDaemon({
|
|
345
373
|
wake: makeWakeFn(cfg, env),
|
|
346
374
|
// M2 arm-on-outbound (see makeArmEphemeralOnDelivered): ephemeral caller's ok
|
|
347
375
|
// send ⇒ armed for the supervise quiet-reap. No-op for every other caller.
|
|
@@ -367,7 +395,7 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
367
395
|
// (the same listPeers truth as `iapeer list`), SSE events (tail of the durable
|
|
368
396
|
// logs), commands over the existing verb functions. Same listeners, same bearer
|
|
369
397
|
// gate; advertised in router.json as fleet:1. Contract: docs/15-fleet-api.md.
|
|
370
|
-
fleet: buildFleetHandler({ env }),
|
|
398
|
+
fleet: buildFleetHandler({ env, board }),
|
|
371
399
|
supervise: {
|
|
372
400
|
intervalMs: opts.superviseIntervalMs ?? DEFAULT_SUPERVISE_INTERVAL_MS,
|
|
373
401
|
// idle-reap / zombie-sweep, THEN the eager fresh re-launch for any peer whose
|
|
@@ -412,6 +440,10 @@ export async function startConfiguredDaemon(opts: ConfiguredDaemonOptions = {}):
|
|
|
412
440
|
env,
|
|
413
441
|
rootDir: opts.rootDir,
|
|
414
442
|
})
|
|
443
|
+
// The mute-watch timer is OURS, not startDaemon's — so its teardown must hang off the
|
|
444
|
+
// handle we return, or a closed daemon would leave a live timer sweeping the disk (and
|
|
445
|
+
// every test that starts a daemon would hang on exit).
|
|
446
|
+
return { ...handle, close: async () => { stopMuteWatch(); await handle.close() } }
|
|
415
447
|
}
|
|
416
448
|
|
|
417
449
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
// Mute-watch — the daemon's detector for the ONE class of failure that is invisible to
|
|
2
|
+
// every health signal it has: a structural API error left a peer unable to answer (docs/19).
|
|
3
|
+
//
|
|
4
|
+
// WHY THIS EXISTS. When a model's limit is exhausted the peer's session does NOT die. Proven
|
|
5
|
+
// live (15.07.2026, exhausted fable bucket): the banner paints, the composer paints, the
|
|
6
|
+
// ready-gate would flip READY, the delivered turn is CONSUMED, the runtime paints its refusal,
|
|
7
|
+
// and the session returns to the composer and lives on. The process is alive, there is no exit
|
|
8
|
+
// code, and the statusline keeps re-rendering — which advances the pane-log mtime, so the
|
|
9
|
+
// activity proxy reports the delivery CONFIRMED. Every daemon health signal is green and the
|
|
10
|
+
// peer is mute. The owner writes and gets silence, with nothing anywhere saying why.
|
|
11
|
+
// The exit code is no help either: `-p` (headless) exits 1 on a limit, but live peers are
|
|
12
|
+
// never launched that way — an interactive session just sits there.
|
|
13
|
+
//
|
|
14
|
+
// WHERE THE TRUTH IS. Not the pane: the refusal is prose that line-wraps, and matching wrapped
|
|
15
|
+
// ANSI is a losing game. Both runtimes write the fact STRUCTURALLY to their own session JSONL:
|
|
16
|
+
//
|
|
17
|
+
// • claude → the transcript (~/.claude/projects/<slug>/<sessionId>.jsonl):
|
|
18
|
+
// isApiErrorMessage: true, error: "rate_limit", apiErrorStatus: 429,
|
|
19
|
+
// message.model: "<synthetic>" (a fabricated message, not a model reply).
|
|
20
|
+
// `error` is a locale-stable enum and the WHOLE CLASS rides it — rate_limit is one value;
|
|
21
|
+
// overloaded / expired auth arrive on the same field, which is why this detector keys on
|
|
22
|
+
// isApiErrorMessage and reports `error` verbatim instead of hunting one string.
|
|
23
|
+
// Claude does NOT state when a per-model bucket lifts ("Please try again later") → the
|
|
24
|
+
// notice omits the reset rather than extrapolating from the 5h/7d buckets (a DIFFERENT
|
|
25
|
+
// limit: measured live at 5h 11% / 7d 66% while fable was fully exhausted).
|
|
26
|
+
//
|
|
27
|
+
// • codex → the rollout (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl): an `event_msg` /
|
|
28
|
+
// `token_count` payload carries RateLimitSnapshot.rate_limit_reached_type — null while
|
|
29
|
+
// healthy, non-null once a wall is hit. We key on NON-NULL, never on the variant strings
|
|
30
|
+
// (`rate_limit_reached`, `workspace_owner_usage_limit_reached`, … — read out of the 0.144.1
|
|
31
|
+
// binary but never observed live here), so an unknown future variant still detects.
|
|
32
|
+
// Codex DOES carry resets_at per window → its notice states the reset time.
|
|
33
|
+
//
|
|
34
|
+
// PATHS ARE NEVER DERIVED FROM cwd. The obvious route — slugify the peer's registry cwd into
|
|
35
|
+
// ~/.claude/projects/<slug> — is a trap: the slug mirrors the cwd STRING as the process was
|
|
36
|
+
// launched, not the real path. This host carries BOTH `-Users-macmini-Projects-IAPeer` and
|
|
37
|
+
// `-Users-macmini-Projects-iapeer` for ONE case-insensitive directory. So we go the other way:
|
|
38
|
+
// scan the session files that CHANGED, read the cwd each file states about ITSELF (claude: the
|
|
39
|
+
// `cwd` field on every line; codex: session_meta.payload.cwd), and match that against the
|
|
40
|
+
// registry. A file whose cwd is no peer's is ignored — a human's own session never notifies.
|
|
41
|
+
|
|
42
|
+
import { existsSync, openSync, closeSync, fstatSync, readSync, readdirSync, statSync } from 'node:fs'
|
|
43
|
+
import { join } from 'node:path'
|
|
44
|
+
import type { LifecycleConfig } from '../lifecycle/index.ts'
|
|
45
|
+
import type { NoticeBoard } from './notices.ts'
|
|
46
|
+
|
|
47
|
+
/** How much of a session file's tail is parsed. The evidence is always at the end. */
|
|
48
|
+
const TAIL_BYTES = 256 * 1024
|
|
49
|
+
/** Overlap added to the since-boundary so an event landing between two sweeps is never
|
|
50
|
+
* missed. Re-detections are free: the board's dedup folds them into the live notice. */
|
|
51
|
+
const SCAN_OVERLAP_MS = 15_000
|
|
52
|
+
/** Walk depth for the codex YYYY/MM/DD tree (3 levels + the file). */
|
|
53
|
+
const CODEX_WALK_DEPTH = 3
|
|
54
|
+
|
|
55
|
+
export interface MuteDetection {
|
|
56
|
+
personality: string
|
|
57
|
+
runtime: 'claude' | 'codex'
|
|
58
|
+
/** The runtime's OWN taxonomy value — claude's `error`, codex's rate_limit_reached_type. */
|
|
59
|
+
errorType: string
|
|
60
|
+
model?: string
|
|
61
|
+
resetsAtMs?: number
|
|
62
|
+
/** Claude: the runtime's VERBATIM refusal. Codex: a line rendered from its typed fields
|
|
63
|
+
* (the rollout carries no prose). Either way, never our interpretation of the cause. */
|
|
64
|
+
content: string
|
|
65
|
+
sessionId?: string
|
|
66
|
+
atMs: number
|
|
67
|
+
cwd: string
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
type RawDetection = Omit<MuteDetection, 'personality' | 'runtime'>
|
|
71
|
+
|
|
72
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
73
|
+
// Pure parsers — string in, detection out (fs-free: tests need no temp tree)
|
|
74
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
75
|
+
|
|
76
|
+
function toMs(ts: unknown): number | null {
|
|
77
|
+
if (typeof ts !== 'string') return null
|
|
78
|
+
const ms = Date.parse(ts)
|
|
79
|
+
return Number.isFinite(ms) ? ms : null
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The model out of the runtime's own prose ("You've reached your Fable 5 limit"). Used ONLY
|
|
83
|
+
* when the transcript carries no real model reply to read it from — a session that hits the
|
|
84
|
+
* wall on its FIRST turn has no such line. Bounded and anchored; a miss → undefined (omit). */
|
|
85
|
+
export function modelFromClaudeText(text: string): string | undefined {
|
|
86
|
+
const m = /reached your\s+(.{1,40}?)\s+limit/i.exec(text)
|
|
87
|
+
return m?.[1]?.trim() || undefined
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The NEWEST structural API-error line in a claude transcript newer than `sinceMs`, or null.
|
|
92
|
+
* Keys on `isApiErrorMessage` — the class, not the `rate_limit` instance.
|
|
93
|
+
*/
|
|
94
|
+
export function parseClaudeTranscript(text: string, sinceMs: number): RawDetection | null {
|
|
95
|
+
let lastRealModel: string | undefined
|
|
96
|
+
let found: RawDetection | null = null
|
|
97
|
+
for (const line of text.split('\n')) {
|
|
98
|
+
const s = line.trim()
|
|
99
|
+
if (!s.startsWith('{')) continue // a partial first line from the tail cut — skip
|
|
100
|
+
let d: Record<string, unknown>
|
|
101
|
+
try {
|
|
102
|
+
d = JSON.parse(s) as Record<string, unknown>
|
|
103
|
+
} catch {
|
|
104
|
+
continue // truncated/garbled line — never fail the sweep over one bad line
|
|
105
|
+
}
|
|
106
|
+
const msg = d.message as { model?: unknown; content?: unknown } | undefined
|
|
107
|
+
// Track the last REAL model reply: the error line itself is "<synthetic>" and names no model.
|
|
108
|
+
if (msg && typeof msg.model === 'string' && msg.model && msg.model !== '<synthetic>') lastRealModel = msg.model
|
|
109
|
+
if (d.isApiErrorMessage !== true) continue
|
|
110
|
+
const atMs = toMs(d.timestamp)
|
|
111
|
+
if (atMs === null || atMs <= sinceMs) continue
|
|
112
|
+
const cwd = typeof d.cwd === 'string' ? d.cwd : ''
|
|
113
|
+
if (!cwd) continue // no cwd → unattributable to a peer; ignore rather than guess
|
|
114
|
+
const errorType = typeof d.error === 'string' && d.error.trim() ? d.error.trim() : 'api-error'
|
|
115
|
+
let content = ''
|
|
116
|
+
if (Array.isArray(msg?.content)) {
|
|
117
|
+
content = (msg.content as Array<{ type?: string; text?: string }>)
|
|
118
|
+
.filter(c => c?.type === 'text' && typeof c.text === 'string')
|
|
119
|
+
.map(c => c.text as string)
|
|
120
|
+
.join('\n')
|
|
121
|
+
.trim()
|
|
122
|
+
}
|
|
123
|
+
if (!content) content = `${errorType} (api error ${String(d.apiErrorStatus ?? '')})`.trim()
|
|
124
|
+
found = {
|
|
125
|
+
errorType,
|
|
126
|
+
model: lastRealModel ?? modelFromClaudeText(content),
|
|
127
|
+
content,
|
|
128
|
+
sessionId: typeof d.sessionId === 'string' ? d.sessionId : undefined,
|
|
129
|
+
atMs,
|
|
130
|
+
cwd,
|
|
131
|
+
// Claude states no reset for a per-model bucket → resetsAtMs deliberately absent.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return found
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
interface CodexWindow {
|
|
138
|
+
used_percent?: unknown
|
|
139
|
+
window_minutes?: unknown
|
|
140
|
+
resets_at?: unknown
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Describe a codex window for the human-readable content line. */
|
|
144
|
+
function describeWindow(name: string, w: CodexWindow): string | null {
|
|
145
|
+
if (!w || typeof w.used_percent !== 'number') return null
|
|
146
|
+
const win = typeof w.window_minutes === 'number' ? `${w.window_minutes}m window` : 'window'
|
|
147
|
+
const reset =
|
|
148
|
+
typeof w.resets_at === 'number' ? `, resets ${new Date(w.resets_at * 1000).toISOString()}` : ''
|
|
149
|
+
return `${name} ${w.used_percent}% (${win}${reset})`
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The codex rollout's newest limit-reached snapshot newer than `sinceMs`, or null.
|
|
154
|
+
* Keys on `rate_limit_reached_type` being NON-NULL — never on its variant strings.
|
|
155
|
+
* `cwd` comes from session_meta, which is the file's FIRST line — pass the file HEAD in
|
|
156
|
+
* `head` when the tail alone may have cut it away.
|
|
157
|
+
*/
|
|
158
|
+
export function parseCodexRollout(text: string, sinceMs: number, head = ''): RawDetection | null {
|
|
159
|
+
let cwd = ''
|
|
160
|
+
let sessionId: string | undefined
|
|
161
|
+
const scanMeta = (src: string): void => {
|
|
162
|
+
for (const line of src.split('\n')) {
|
|
163
|
+
const s = line.trim()
|
|
164
|
+
if (!s.startsWith('{') || !s.includes('session_meta')) continue
|
|
165
|
+
try {
|
|
166
|
+
const d = JSON.parse(s) as { type?: string; payload?: { cwd?: unknown; session_id?: unknown } }
|
|
167
|
+
if (d.type !== 'session_meta' || !d.payload) continue
|
|
168
|
+
if (typeof d.payload.cwd === 'string') cwd = d.payload.cwd
|
|
169
|
+
if (typeof d.payload.session_id === 'string') sessionId = d.payload.session_id
|
|
170
|
+
} catch {
|
|
171
|
+
/* ignore */
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
scanMeta(head)
|
|
176
|
+
if (!cwd) scanMeta(text)
|
|
177
|
+
if (!cwd) return null // unattributable to a peer
|
|
178
|
+
|
|
179
|
+
let found: RawDetection | null = null
|
|
180
|
+
for (const line of text.split('\n')) {
|
|
181
|
+
const s = line.trim()
|
|
182
|
+
if (!s.startsWith('{') || !s.includes('rate_limits')) continue
|
|
183
|
+
let d: { timestamp?: unknown; type?: unknown; payload?: { type?: unknown; rate_limits?: Record<string, unknown> } }
|
|
184
|
+
try {
|
|
185
|
+
d = JSON.parse(s)
|
|
186
|
+
} catch {
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
const rl = d.payload?.rate_limits
|
|
190
|
+
if (!rl) continue
|
|
191
|
+
const reached = rl.rate_limit_reached_type
|
|
192
|
+
// NON-NULL is the signal. An unknown future variant must still detect, so we never
|
|
193
|
+
// compare against the variant strings read out of the binary.
|
|
194
|
+
if (reached === null || reached === undefined || reached === '') continue
|
|
195
|
+
const atMs = toMs(d.timestamp)
|
|
196
|
+
if (atMs === null || atMs <= sinceMs) continue
|
|
197
|
+
const primary = (rl.primary ?? {}) as CodexWindow
|
|
198
|
+
const secondary = (rl.secondary ?? {}) as CodexWindow
|
|
199
|
+
// Which window is blocking? Codex does not say — reported for the FULLEST one, the
|
|
200
|
+
// only defensible read. This mapping is a HEURISTIC: it could not be verified against
|
|
201
|
+
// a real reached-snapshot (the quota never ran out on this host). Both windows ride in
|
|
202
|
+
// `content`, so the human sees the raw fact even if the heuristic picked wrong.
|
|
203
|
+
const pu = typeof primary.used_percent === 'number' ? primary.used_percent : -1
|
|
204
|
+
const su = typeof secondary.used_percent === 'number' ? secondary.used_percent : -1
|
|
205
|
+
const blocking = su > pu ? secondary : primary
|
|
206
|
+
const resetsAtMs = typeof blocking.resets_at === 'number' ? blocking.resets_at * 1000 : undefined
|
|
207
|
+
const parts = [describeWindow('primary', primary), describeWindow('secondary', secondary)].filter(Boolean)
|
|
208
|
+
const plan = typeof rl.plan_type === 'string' ? ` plan=${rl.plan_type}` : ''
|
|
209
|
+
found = {
|
|
210
|
+
errorType: String(reached),
|
|
211
|
+
content: `Codex usage limit reached (${String(reached)})${plan}${parts.length ? ` — ${parts.join('; ')}` : ''}`,
|
|
212
|
+
resetsAtMs,
|
|
213
|
+
sessionId,
|
|
214
|
+
atMs,
|
|
215
|
+
cwd,
|
|
216
|
+
// Codex names no model in the snapshot → model omitted rather than guessed.
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return found
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
223
|
+
// The sweep — find changed session files, attribute them, raise notices
|
|
224
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
225
|
+
|
|
226
|
+
function readTail(path: string, maxBytes = TAIL_BYTES): { tail: string; head: string } | null {
|
|
227
|
+
try {
|
|
228
|
+
const fd = openSync(path, 'r')
|
|
229
|
+
try {
|
|
230
|
+
const size = fstatSync(fd).size
|
|
231
|
+
if (size <= 0) return null
|
|
232
|
+
const off = Math.max(0, size - maxBytes)
|
|
233
|
+
const buf = Buffer.alloc(size - off)
|
|
234
|
+
readSync(fd, buf, 0, buf.length, off)
|
|
235
|
+
let head = ''
|
|
236
|
+
if (off > 0) {
|
|
237
|
+
// The tail cut away the file's start — read a HEAD slice too (codex session_meta
|
|
238
|
+
// is line 1, and a long session pushes it far out of the tail).
|
|
239
|
+
const hb = Buffer.alloc(Math.min(64 * 1024, off))
|
|
240
|
+
readSync(fd, hb, 0, hb.length, 0)
|
|
241
|
+
head = hb.toString('utf8')
|
|
242
|
+
}
|
|
243
|
+
return { tail: buf.toString('utf8'), head }
|
|
244
|
+
} finally {
|
|
245
|
+
closeSync(fd)
|
|
246
|
+
}
|
|
247
|
+
} catch {
|
|
248
|
+
return null
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function safeReaddir(dir: string): string[] {
|
|
253
|
+
try {
|
|
254
|
+
return readdirSync(dir)
|
|
255
|
+
} catch {
|
|
256
|
+
return []
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function mtimeMs(path: string): number | null {
|
|
261
|
+
try {
|
|
262
|
+
return statSync(path).mtimeMs
|
|
263
|
+
} catch {
|
|
264
|
+
return null
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** *.jsonl files under `root` (depth-bounded) whose mtime is at/after `sinceMs`. */
|
|
269
|
+
function changedJsonl(root: string, sinceMs: number, depth: number): string[] {
|
|
270
|
+
if (!existsSync(root)) return []
|
|
271
|
+
const out: string[] = []
|
|
272
|
+
const walk = (dir: string, left: number): void => {
|
|
273
|
+
for (const name of safeReaddir(dir)) {
|
|
274
|
+
const p = join(dir, name)
|
|
275
|
+
if (name.endsWith('.jsonl')) {
|
|
276
|
+
const m = mtimeMs(p)
|
|
277
|
+
if (m !== null && m >= sinceMs) out.push(p)
|
|
278
|
+
continue
|
|
279
|
+
}
|
|
280
|
+
if (left <= 0 || name.startsWith('.')) continue
|
|
281
|
+
let isDir = false
|
|
282
|
+
try {
|
|
283
|
+
isDir = statSync(p).isDirectory()
|
|
284
|
+
} catch {
|
|
285
|
+
continue
|
|
286
|
+
}
|
|
287
|
+
if (isDir) walk(p, left - 1)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
walk(root, depth)
|
|
291
|
+
return out
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export interface MuteWatchPaths {
|
|
295
|
+
/** ~/.claude/projects — INJECTED, never re-resolved from env inside the sweep. */
|
|
296
|
+
claudeProjectsDir: string
|
|
297
|
+
/** ~/.codex/sessions — same rule. */
|
|
298
|
+
codexSessionsDir: string
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export interface ScanDeps extends MuteWatchPaths {
|
|
302
|
+
/** personality ← cwd. Built from the registry by the composition point. */
|
|
303
|
+
peerByCwd: (cwd: string) => string | undefined
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* The runtime session-file roots. These are the RUNTIMES' own trees, not iapeer's, so they
|
|
308
|
+
* are not LifecycleConfig fields; the composition point resolves them ONCE and injects them
|
|
309
|
+
* (a sweep never re-resolves a path mid-run). The env overrides exist so a test points the
|
|
310
|
+
* sweep at a temp tree instead of the real ~/.claude — the sweep is READ-ONLY, and its only
|
|
311
|
+
* write (notices.log) goes through cfg.eventLogDir like every other daemon log.
|
|
312
|
+
*/
|
|
313
|
+
export function resolveMuteWatchPaths(env: NodeJS.ProcessEnv): MuteWatchPaths {
|
|
314
|
+
const home = env.HOME?.trim() || ''
|
|
315
|
+
return {
|
|
316
|
+
claudeProjectsDir: env.IAPEER_CLAUDE_PROJECTS_DIR?.trim() || join(home, '.claude', 'projects'),
|
|
317
|
+
codexSessionsDir: env.IAPEER_CODEX_SESSIONS_DIR?.trim() || join(home, '.codex', 'sessions'),
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** One sweep: every peer-attributable structural API error newer than `sinceMs`. */
|
|
322
|
+
export function scanMuteEvents(deps: ScanDeps, sinceMs: number): MuteDetection[] {
|
|
323
|
+
const floor = sinceMs - SCAN_OVERLAP_MS
|
|
324
|
+
const out: MuteDetection[] = []
|
|
325
|
+
const push = (raw: RawDetection | null, runtime: 'claude' | 'codex'): void => {
|
|
326
|
+
if (!raw) return
|
|
327
|
+
const personality = deps.peerByCwd(raw.cwd)
|
|
328
|
+
if (!personality) return // not a peer's session (a human's own shell) — never notify
|
|
329
|
+
out.push({ ...raw, personality, runtime })
|
|
330
|
+
}
|
|
331
|
+
// claude: ~/.claude/projects/<slug>/<sessionId>.jsonl — one level of slug dirs.
|
|
332
|
+
for (const file of changedJsonl(deps.claudeProjectsDir, floor, 1)) {
|
|
333
|
+
const t = readTail(file)
|
|
334
|
+
if (t) push(parseClaudeTranscript(t.tail, floor), 'claude')
|
|
335
|
+
}
|
|
336
|
+
// codex: ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl
|
|
337
|
+
for (const file of changedJsonl(deps.codexSessionsDir, floor, CODEX_WALK_DEPTH)) {
|
|
338
|
+
const t = readTail(file)
|
|
339
|
+
if (t) push(parseCodexRollout(t.tail, floor, t.head), 'codex')
|
|
340
|
+
}
|
|
341
|
+
return out
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
export interface MuteWatchDeps extends ScanDeps {
|
|
345
|
+
board: NoticeBoard
|
|
346
|
+
now?: () => number
|
|
347
|
+
/** Reported, never thrown — a detector must never take the daemon down. */
|
|
348
|
+
onError?: (err: unknown) => void
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Run one watch tick: sweep, raise. Returns the notices actually RAISED (deduped repeats
|
|
353
|
+
* are folded by the board and not returned) plus the new since-boundary.
|
|
354
|
+
*/
|
|
355
|
+
export function muteWatchTick(deps: MuteWatchDeps, sinceMs: number): { raised: number; sinceMs: number } {
|
|
356
|
+
const now = deps.now ?? Date.now
|
|
357
|
+
const startedMs = now()
|
|
358
|
+
let raised = 0
|
|
359
|
+
try {
|
|
360
|
+
for (const d of scanMuteEvents(deps, sinceMs)) {
|
|
361
|
+
const { deduped } = deps.board.raise({
|
|
362
|
+
personality: d.personality,
|
|
363
|
+
runtime: d.runtime,
|
|
364
|
+
kind: 'peer-mute',
|
|
365
|
+
errorType: d.errorType,
|
|
366
|
+
model: d.model,
|
|
367
|
+
resetsAtMs: d.resetsAtMs,
|
|
368
|
+
content: d.content,
|
|
369
|
+
sessionId: d.sessionId,
|
|
370
|
+
// The runtime event's OWN timestamp — lets the board tell a new occurrence from the
|
|
371
|
+
// same line re-read by the deliberately-overlapping sweep window.
|
|
372
|
+
eventAtMs: d.atMs,
|
|
373
|
+
summary: `${d.personality} · ${d.runtime} — ${d.errorType}${d.model ? ` (${d.model})` : ''}`,
|
|
374
|
+
})
|
|
375
|
+
if (!deduped) raised += 1
|
|
376
|
+
}
|
|
377
|
+
} catch (e) {
|
|
378
|
+
deps.onError?.(e)
|
|
379
|
+
}
|
|
380
|
+
return { raised, sinceMs: startedMs }
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
export interface StartMuteWatchOptions {
|
|
384
|
+
env?: NodeJS.ProcessEnv
|
|
385
|
+
intervalMs?: number
|
|
386
|
+
/** Override the runtime session roots (tests). Default: resolveMuteWatchPaths(env). */
|
|
387
|
+
paths?: MuteWatchPaths
|
|
388
|
+
/** Override the registry read (tests). Default: a lazy import of the CLI's listPeers. */
|
|
389
|
+
listPeers?: () => Array<{ personality: string; cwd: string }>
|
|
390
|
+
now?: () => number
|
|
391
|
+
onError?: (err: unknown) => void
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Start the mute-watch timer. Returns its stop function — the caller OWNS teardown.
|
|
396
|
+
*
|
|
397
|
+
* The registry is re-read every sweep (a peer created since daemon start must be watched
|
|
398
|
+
* without a restart), and `listPeers` is imported LAZILY at first use, mirroring fleet.ts:
|
|
399
|
+
* daemon → cli is not a load-time dependency in this codebase.
|
|
400
|
+
*/
|
|
401
|
+
export function startMuteWatch(cfg: LifecycleConfig, board: NoticeBoard, opts: StartMuteWatchOptions = {}): () => void {
|
|
402
|
+
const env = opts.env ?? process.env
|
|
403
|
+
const paths = opts.paths ?? resolveMuteWatchPaths(env)
|
|
404
|
+
const now = opts.now ?? Date.now
|
|
405
|
+
let listPeers = opts.listPeers
|
|
406
|
+
let since = now()
|
|
407
|
+
let running = false
|
|
408
|
+
|
|
409
|
+
const sweep = async (): Promise<void> => {
|
|
410
|
+
if (running) return // a slow sweep must never overlap itself into a stampede
|
|
411
|
+
running = true
|
|
412
|
+
try {
|
|
413
|
+
if (!listPeers) {
|
|
414
|
+
const cli = await import('../cli/index.ts')
|
|
415
|
+
listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
|
|
416
|
+
}
|
|
417
|
+
const rows = listPeers()
|
|
418
|
+
// cwd → personality. Compared case-insensitively: macOS paths are case-insensitive and
|
|
419
|
+
// the runtimes record the cwd STRING as launched, which is not always the registry's
|
|
420
|
+
// capitalization (the same trap that rules out deriving the transcript dir from cwd).
|
|
421
|
+
const byCwd = new Map<string, string>()
|
|
422
|
+
for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
|
|
423
|
+
const deps: MuteWatchDeps = {
|
|
424
|
+
...paths,
|
|
425
|
+
board,
|
|
426
|
+
now,
|
|
427
|
+
onError: opts.onError,
|
|
428
|
+
peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
|
|
429
|
+
}
|
|
430
|
+
since = muteWatchTick(deps, since).sinceMs
|
|
431
|
+
} catch (e) {
|
|
432
|
+
opts.onError?.(e)
|
|
433
|
+
} finally {
|
|
434
|
+
running = false
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const timer = setInterval(() => void sweep(), opts.intervalMs ?? 20_000)
|
|
439
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
440
|
+
return () => clearInterval(timer)
|
|
441
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// Notice board — the in-daemon store of OWNER-FACING notices (docs/19). The daemon
|
|
2
|
+
// noticed something the owner must know about a peer, and no one else can tell him: the
|
|
3
|
+
// canonical case is a peer left MUTE by a structural API error (rate limit, overload,
|
|
4
|
+
// expired auth) — such a peer cannot report its own failure, and an Implementer peer has
|
|
5
|
+
// no telegram face of its own. So the notice travels the daemon's OWN surface, exactly as
|
|
6
|
+
// an approval request does.
|
|
7
|
+
//
|
|
8
|
+
// DELIBERATELY NOT AN APPROVAL. An approval is request → decision → unblock: a caller
|
|
9
|
+
// long-polls and something is BLOCKED until a human answers. A notice is one-way: nobody
|
|
10
|
+
// waits, nothing unblocks, there is no decision to make. Reusing the approval broker here
|
|
11
|
+
// would drag a promise/timeout/fail-safe-deny machine into a problem that has none of
|
|
12
|
+
// those. What IS shared is the SURFACE: a durable log tailed into the fleet SSE stream, an
|
|
13
|
+
// additive snapshot field, and a GET endpoint for the verbatim content.
|
|
14
|
+
//
|
|
15
|
+
// TTL + DEDUP, and the two are ONE mechanism. A mute peer re-emits its error on EVERY
|
|
16
|
+
// attempted turn — a naive board would raise a card per attempt and bury the owner. So a
|
|
17
|
+
// notice carries a `dedupKey` (peer + runtime + kind + errorType + model): while a notice
|
|
18
|
+
// with that key is LIVE, a repeat detection only bumps `count`/`lastMs` — no new id, no new
|
|
19
|
+
// log line, no new SSE event. The owner sees one card that says "×7", not seven cards.
|
|
20
|
+
// When it expires (TTL) and the peer is STILL broken, the next detection raises a fresh
|
|
21
|
+
// notice — a deliberate periodic reminder rather than one card that scrolls away forever.
|
|
22
|
+
//
|
|
23
|
+
// In-memory + ephemeral: notices do NOT survive a daemon restart (a restart re-detects
|
|
24
|
+
// within one watch tick — the transcript evidence is on disk and is re-read). The durable
|
|
25
|
+
// notices.log is the audit/observability trace, not a recovery store.
|
|
26
|
+
|
|
27
|
+
import { appendNoticeEvent } from './noticeslog.ts'
|
|
28
|
+
|
|
29
|
+
/** v1 taxonomy of notices. `peer-mute` = a structural API error left the peer unable to
|
|
30
|
+
* answer. Free-form-tolerant — new kinds ride the same surface without a client change. */
|
|
31
|
+
export type NoticeKind = 'peer-mute' | string
|
|
32
|
+
|
|
33
|
+
export interface NoticeInput {
|
|
34
|
+
personality: string
|
|
35
|
+
runtime: string
|
|
36
|
+
/** Taxonomy tag (peer-mute | …). */
|
|
37
|
+
kind: NoticeKind
|
|
38
|
+
/** The runtime's OWN error taxonomy value — `rate_limit`, `overloaded`, … Never
|
|
39
|
+
* invented by us: it is the structural field the runtime wrote (docs/19 §2). */
|
|
40
|
+
errorType: string
|
|
41
|
+
/** The model that hit the wall, when the runtime says which (claude names it only in
|
|
42
|
+
* prose; codex does not name it at all). ABSENT rather than guessed. */
|
|
43
|
+
model?: string
|
|
44
|
+
/** When the limit lifts, epoch-ms — ONLY when the runtime states it. Claude does not
|
|
45
|
+
* for a per-model bucket ("try again later"), so this is absent there: an honest
|
|
46
|
+
* omission, never an extrapolation from the 5h/7d buckets (a DIFFERENT limit). */
|
|
47
|
+
resetsAtMs?: number
|
|
48
|
+
/** One-line summary for the SSE event / badge. */
|
|
49
|
+
summary?: string
|
|
50
|
+
/** The runtime's VERBATIM message — shown to the human in every face. */
|
|
51
|
+
content: string
|
|
52
|
+
/** The session that hit it (correlation with the transcript on disk). */
|
|
53
|
+
sessionId?: string
|
|
54
|
+
/** When the underlying runtime event happened (the transcript line's own timestamp) —
|
|
55
|
+
* NOT when we swept. This is what makes `count` mean OCCURRENCES rather than sweeps:
|
|
56
|
+
* the sweep window deliberately overlaps, so the same line is re-read across passes.
|
|
57
|
+
* Absent → every raise counts (the caller has no event clock to dedup against). */
|
|
58
|
+
eventAtMs?: number
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface Notice extends NoticeInput {
|
|
62
|
+
id: string
|
|
63
|
+
summary: string
|
|
64
|
+
dedupKey: string
|
|
65
|
+
/** First occurrence. */
|
|
66
|
+
createdMs: number
|
|
67
|
+
/** Latest OCCURRENCE (not the latest sweep that re-read it). */
|
|
68
|
+
lastMs: number
|
|
69
|
+
expiresMs: number
|
|
70
|
+
/** Distinct occurrences folded into this notice (≥1) — how many times the peer actually
|
|
71
|
+
* hit the wall, which is what "×N" claims to the owner. */
|
|
72
|
+
count: number
|
|
73
|
+
/** The newest event timestamp already counted — the overlap discriminator. */
|
|
74
|
+
lastEventMs?: number
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface NoticeBoardOptions {
|
|
78
|
+
/** cfg.eventLogDir — notices.log lives here (falsy → no durable log, hermetic). */
|
|
79
|
+
logDir?: string
|
|
80
|
+
env?: NodeJS.ProcessEnv
|
|
81
|
+
/** How long a notice stays live, and therefore how long the same condition is deduped.
|
|
82
|
+
* Default IAPEER_NOTICE_TTL_MS or 1 h. */
|
|
83
|
+
ttlMs?: number
|
|
84
|
+
/** Injectable clock (tests). Default Date.now. */
|
|
85
|
+
now?: () => number
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const DEFAULT_TTL_MS = 3_600_000
|
|
89
|
+
|
|
90
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
91
|
+
const n = parseInt(raw ?? '', 10)
|
|
92
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function firstLine(s: string): string {
|
|
96
|
+
const nl = s.indexOf('\n')
|
|
97
|
+
return (nl >= 0 ? s.slice(0, nl) : s).trim()
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The dedup identity of a condition: the same peer hitting the same wall on the same
|
|
101
|
+
* runtime with the same model. Model is part of the key on purpose — a peer that falls
|
|
102
|
+
* back from one model to another and hits ITS limit too is a NEW fact for the owner. */
|
|
103
|
+
export function noticeDedupKey(input: Pick<NoticeInput, 'personality' | 'runtime' | 'kind' | 'errorType' | 'model'>): string {
|
|
104
|
+
return [input.personality, input.runtime, input.kind, input.errorType, input.model ?? ''].join('|')
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export class NoticeBoard {
|
|
108
|
+
private readonly live = new Map<string, Notice>()
|
|
109
|
+
private counter = 0
|
|
110
|
+
private readonly logDir?: string
|
|
111
|
+
private readonly env: NodeJS.ProcessEnv
|
|
112
|
+
private readonly ttlMs: number
|
|
113
|
+
private readonly now: () => number
|
|
114
|
+
|
|
115
|
+
constructor(opts: NoticeBoardOptions = {}) {
|
|
116
|
+
this.logDir = opts.logDir
|
|
117
|
+
this.env = opts.env ?? process.env
|
|
118
|
+
this.ttlMs = opts.ttlMs ?? envPosInt(this.env.IAPEER_NOTICE_TTL_MS, DEFAULT_TTL_MS)
|
|
119
|
+
this.now = opts.now ?? Date.now
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Raise a notice, or fold it into the live one with the same dedupKey.
|
|
124
|
+
* Returns the notice and whether this call DEDUPED (folded) rather than raised. A
|
|
125
|
+
* folded call is deliberately silent: no id, no log line, no SSE event — only
|
|
126
|
+
* `count`/`lastMs` move (see the dedup note in the module header).
|
|
127
|
+
*/
|
|
128
|
+
raise(input: NoticeInput): { notice: Notice; deduped: boolean } {
|
|
129
|
+
const nowMs = this.now()
|
|
130
|
+
this.prune(nowMs)
|
|
131
|
+
const dedupKey = noticeDedupKey(input)
|
|
132
|
+
const existing = this.live.get(dedupKey)
|
|
133
|
+
if (existing) {
|
|
134
|
+
// Same condition. Is this a NEW occurrence, or the same runtime event re-read by an
|
|
135
|
+
// overlapping sweep? Counting the latter would inflate "×N" into a claim the owner
|
|
136
|
+
// cannot check — 2 real refusals rendering as ×3. Only a strictly newer event counts.
|
|
137
|
+
const at = input.eventAtMs
|
|
138
|
+
const isNewOccurrence = at === undefined || existing.lastEventMs === undefined || at > existing.lastEventMs
|
|
139
|
+
if (isNewOccurrence) {
|
|
140
|
+
existing.count += 1
|
|
141
|
+
existing.lastMs = nowMs
|
|
142
|
+
if (at !== undefined) existing.lastEventMs = at
|
|
143
|
+
}
|
|
144
|
+
return { notice: existing, deduped: true }
|
|
145
|
+
}
|
|
146
|
+
const id = `n${++this.counter}`
|
|
147
|
+
const summary = (input.summary?.trim() || firstLine(input.content) || input.errorType).slice(0, 240)
|
|
148
|
+
const notice: Notice = {
|
|
149
|
+
...input,
|
|
150
|
+
id,
|
|
151
|
+
summary,
|
|
152
|
+
dedupKey,
|
|
153
|
+
createdMs: nowMs,
|
|
154
|
+
lastMs: nowMs,
|
|
155
|
+
expiresMs: nowMs + this.ttlMs,
|
|
156
|
+
count: 1,
|
|
157
|
+
lastEventMs: input.eventAtMs,
|
|
158
|
+
}
|
|
159
|
+
this.live.set(dedupKey, notice)
|
|
160
|
+
appendNoticeEvent(
|
|
161
|
+
this.logDir,
|
|
162
|
+
{
|
|
163
|
+
ev: 'notice-raised',
|
|
164
|
+
id,
|
|
165
|
+
personality: input.personality,
|
|
166
|
+
runtime: input.runtime,
|
|
167
|
+
kind: input.kind,
|
|
168
|
+
error_type: input.errorType,
|
|
169
|
+
model: input.model,
|
|
170
|
+
// Emit the reset as BOTH epoch-ms and a readable stamp, or omit entirely.
|
|
171
|
+
// A reader must never have to guess whether "unknown" means "no limit" or
|
|
172
|
+
// "we didn't look" — an absent field means the runtime did not state it.
|
|
173
|
+
resets_at: input.resetsAtMs,
|
|
174
|
+
resets_at_iso: input.resetsAtMs ? new Date(input.resetsAtMs).toISOString() : undefined,
|
|
175
|
+
session: input.sessionId,
|
|
176
|
+
summary,
|
|
177
|
+
created: nowMs,
|
|
178
|
+
expires: notice.expiresMs,
|
|
179
|
+
},
|
|
180
|
+
{ env: this.env, nowMs },
|
|
181
|
+
)
|
|
182
|
+
return { notice, deduped: false }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** All live notices, oldest first (GET /fleet/v1/notices + the snapshot field). */
|
|
186
|
+
list(nowMs: number = this.now()): Notice[] {
|
|
187
|
+
this.prune(nowMs)
|
|
188
|
+
return [...this.live.values()].sort((a, b) => a.createdMs - b.createdMs)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** One live notice by id (GET /fleet/v1/notices/<id> — full verbatim content). */
|
|
192
|
+
get(id: string, nowMs: number = this.now()): Notice | undefined {
|
|
193
|
+
this.prune(nowMs)
|
|
194
|
+
return [...this.live.values()].find(n => n.id === id)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Live count (badge/metrics). */
|
|
198
|
+
size(nowMs: number = this.now()): number {
|
|
199
|
+
this.prune(nowMs)
|
|
200
|
+
return this.live.size
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Drop expired notices. Silent by design: an expiry is not news — either the peer
|
|
204
|
+
* recovered (nothing to say) or it is still broken and the next detection re-raises. */
|
|
205
|
+
prune(nowMs: number = this.now()): void {
|
|
206
|
+
for (const [key, n] of this.live) if (n.expiresMs <= nowMs) this.live.delete(key)
|
|
207
|
+
}
|
|
208
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Notice-board durable log — the daemon's DURABLE trace of every owner-facing notice it
|
|
2
|
+
// raises (docs/19). Sibling to lifecycle.log / delivery.log / exits.log / approvals.log
|
|
3
|
+
// under cfg.eventLogDir; tailed by the fleet-API SSE stream (fleetEventFiles), so the
|
|
4
|
+
// `notice-raised` event reaches every operator client (tray, telegram, web) with
|
|
5
|
+
// at-least-once semantics — and a postmortem can reconstruct which peer went mute, why,
|
|
6
|
+
// and when, after the in-memory board is gone.
|
|
7
|
+
//
|
|
8
|
+
// What is logged: notice METADATA + a one-line `summary` — NOT the full verbatim runtime
|
|
9
|
+
// message (`content`), which can be multi-line. The full content lives in the in-memory
|
|
10
|
+
// board and is served by GET /fleet/v1/notices/<id>; the log carries enough to correlate
|
|
11
|
+
// + render a badge/line.
|
|
12
|
+
//
|
|
13
|
+
// Same rotation class as its siblings: written by OUR code → self-rotates via the shared
|
|
14
|
+
// storage/rotatelog primitive. The dir is passed IN by the composition point, never
|
|
15
|
+
// re-resolved from env — a falsy dir → no-op (library/test daemons stay hermetic).
|
|
16
|
+
|
|
17
|
+
import { join } from 'path'
|
|
18
|
+
import { DEFAULT_LOG_KEEP, DEFAULT_LOG_MAX_BYTES, appendRotatedEvent } from '../storage/rotatelog.ts'
|
|
19
|
+
|
|
20
|
+
/** The notice-board log inside `logDir` (sibling to approvals.log). */
|
|
21
|
+
export function noticesLogPath(logDir: string): string {
|
|
22
|
+
return join(logDir, 'notices.log')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
26
|
+
const n = parseInt(raw ?? '', 10)
|
|
27
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface AppendNoticeOptions {
|
|
31
|
+
/** Reads the rotation knobs IAPEER_NOTICES_LOG_MAX_BYTES / _KEEP. */
|
|
32
|
+
env?: NodeJS.ProcessEnv
|
|
33
|
+
/** Stamp the line with this epoch-ms. Default Date.now(). */
|
|
34
|
+
nowMs?: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Append one notice event line into `logDir`/notices.log. A falsy `logDir` is a no-op
|
|
39
|
+
* (the default for library/test daemons). Fully best-effort — never throws; logging must
|
|
40
|
+
* never fail a detection sweep.
|
|
41
|
+
*/
|
|
42
|
+
export function appendNoticeEvent(
|
|
43
|
+
logDir: string | undefined,
|
|
44
|
+
fields: Record<string, string | number | undefined>,
|
|
45
|
+
opts: AppendNoticeOptions = {},
|
|
46
|
+
): void {
|
|
47
|
+
if (!logDir) return
|
|
48
|
+
const env = opts.env ?? process.env
|
|
49
|
+
appendRotatedEvent(noticesLogPath(logDir), fields, {
|
|
50
|
+
nowMs: opts.nowMs,
|
|
51
|
+
maxBytes: envPosInt(env.IAPEER_NOTICES_LOG_MAX_BYTES, DEFAULT_LOG_MAX_BYTES),
|
|
52
|
+
keep: envPosInt(env.IAPEER_NOTICES_LOG_KEEP, DEFAULT_LOG_KEEP),
|
|
53
|
+
})
|
|
54
|
+
}
|