@agfpd/iapeer 0.4.91 → 0.4.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/15-fleet-api.md +12 -4
- package/docs/19-notices.md +195 -0
- package/docs/README.md +1 -0
- package/package.json +1 -1
- package/src/approval/brokerClient.ts +10 -0
- package/src/daemon/fleet.ts +77 -1
- package/src/daemon/main.ts +34 -2
- package/src/daemon/mutewatch.ts +438 -0
- package/src/daemon/notices.ts +190 -0
- package/src/daemon/noticeslog.ts +54 -0
- package/src/supervisor/approvalRoute.ts +75 -7
- package/src/supervisor/daemon.ts +11 -0
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
// Mute-watch — the daemon's detector for the ONE class of failure that is invisible to
|
|
2
|
+
// every health signal it has: a structural API error left a peer unable to answer (docs/19).
|
|
3
|
+
//
|
|
4
|
+
// WHY THIS EXISTS. When a model's limit is exhausted the peer's session does NOT die. Proven
|
|
5
|
+
// live (15.07.2026, exhausted fable bucket): the banner paints, the composer paints, the
|
|
6
|
+
// ready-gate would flip READY, the delivered turn is CONSUMED, the runtime paints its refusal,
|
|
7
|
+
// and the session returns to the composer and lives on. The process is alive, there is no exit
|
|
8
|
+
// code, and the statusline keeps re-rendering — which advances the pane-log mtime, so the
|
|
9
|
+
// activity proxy reports the delivery CONFIRMED. Every daemon health signal is green and the
|
|
10
|
+
// peer is mute. The owner writes and gets silence, with nothing anywhere saying why.
|
|
11
|
+
// The exit code is no help either: `-p` (headless) exits 1 on a limit, but live peers are
|
|
12
|
+
// never launched that way — an interactive session just sits there.
|
|
13
|
+
//
|
|
14
|
+
// WHERE THE TRUTH IS. Not the pane: the refusal is prose that line-wraps, and matching wrapped
|
|
15
|
+
// ANSI is a losing game. Both runtimes write the fact STRUCTURALLY to their own session JSONL:
|
|
16
|
+
//
|
|
17
|
+
// • claude → the transcript (~/.claude/projects/<slug>/<sessionId>.jsonl):
|
|
18
|
+
// isApiErrorMessage: true, error: "rate_limit", apiErrorStatus: 429,
|
|
19
|
+
// message.model: "<synthetic>" (a fabricated message, not a model reply).
|
|
20
|
+
// `error` is a locale-stable enum and the WHOLE CLASS rides it — rate_limit is one value;
|
|
21
|
+
// overloaded / expired auth arrive on the same field, which is why this detector keys on
|
|
22
|
+
// isApiErrorMessage and reports `error` verbatim instead of hunting one string.
|
|
23
|
+
// Claude does NOT state when a per-model bucket lifts ("Please try again later") → the
|
|
24
|
+
// notice omits the reset rather than extrapolating from the 5h/7d buckets (a DIFFERENT
|
|
25
|
+
// limit: measured live at 5h 11% / 7d 66% while fable was fully exhausted).
|
|
26
|
+
//
|
|
27
|
+
// • codex → the rollout (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl): an `event_msg` /
|
|
28
|
+
// `token_count` payload carries RateLimitSnapshot.rate_limit_reached_type — null while
|
|
29
|
+
// healthy, non-null once a wall is hit. We key on NON-NULL, never on the variant strings
|
|
30
|
+
// (`rate_limit_reached`, `workspace_owner_usage_limit_reached`, … — read out of the 0.144.1
|
|
31
|
+
// binary but never observed live here), so an unknown future variant still detects.
|
|
32
|
+
// Codex DOES carry resets_at per window → its notice states the reset time.
|
|
33
|
+
//
|
|
34
|
+
// PATHS ARE NEVER DERIVED FROM cwd. The obvious route — slugify the peer's registry cwd into
|
|
35
|
+
// ~/.claude/projects/<slug> — is a trap: the slug mirrors the cwd STRING as the process was
|
|
36
|
+
// launched, not the real path. This host carries BOTH `-Users-macmini-Projects-IAPeer` and
|
|
37
|
+
// `-Users-macmini-Projects-iapeer` for ONE case-insensitive directory. So we go the other way:
|
|
38
|
+
// scan the session files that CHANGED, read the cwd each file states about ITSELF (claude: the
|
|
39
|
+
// `cwd` field on every line; codex: session_meta.payload.cwd), and match that against the
|
|
40
|
+
// registry. A file whose cwd is no peer's is ignored — a human's own session never notifies.
|
|
41
|
+
|
|
42
|
+
import { existsSync, openSync, closeSync, fstatSync, readSync, readdirSync, statSync } from 'node:fs'
|
|
43
|
+
import { join } from 'node:path'
|
|
44
|
+
import type { LifecycleConfig } from '../lifecycle/index.ts'
|
|
45
|
+
import type { NoticeBoard } from './notices.ts'
|
|
46
|
+
|
|
47
|
+
/** How much of a session file's tail is parsed. The evidence is always at the end. */
|
|
48
|
+
const TAIL_BYTES = 256 * 1024
|
|
49
|
+
/** Overlap added to the since-boundary so an event landing between two sweeps is never
|
|
50
|
+
* missed. Re-detections are free: the board's dedup folds them into the live notice. */
|
|
51
|
+
const SCAN_OVERLAP_MS = 15_000
|
|
52
|
+
/** Walk depth for the codex YYYY/MM/DD tree (3 levels + the file). */
|
|
53
|
+
const CODEX_WALK_DEPTH = 3
|
|
54
|
+
|
|
55
|
+
export interface MuteDetection {
|
|
56
|
+
personality: string
|
|
57
|
+
runtime: 'claude' | 'codex'
|
|
58
|
+
/** The runtime's OWN taxonomy value — claude's `error`, codex's rate_limit_reached_type. */
|
|
59
|
+
errorType: string
|
|
60
|
+
model?: string
|
|
61
|
+
resetsAtMs?: number
|
|
62
|
+
/** Claude: the runtime's VERBATIM refusal. Codex: a line rendered from its typed fields
|
|
63
|
+
* (the rollout carries no prose). Either way, never our interpretation of the cause. */
|
|
64
|
+
content: string
|
|
65
|
+
sessionId?: string
|
|
66
|
+
atMs: number
|
|
67
|
+
cwd: string
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
type RawDetection = Omit<MuteDetection, 'personality' | 'runtime'>
|
|
71
|
+
|
|
72
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
73
|
+
// Pure parsers — string in, detection out (fs-free: tests need no temp tree)
|
|
74
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
75
|
+
|
|
76
|
+
function toMs(ts: unknown): number | null {
|
|
77
|
+
if (typeof ts !== 'string') return null
|
|
78
|
+
const ms = Date.parse(ts)
|
|
79
|
+
return Number.isFinite(ms) ? ms : null
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The model out of the runtime's own prose ("You've reached your Fable 5 limit"). Used ONLY
|
|
83
|
+
* when the transcript carries no real model reply to read it from — a session that hits the
|
|
84
|
+
* wall on its FIRST turn has no such line. Bounded and anchored; a miss → undefined (omit). */
|
|
85
|
+
export function modelFromClaudeText(text: string): string | undefined {
|
|
86
|
+
const m = /reached your\s+(.{1,40}?)\s+limit/i.exec(text)
|
|
87
|
+
return m?.[1]?.trim() || undefined
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The NEWEST structural API-error line in a claude transcript newer than `sinceMs`, or null.
|
|
92
|
+
* Keys on `isApiErrorMessage` — the class, not the `rate_limit` instance.
|
|
93
|
+
*/
|
|
94
|
+
export function parseClaudeTranscript(text: string, sinceMs: number): RawDetection | null {
|
|
95
|
+
let lastRealModel: string | undefined
|
|
96
|
+
let found: RawDetection | null = null
|
|
97
|
+
for (const line of text.split('\n')) {
|
|
98
|
+
const s = line.trim()
|
|
99
|
+
if (!s.startsWith('{')) continue // a partial first line from the tail cut — skip
|
|
100
|
+
let d: Record<string, unknown>
|
|
101
|
+
try {
|
|
102
|
+
d = JSON.parse(s) as Record<string, unknown>
|
|
103
|
+
} catch {
|
|
104
|
+
continue // truncated/garbled line — never fail the sweep over one bad line
|
|
105
|
+
}
|
|
106
|
+
const msg = d.message as { model?: unknown; content?: unknown } | undefined
|
|
107
|
+
// Track the last REAL model reply: the error line itself is "<synthetic>" and names no model.
|
|
108
|
+
if (msg && typeof msg.model === 'string' && msg.model && msg.model !== '<synthetic>') lastRealModel = msg.model
|
|
109
|
+
if (d.isApiErrorMessage !== true) continue
|
|
110
|
+
const atMs = toMs(d.timestamp)
|
|
111
|
+
if (atMs === null || atMs <= sinceMs) continue
|
|
112
|
+
const cwd = typeof d.cwd === 'string' ? d.cwd : ''
|
|
113
|
+
if (!cwd) continue // no cwd → unattributable to a peer; ignore rather than guess
|
|
114
|
+
const errorType = typeof d.error === 'string' && d.error.trim() ? d.error.trim() : 'api-error'
|
|
115
|
+
let content = ''
|
|
116
|
+
if (Array.isArray(msg?.content)) {
|
|
117
|
+
content = (msg.content as Array<{ type?: string; text?: string }>)
|
|
118
|
+
.filter(c => c?.type === 'text' && typeof c.text === 'string')
|
|
119
|
+
.map(c => c.text as string)
|
|
120
|
+
.join('\n')
|
|
121
|
+
.trim()
|
|
122
|
+
}
|
|
123
|
+
if (!content) content = `${errorType} (api error ${String(d.apiErrorStatus ?? '')})`.trim()
|
|
124
|
+
found = {
|
|
125
|
+
errorType,
|
|
126
|
+
model: lastRealModel ?? modelFromClaudeText(content),
|
|
127
|
+
content,
|
|
128
|
+
sessionId: typeof d.sessionId === 'string' ? d.sessionId : undefined,
|
|
129
|
+
atMs,
|
|
130
|
+
cwd,
|
|
131
|
+
// Claude states no reset for a per-model bucket → resetsAtMs deliberately absent.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return found
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
interface CodexWindow {
|
|
138
|
+
used_percent?: unknown
|
|
139
|
+
window_minutes?: unknown
|
|
140
|
+
resets_at?: unknown
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Describe a codex window for the human-readable content line. */
|
|
144
|
+
function describeWindow(name: string, w: CodexWindow): string | null {
|
|
145
|
+
if (!w || typeof w.used_percent !== 'number') return null
|
|
146
|
+
const win = typeof w.window_minutes === 'number' ? `${w.window_minutes}m window` : 'window'
|
|
147
|
+
const reset =
|
|
148
|
+
typeof w.resets_at === 'number' ? `, resets ${new Date(w.resets_at * 1000).toISOString()}` : ''
|
|
149
|
+
return `${name} ${w.used_percent}% (${win}${reset})`
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The codex rollout's newest limit-reached snapshot newer than `sinceMs`, or null.
|
|
154
|
+
* Keys on `rate_limit_reached_type` being NON-NULL — never on its variant strings.
|
|
155
|
+
* `cwd` comes from session_meta, which is the file's FIRST line — pass the file HEAD in
|
|
156
|
+
* `head` when the tail alone may have cut it away.
|
|
157
|
+
*/
|
|
158
|
+
export function parseCodexRollout(text: string, sinceMs: number, head = ''): RawDetection | null {
|
|
159
|
+
let cwd = ''
|
|
160
|
+
let sessionId: string | undefined
|
|
161
|
+
const scanMeta = (src: string): void => {
|
|
162
|
+
for (const line of src.split('\n')) {
|
|
163
|
+
const s = line.trim()
|
|
164
|
+
if (!s.startsWith('{') || !s.includes('session_meta')) continue
|
|
165
|
+
try {
|
|
166
|
+
const d = JSON.parse(s) as { type?: string; payload?: { cwd?: unknown; session_id?: unknown } }
|
|
167
|
+
if (d.type !== 'session_meta' || !d.payload) continue
|
|
168
|
+
if (typeof d.payload.cwd === 'string') cwd = d.payload.cwd
|
|
169
|
+
if (typeof d.payload.session_id === 'string') sessionId = d.payload.session_id
|
|
170
|
+
} catch {
|
|
171
|
+
/* ignore */
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
scanMeta(head)
|
|
176
|
+
if (!cwd) scanMeta(text)
|
|
177
|
+
if (!cwd) return null // unattributable to a peer
|
|
178
|
+
|
|
179
|
+
let found: RawDetection | null = null
|
|
180
|
+
for (const line of text.split('\n')) {
|
|
181
|
+
const s = line.trim()
|
|
182
|
+
if (!s.startsWith('{') || !s.includes('rate_limits')) continue
|
|
183
|
+
let d: { timestamp?: unknown; type?: unknown; payload?: { type?: unknown; rate_limits?: Record<string, unknown> } }
|
|
184
|
+
try {
|
|
185
|
+
d = JSON.parse(s)
|
|
186
|
+
} catch {
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
const rl = d.payload?.rate_limits
|
|
190
|
+
if (!rl) continue
|
|
191
|
+
const reached = rl.rate_limit_reached_type
|
|
192
|
+
// NON-NULL is the signal. An unknown future variant must still detect, so we never
|
|
193
|
+
// compare against the variant strings read out of the binary.
|
|
194
|
+
if (reached === null || reached === undefined || reached === '') continue
|
|
195
|
+
const atMs = toMs(d.timestamp)
|
|
196
|
+
if (atMs === null || atMs <= sinceMs) continue
|
|
197
|
+
const primary = (rl.primary ?? {}) as CodexWindow
|
|
198
|
+
const secondary = (rl.secondary ?? {}) as CodexWindow
|
|
199
|
+
// Which window is blocking? Codex does not say — reported for the FULLEST one, the
|
|
200
|
+
// only defensible read. This mapping is a HEURISTIC: it could not be verified against
|
|
201
|
+
// a real reached-snapshot (the quota never ran out on this host). Both windows ride in
|
|
202
|
+
// `content`, so the human sees the raw fact even if the heuristic picked wrong.
|
|
203
|
+
const pu = typeof primary.used_percent === 'number' ? primary.used_percent : -1
|
|
204
|
+
const su = typeof secondary.used_percent === 'number' ? secondary.used_percent : -1
|
|
205
|
+
const blocking = su > pu ? secondary : primary
|
|
206
|
+
const resetsAtMs = typeof blocking.resets_at === 'number' ? blocking.resets_at * 1000 : undefined
|
|
207
|
+
const parts = [describeWindow('primary', primary), describeWindow('secondary', secondary)].filter(Boolean)
|
|
208
|
+
const plan = typeof rl.plan_type === 'string' ? ` plan=${rl.plan_type}` : ''
|
|
209
|
+
found = {
|
|
210
|
+
errorType: String(reached),
|
|
211
|
+
content: `Codex usage limit reached (${String(reached)})${plan}${parts.length ? ` — ${parts.join('; ')}` : ''}`,
|
|
212
|
+
resetsAtMs,
|
|
213
|
+
sessionId,
|
|
214
|
+
atMs,
|
|
215
|
+
cwd,
|
|
216
|
+
// Codex names no model in the snapshot → model omitted rather than guessed.
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return found
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
223
|
+
// The sweep — find changed session files, attribute them, raise notices
|
|
224
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
225
|
+
|
|
226
|
+
function readTail(path: string, maxBytes = TAIL_BYTES): { tail: string; head: string } | null {
|
|
227
|
+
try {
|
|
228
|
+
const fd = openSync(path, 'r')
|
|
229
|
+
try {
|
|
230
|
+
const size = fstatSync(fd).size
|
|
231
|
+
if (size <= 0) return null
|
|
232
|
+
const off = Math.max(0, size - maxBytes)
|
|
233
|
+
const buf = Buffer.alloc(size - off)
|
|
234
|
+
readSync(fd, buf, 0, buf.length, off)
|
|
235
|
+
let head = ''
|
|
236
|
+
if (off > 0) {
|
|
237
|
+
// The tail cut away the file's start — read a HEAD slice too (codex session_meta
|
|
238
|
+
// is line 1, and a long session pushes it far out of the tail).
|
|
239
|
+
const hb = Buffer.alloc(Math.min(64 * 1024, off))
|
|
240
|
+
readSync(fd, hb, 0, hb.length, 0)
|
|
241
|
+
head = hb.toString('utf8')
|
|
242
|
+
}
|
|
243
|
+
return { tail: buf.toString('utf8'), head }
|
|
244
|
+
} finally {
|
|
245
|
+
closeSync(fd)
|
|
246
|
+
}
|
|
247
|
+
} catch {
|
|
248
|
+
return null
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function safeReaddir(dir: string): string[] {
|
|
253
|
+
try {
|
|
254
|
+
return readdirSync(dir)
|
|
255
|
+
} catch {
|
|
256
|
+
return []
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function mtimeMs(path: string): number | null {
|
|
261
|
+
try {
|
|
262
|
+
return statSync(path).mtimeMs
|
|
263
|
+
} catch {
|
|
264
|
+
return null
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** *.jsonl files under `root` (depth-bounded) whose mtime is at/after `sinceMs`. */
|
|
269
|
+
function changedJsonl(root: string, sinceMs: number, depth: number): string[] {
|
|
270
|
+
if (!existsSync(root)) return []
|
|
271
|
+
const out: string[] = []
|
|
272
|
+
const walk = (dir: string, left: number): void => {
|
|
273
|
+
for (const name of safeReaddir(dir)) {
|
|
274
|
+
const p = join(dir, name)
|
|
275
|
+
if (name.endsWith('.jsonl')) {
|
|
276
|
+
const m = mtimeMs(p)
|
|
277
|
+
if (m !== null && m >= sinceMs) out.push(p)
|
|
278
|
+
continue
|
|
279
|
+
}
|
|
280
|
+
if (left <= 0 || name.startsWith('.')) continue
|
|
281
|
+
let isDir = false
|
|
282
|
+
try {
|
|
283
|
+
isDir = statSync(p).isDirectory()
|
|
284
|
+
} catch {
|
|
285
|
+
continue
|
|
286
|
+
}
|
|
287
|
+
if (isDir) walk(p, left - 1)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
walk(root, depth)
|
|
291
|
+
return out
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export interface MuteWatchPaths {
|
|
295
|
+
/** ~/.claude/projects — INJECTED, never re-resolved from env inside the sweep. */
|
|
296
|
+
claudeProjectsDir: string
|
|
297
|
+
/** ~/.codex/sessions — same rule. */
|
|
298
|
+
codexSessionsDir: string
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export interface ScanDeps extends MuteWatchPaths {
|
|
302
|
+
/** personality ← cwd. Built from the registry by the composition point. */
|
|
303
|
+
peerByCwd: (cwd: string) => string | undefined
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* The runtime session-file roots. These are the RUNTIMES' own trees, not iapeer's, so they
|
|
308
|
+
* are not LifecycleConfig fields; the composition point resolves them ONCE and injects them
|
|
309
|
+
* (a sweep never re-resolves a path mid-run). The env overrides exist so a test points the
|
|
310
|
+
* sweep at a temp tree instead of the real ~/.claude — the sweep is READ-ONLY, and its only
|
|
311
|
+
* write (notices.log) goes through cfg.eventLogDir like every other daemon log.
|
|
312
|
+
*/
|
|
313
|
+
export function resolveMuteWatchPaths(env: NodeJS.ProcessEnv): MuteWatchPaths {
|
|
314
|
+
const home = env.HOME?.trim() || ''
|
|
315
|
+
return {
|
|
316
|
+
claudeProjectsDir: env.IAPEER_CLAUDE_PROJECTS_DIR?.trim() || join(home, '.claude', 'projects'),
|
|
317
|
+
codexSessionsDir: env.IAPEER_CODEX_SESSIONS_DIR?.trim() || join(home, '.codex', 'sessions'),
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** One sweep: every peer-attributable structural API error newer than `sinceMs`. */
|
|
322
|
+
export function scanMuteEvents(deps: ScanDeps, sinceMs: number): MuteDetection[] {
|
|
323
|
+
const floor = sinceMs - SCAN_OVERLAP_MS
|
|
324
|
+
const out: MuteDetection[] = []
|
|
325
|
+
const push = (raw: RawDetection | null, runtime: 'claude' | 'codex'): void => {
|
|
326
|
+
if (!raw) return
|
|
327
|
+
const personality = deps.peerByCwd(raw.cwd)
|
|
328
|
+
if (!personality) return // not a peer's session (a human's own shell) — never notify
|
|
329
|
+
out.push({ ...raw, personality, runtime })
|
|
330
|
+
}
|
|
331
|
+
// claude: ~/.claude/projects/<slug>/<sessionId>.jsonl — one level of slug dirs.
|
|
332
|
+
for (const file of changedJsonl(deps.claudeProjectsDir, floor, 1)) {
|
|
333
|
+
const t = readTail(file)
|
|
334
|
+
if (t) push(parseClaudeTranscript(t.tail, floor), 'claude')
|
|
335
|
+
}
|
|
336
|
+
// codex: ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl
|
|
337
|
+
for (const file of changedJsonl(deps.codexSessionsDir, floor, CODEX_WALK_DEPTH)) {
|
|
338
|
+
const t = readTail(file)
|
|
339
|
+
if (t) push(parseCodexRollout(t.tail, floor, t.head), 'codex')
|
|
340
|
+
}
|
|
341
|
+
return out
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
export interface MuteWatchDeps extends ScanDeps {
|
|
345
|
+
board: NoticeBoard
|
|
346
|
+
now?: () => number
|
|
347
|
+
/** Reported, never thrown — a detector must never take the daemon down. */
|
|
348
|
+
onError?: (err: unknown) => void
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Run one watch tick: sweep, raise. Returns the notices actually RAISED (deduped repeats
|
|
353
|
+
* are folded by the board and not returned) plus the new since-boundary.
|
|
354
|
+
*/
|
|
355
|
+
export function muteWatchTick(deps: MuteWatchDeps, sinceMs: number): { raised: number; sinceMs: number } {
|
|
356
|
+
const now = deps.now ?? Date.now
|
|
357
|
+
const startedMs = now()
|
|
358
|
+
let raised = 0
|
|
359
|
+
try {
|
|
360
|
+
for (const d of scanMuteEvents(deps, sinceMs)) {
|
|
361
|
+
const { deduped } = deps.board.raise({
|
|
362
|
+
personality: d.personality,
|
|
363
|
+
runtime: d.runtime,
|
|
364
|
+
kind: 'peer-mute',
|
|
365
|
+
errorType: d.errorType,
|
|
366
|
+
model: d.model,
|
|
367
|
+
resetsAtMs: d.resetsAtMs,
|
|
368
|
+
content: d.content,
|
|
369
|
+
sessionId: d.sessionId,
|
|
370
|
+
summary: `${d.personality} · ${d.runtime} — ${d.errorType}${d.model ? ` (${d.model})` : ''}`,
|
|
371
|
+
})
|
|
372
|
+
if (!deduped) raised += 1
|
|
373
|
+
}
|
|
374
|
+
} catch (e) {
|
|
375
|
+
deps.onError?.(e)
|
|
376
|
+
}
|
|
377
|
+
return { raised, sinceMs: startedMs }
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
export interface StartMuteWatchOptions {
|
|
381
|
+
env?: NodeJS.ProcessEnv
|
|
382
|
+
intervalMs?: number
|
|
383
|
+
/** Override the runtime session roots (tests). Default: resolveMuteWatchPaths(env). */
|
|
384
|
+
paths?: MuteWatchPaths
|
|
385
|
+
/** Override the registry read (tests). Default: a lazy import of the CLI's listPeers. */
|
|
386
|
+
listPeers?: () => Array<{ personality: string; cwd: string }>
|
|
387
|
+
now?: () => number
|
|
388
|
+
onError?: (err: unknown) => void
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Start the mute-watch timer. Returns its stop function — the caller OWNS teardown.
|
|
393
|
+
*
|
|
394
|
+
* The registry is re-read every sweep (a peer created since daemon start must be watched
|
|
395
|
+
* without a restart), and `listPeers` is imported LAZILY at first use, mirroring fleet.ts:
|
|
396
|
+
* daemon → cli is not a load-time dependency in this codebase.
|
|
397
|
+
*/
|
|
398
|
+
export function startMuteWatch(cfg: LifecycleConfig, board: NoticeBoard, opts: StartMuteWatchOptions = {}): () => void {
|
|
399
|
+
const env = opts.env ?? process.env
|
|
400
|
+
const paths = opts.paths ?? resolveMuteWatchPaths(env)
|
|
401
|
+
const now = opts.now ?? Date.now
|
|
402
|
+
let listPeers = opts.listPeers
|
|
403
|
+
let since = now()
|
|
404
|
+
let running = false
|
|
405
|
+
|
|
406
|
+
const sweep = async (): Promise<void> => {
|
|
407
|
+
if (running) return // a slow sweep must never overlap itself into a stampede
|
|
408
|
+
running = true
|
|
409
|
+
try {
|
|
410
|
+
if (!listPeers) {
|
|
411
|
+
const cli = await import('../cli/index.ts')
|
|
412
|
+
listPeers = () => cli.listPeers({ env }).map(p => ({ personality: p.personality, cwd: p.cwd }))
|
|
413
|
+
}
|
|
414
|
+
const rows = listPeers()
|
|
415
|
+
// cwd → personality. Compared case-insensitively: macOS paths are case-insensitive and
|
|
416
|
+
// the runtimes record the cwd STRING as launched, which is not always the registry's
|
|
417
|
+
// capitalization (the same trap that rules out deriving the transcript dir from cwd).
|
|
418
|
+
const byCwd = new Map<string, string>()
|
|
419
|
+
for (const r of rows) if (r.cwd) byCwd.set(r.cwd.replace(/\/+$/, '').toLowerCase(), r.personality)
|
|
420
|
+
const deps: MuteWatchDeps = {
|
|
421
|
+
...paths,
|
|
422
|
+
board,
|
|
423
|
+
now,
|
|
424
|
+
onError: opts.onError,
|
|
425
|
+
peerByCwd: (cwd: string) => byCwd.get(cwd.replace(/\/+$/, '').toLowerCase()),
|
|
426
|
+
}
|
|
427
|
+
since = muteWatchTick(deps, since).sinceMs
|
|
428
|
+
} catch (e) {
|
|
429
|
+
opts.onError?.(e)
|
|
430
|
+
} finally {
|
|
431
|
+
running = false
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const timer = setInterval(() => void sweep(), opts.intervalMs ?? 20_000)
|
|
436
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
437
|
+
return () => clearInterval(timer)
|
|
438
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// Notice board — the in-daemon store of OWNER-FACING notices (docs/19). The daemon
|
|
2
|
+
// noticed something the owner must know about a peer, and no one else can tell him: the
|
|
3
|
+
// canonical case is a peer left MUTE by a structural API error (rate limit, overload,
|
|
4
|
+
// expired auth) — such a peer cannot report its own failure, and an Implementer peer has
|
|
5
|
+
// no telegram face of its own. So the notice travels the daemon's OWN surface, exactly as
|
|
6
|
+
// an approval request does.
|
|
7
|
+
//
|
|
8
|
+
// DELIBERATELY NOT AN APPROVAL. An approval is request → decision → unblock: a caller
|
|
9
|
+
// long-polls and something is BLOCKED until a human answers. A notice is one-way: nobody
|
|
10
|
+
// waits, nothing unblocks, there is no decision to make. Reusing the approval broker here
|
|
11
|
+
// would drag a promise/timeout/fail-safe-deny machine into a problem that has none of
|
|
12
|
+
// those. What IS shared is the SURFACE: a durable log tailed into the fleet SSE stream, an
|
|
13
|
+
// additive snapshot field, and a GET endpoint for the verbatim content.
|
|
14
|
+
//
|
|
15
|
+
// TTL + DEDUP, and the two are ONE mechanism. A mute peer re-emits its error on EVERY
|
|
16
|
+
// attempted turn — a naive board would raise a card per attempt and bury the owner. So a
|
|
17
|
+
// notice carries a `dedupKey` (peer + runtime + kind + errorType + model): while a notice
|
|
18
|
+
// with that key is LIVE, a repeat detection only bumps `count`/`lastMs` — no new id, no new
|
|
19
|
+
// log line, no new SSE event. The owner sees one card that says "×7", not seven cards.
|
|
20
|
+
// When it expires (TTL) and the peer is STILL broken, the next detection raises a fresh
|
|
21
|
+
// notice — a deliberate periodic reminder rather than one card that scrolls away forever.
|
|
22
|
+
//
|
|
23
|
+
// In-memory + ephemeral: notices do NOT survive a daemon restart (a restart re-detects
|
|
24
|
+
// within one watch tick — the transcript evidence is on disk and is re-read). The durable
|
|
25
|
+
// notices.log is the audit/observability trace, not a recovery store.
|
|
26
|
+
|
|
27
|
+
import { appendNoticeEvent } from './noticeslog.ts'
|
|
28
|
+
|
|
29
|
+
/** v1 taxonomy of notices. `peer-mute` = a structural API error left the peer unable to
|
|
30
|
+
* answer. Free-form-tolerant — new kinds ride the same surface without a client change. */
|
|
31
|
+
export type NoticeKind = 'peer-mute' | string
|
|
32
|
+
|
|
33
|
+
export interface NoticeInput {
|
|
34
|
+
personality: string
|
|
35
|
+
runtime: string
|
|
36
|
+
/** Taxonomy tag (peer-mute | …). */
|
|
37
|
+
kind: NoticeKind
|
|
38
|
+
/** The runtime's OWN error taxonomy value — `rate_limit`, `overloaded`, … Never
|
|
39
|
+
* invented by us: it is the structural field the runtime wrote (docs/19 §2). */
|
|
40
|
+
errorType: string
|
|
41
|
+
/** The model that hit the wall, when the runtime says which (claude names it only in
|
|
42
|
+
* prose; codex does not name it at all). ABSENT rather than guessed. */
|
|
43
|
+
model?: string
|
|
44
|
+
/** When the limit lifts, epoch-ms — ONLY when the runtime states it. Claude does not
|
|
45
|
+
* for a per-model bucket ("try again later"), so this is absent there: an honest
|
|
46
|
+
* omission, never an extrapolation from the 5h/7d buckets (a DIFFERENT limit). */
|
|
47
|
+
resetsAtMs?: number
|
|
48
|
+
/** One-line summary for the SSE event / badge. */
|
|
49
|
+
summary?: string
|
|
50
|
+
/** The runtime's VERBATIM message — shown to the human in every face. */
|
|
51
|
+
content: string
|
|
52
|
+
/** The session that hit it (correlation with the transcript on disk). */
|
|
53
|
+
sessionId?: string
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface Notice extends NoticeInput {
|
|
57
|
+
id: string
|
|
58
|
+
summary: string
|
|
59
|
+
dedupKey: string
|
|
60
|
+
createdMs: number
|
|
61
|
+
/** Last time this same condition was re-detected (dedup bump). */
|
|
62
|
+
lastMs: number
|
|
63
|
+
expiresMs: number
|
|
64
|
+
/** How many times the condition was detected while this notice stayed live (≥1). */
|
|
65
|
+
count: number
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface NoticeBoardOptions {
|
|
69
|
+
/** cfg.eventLogDir — notices.log lives here (falsy → no durable log, hermetic). */
|
|
70
|
+
logDir?: string
|
|
71
|
+
env?: NodeJS.ProcessEnv
|
|
72
|
+
/** How long a notice stays live, and therefore how long the same condition is deduped.
|
|
73
|
+
* Default IAPEER_NOTICE_TTL_MS or 1 h. */
|
|
74
|
+
ttlMs?: number
|
|
75
|
+
/** Injectable clock (tests). Default Date.now. */
|
|
76
|
+
now?: () => number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const DEFAULT_TTL_MS = 3_600_000
|
|
80
|
+
|
|
81
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
82
|
+
const n = parseInt(raw ?? '', 10)
|
|
83
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function firstLine(s: string): string {
|
|
87
|
+
const nl = s.indexOf('\n')
|
|
88
|
+
return (nl >= 0 ? s.slice(0, nl) : s).trim()
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** The dedup identity of a condition: the same peer hitting the same wall on the same
|
|
92
|
+
* runtime with the same model. Model is part of the key on purpose — a peer that falls
|
|
93
|
+
* back from one model to another and hits ITS limit too is a NEW fact for the owner. */
|
|
94
|
+
export function noticeDedupKey(input: Pick<NoticeInput, 'personality' | 'runtime' | 'kind' | 'errorType' | 'model'>): string {
|
|
95
|
+
return [input.personality, input.runtime, input.kind, input.errorType, input.model ?? ''].join('|')
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export class NoticeBoard {
|
|
99
|
+
private readonly live = new Map<string, Notice>()
|
|
100
|
+
private counter = 0
|
|
101
|
+
private readonly logDir?: string
|
|
102
|
+
private readonly env: NodeJS.ProcessEnv
|
|
103
|
+
private readonly ttlMs: number
|
|
104
|
+
private readonly now: () => number
|
|
105
|
+
|
|
106
|
+
constructor(opts: NoticeBoardOptions = {}) {
|
|
107
|
+
this.logDir = opts.logDir
|
|
108
|
+
this.env = opts.env ?? process.env
|
|
109
|
+
this.ttlMs = opts.ttlMs ?? envPosInt(this.env.IAPEER_NOTICE_TTL_MS, DEFAULT_TTL_MS)
|
|
110
|
+
this.now = opts.now ?? Date.now
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Raise a notice, or fold it into the live one with the same dedupKey.
|
|
115
|
+
* Returns the notice and whether this call DEDUPED (folded) rather than raised. A
|
|
116
|
+
* folded call is deliberately silent: no id, no log line, no SSE event — only
|
|
117
|
+
* `count`/`lastMs` move (see the dedup note in the module header).
|
|
118
|
+
*/
|
|
119
|
+
raise(input: NoticeInput): { notice: Notice; deduped: boolean } {
|
|
120
|
+
const nowMs = this.now()
|
|
121
|
+
this.prune(nowMs)
|
|
122
|
+
const dedupKey = noticeDedupKey(input)
|
|
123
|
+
const existing = this.live.get(dedupKey)
|
|
124
|
+
if (existing) {
|
|
125
|
+
existing.count += 1
|
|
126
|
+
existing.lastMs = nowMs
|
|
127
|
+
return { notice: existing, deduped: true }
|
|
128
|
+
}
|
|
129
|
+
const id = `n${++this.counter}`
|
|
130
|
+
const summary = (input.summary?.trim() || firstLine(input.content) || input.errorType).slice(0, 240)
|
|
131
|
+
const notice: Notice = {
|
|
132
|
+
...input,
|
|
133
|
+
id,
|
|
134
|
+
summary,
|
|
135
|
+
dedupKey,
|
|
136
|
+
createdMs: nowMs,
|
|
137
|
+
lastMs: nowMs,
|
|
138
|
+
expiresMs: nowMs + this.ttlMs,
|
|
139
|
+
count: 1,
|
|
140
|
+
}
|
|
141
|
+
this.live.set(dedupKey, notice)
|
|
142
|
+
appendNoticeEvent(
|
|
143
|
+
this.logDir,
|
|
144
|
+
{
|
|
145
|
+
ev: 'notice-raised',
|
|
146
|
+
id,
|
|
147
|
+
personality: input.personality,
|
|
148
|
+
runtime: input.runtime,
|
|
149
|
+
kind: input.kind,
|
|
150
|
+
error_type: input.errorType,
|
|
151
|
+
model: input.model,
|
|
152
|
+
// Emit the reset as BOTH epoch-ms and a readable stamp, or omit entirely.
|
|
153
|
+
// A reader must never have to guess whether "unknown" means "no limit" or
|
|
154
|
+
// "we didn't look" — an absent field means the runtime did not state it.
|
|
155
|
+
resets_at: input.resetsAtMs,
|
|
156
|
+
resets_at_iso: input.resetsAtMs ? new Date(input.resetsAtMs).toISOString() : undefined,
|
|
157
|
+
session: input.sessionId,
|
|
158
|
+
summary,
|
|
159
|
+
created: nowMs,
|
|
160
|
+
expires: notice.expiresMs,
|
|
161
|
+
},
|
|
162
|
+
{ env: this.env, nowMs },
|
|
163
|
+
)
|
|
164
|
+
return { notice, deduped: false }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** All live notices, oldest first (GET /fleet/v1/notices + the snapshot field). */
|
|
168
|
+
list(nowMs: number = this.now()): Notice[] {
|
|
169
|
+
this.prune(nowMs)
|
|
170
|
+
return [...this.live.values()].sort((a, b) => a.createdMs - b.createdMs)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** One live notice by id (GET /fleet/v1/notices/<id> — full verbatim content). */
|
|
174
|
+
get(id: string, nowMs: number = this.now()): Notice | undefined {
|
|
175
|
+
this.prune(nowMs)
|
|
176
|
+
return [...this.live.values()].find(n => n.id === id)
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Live count (badge/metrics). */
|
|
180
|
+
size(nowMs: number = this.now()): number {
|
|
181
|
+
this.prune(nowMs)
|
|
182
|
+
return this.live.size
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Drop expired notices. Silent by design: an expiry is not news — either the peer
|
|
186
|
+
* recovered (nothing to say) or it is still broken and the next detection re-raises. */
|
|
187
|
+
prune(nowMs: number = this.now()): void {
|
|
188
|
+
for (const [key, n] of this.live) if (n.expiresMs <= nowMs) this.live.delete(key)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Notice-board durable log — the daemon's DURABLE trace of every owner-facing notice it
|
|
2
|
+
// raises (docs/19). Sibling to lifecycle.log / delivery.log / exits.log / approvals.log
|
|
3
|
+
// under cfg.eventLogDir; tailed by the fleet-API SSE stream (fleetEventFiles), so the
|
|
4
|
+
// `notice-raised` event reaches every operator client (tray, telegram, web) with
|
|
5
|
+
// at-least-once semantics — and a postmortem can reconstruct which peer went mute, why,
|
|
6
|
+
// and when, after the in-memory board is gone.
|
|
7
|
+
//
|
|
8
|
+
// What is logged: notice METADATA + a one-line `summary` — NOT the full verbatim runtime
|
|
9
|
+
// message (`content`), which can be multi-line. The full content lives in the in-memory
|
|
10
|
+
// board and is served by GET /fleet/v1/notices/<id>; the log carries enough to correlate
|
|
11
|
+
// + render a badge/line.
|
|
12
|
+
//
|
|
13
|
+
// Same rotation class as its siblings: written by OUR code → self-rotates via the shared
|
|
14
|
+
// storage/rotatelog primitive. The dir is passed IN by the composition point, never
|
|
15
|
+
// re-resolved from env — a falsy dir → no-op (library/test daemons stay hermetic).
|
|
16
|
+
|
|
17
|
+
import { join } from 'path'
|
|
18
|
+
import { DEFAULT_LOG_KEEP, DEFAULT_LOG_MAX_BYTES, appendRotatedEvent } from '../storage/rotatelog.ts'
|
|
19
|
+
|
|
20
|
+
/** The notice-board log inside `logDir` (sibling to approvals.log). */
|
|
21
|
+
export function noticesLogPath(logDir: string): string {
|
|
22
|
+
return join(logDir, 'notices.log')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function envPosInt(raw: string | undefined, dflt: number): number {
|
|
26
|
+
const n = parseInt(raw ?? '', 10)
|
|
27
|
+
return Number.isFinite(n) && n > 0 ? n : dflt
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface AppendNoticeOptions {
|
|
31
|
+
/** Reads the rotation knobs IAPEER_NOTICES_LOG_MAX_BYTES / _KEEP. */
|
|
32
|
+
env?: NodeJS.ProcessEnv
|
|
33
|
+
/** Stamp the line with this epoch-ms. Default Date.now(). */
|
|
34
|
+
nowMs?: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Append one notice event line into `logDir`/notices.log. A falsy `logDir` is a no-op
|
|
39
|
+
* (the default for library/test daemons). Fully best-effort — never throws; logging must
|
|
40
|
+
* never fail a detection sweep.
|
|
41
|
+
*/
|
|
42
|
+
export function appendNoticeEvent(
|
|
43
|
+
logDir: string | undefined,
|
|
44
|
+
fields: Record<string, string | number | undefined>,
|
|
45
|
+
opts: AppendNoticeOptions = {},
|
|
46
|
+
): void {
|
|
47
|
+
if (!logDir) return
|
|
48
|
+
const env = opts.env ?? process.env
|
|
49
|
+
appendRotatedEvent(noticesLogPath(logDir), fields, {
|
|
50
|
+
nowMs: opts.nowMs,
|
|
51
|
+
maxBytes: envPosInt(env.IAPEER_NOTICES_LOG_MAX_BYTES, DEFAULT_LOG_MAX_BYTES),
|
|
52
|
+
keep: envPosInt(env.IAPEER_NOTICES_LOG_KEEP, DEFAULT_LOG_KEEP),
|
|
53
|
+
})
|
|
54
|
+
}
|