@cohortapp/agent-sdk 2.18.36 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +3 -0
- package/docs/guides/billing-console-keys.md +16 -3
- package/docs/runbooks/cadence-requeue-storm-recovery.md +186 -0
- package/lib/assurance/first-reply.mjs +47 -2
- package/lib/assurance/notice-plane.mjs +218 -0
- package/lib/assurance/notice-voice.mjs +35 -2
- package/lib/assurance/operator-plane-store.mjs +164 -0
- package/lib/assurance/operator-plane.mjs +689 -0
- package/lib/budget-escalate.mjs +42 -4
- package/lib/budget-guard.mjs +24 -1
- package/lib/cadence-bus.mjs +359 -46
- package/lib/cli/attention.mjs +300 -0
- package/lib/cli/seat-auth.mjs +86 -6
- package/lib/comms/machine-status.mjs +377 -0
- package/lib/comms/send-gate.mjs +309 -2
- package/lib/org/awareness.mjs +15 -2
- package/lib/org/clause-match.mjs +470 -0
- package/lib/org/client.mjs +27 -1
- package/lib/org/escalation-chain.mjs +134 -0
- package/lib/org/first-run.mjs +263 -0
- package/lib/org/genesis-contract.mjs +271 -0
- package/lib/org/governance-runtime.mjs +409 -0
- package/lib/org/governance-store.mjs +238 -0
- package/lib/org/governance.mjs +326 -0
- package/lib/org/mesh.mjs +135 -0
- package/lib/org/protocol.checksum +1 -1
- package/lib/org/protocol.mjs +6 -0
- package/lib/org/tool-surface.mjs +28 -0
- package/lib/setup/enroll-from-cohort.mjs +11 -0
- package/lib/telemetry/alerts.mjs +12 -0
- package/lib/telemetry/collect.mjs +237 -1
- package/package.json +1 -1
- package/scripts/ci/check-durable-write-seam.mjs +3 -3
- package/scripts/daemon/agent-daemon.mjs +41 -2
- package/scripts/daemon/assurance.mjs +852 -49
- package/scripts/daemon/cadence-consumer.mjs +182 -9
- package/scripts/daemon/context-compiler.mjs +25 -0
- package/scripts/daemon/deliver.mjs +172 -2
- package/scripts/fleet/closing-sweep.mjs +266 -0
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/assurance/operator-plane.mjs — the operator plane's READER, as pure logic.
|
|
3
|
+
*
|
|
4
|
+
* `notice-plane.mjs` decided WHERE a machine's status about itself goes: the
|
|
5
|
+
* operator plane, never a conversation. It says so in its own reason #4 —
|
|
6
|
+
*
|
|
7
|
+
* "What was missing was never information; it was a person looking at the sink."
|
|
8
|
+
*
|
|
9
|
+
* …and then shipped without one. Measured on this machine the day after:
|
|
10
|
+
* `state/obligations/needs-attention/` held **78 rows, 26 Aug → 1 Oct**, five
|
|
11
|
+
* weeks unswept, with **two writers and zero readers** — nothing in `lib/cli/`,
|
|
12
|
+
* nothing in `bin/maestro.mjs`, no doctor check. The only thing that lifted any
|
|
13
|
+
* of it off the box was `lib/daemon/reply-debt.mjs`, and it reads three
|
|
14
|
+
* reply-path COUNTERS, none of which covers the 94% of those rows whose cause is
|
|
15
|
+
* the single label `no_outcome`. So the premise that failed — "someone will look
|
|
16
|
+
* at the sink" — was reproduced, one sink to the left.
|
|
17
|
+
*
|
|
18
|
+
* ── WHAT THIS MODULE IS FOR, AND WHY IT GROUPS ──────────────────────────────
|
|
19
|
+
*
|
|
20
|
+
* There are FIVE sinks, not the three either module's docblock lists (the
|
|
21
|
+
* journal, the row, the counters, the no-send receipt, the daemon log), and a
|
|
22
|
+
* reader has to JOIN them because no single one answers an operator's question.
|
|
23
|
+
* This module is the join and the grouping; `operator-plane-store.mjs` is the
|
|
24
|
+
* disk.
|
|
25
|
+
*
|
|
26
|
+
* Grouping is the point, not a presentation choice. `lib/cli/skills.mjs` learned
|
|
27
|
+
* the same lesson one subsystem over and wrote it down: it counted per id while
|
|
28
|
+
* the real signal was per FAMILY, and so under-reported 117 redundant files as
|
|
29
|
+
* "one". Here the equivalent mistake is listing rows. A screen of 78 individual
|
|
30
|
+
* `no_outcome` lines tells an operator nothing they can act on; two statements
|
|
31
|
+
*
|
|
32
|
+
* "this one ask has failed 31 times in 6 hours"
|
|
33
|
+
* "14 asks in one channel all died the same way — that is the channel, not the asks"
|
|
34
|
+
*
|
|
35
|
+
* are the two shapes of actionable, and both are invisible until you group.
|
|
36
|
+
* {@link groupIncidents} is the first (per obligation, across its whole notice
|
|
37
|
+
* sequence); {@link clusterIncidents} is the second (per channel+cause, across
|
|
38
|
+
* sibling obligations).
|
|
39
|
+
*
|
|
40
|
+
* ── THE HOLE THE READER MUST NOT PAPER OVER ─────────────────────────────────
|
|
41
|
+
*
|
|
42
|
+
* A reader cannot invent data that was never recorded. Two of the survey's three
|
|
43
|
+
* holes were fixed at the write site in this same change (a terminal
|
|
44
|
+
* `resolution` row, and an attempt-scoped journal line so the sequence is real).
|
|
45
|
+
* The third is reported as a hole:
|
|
46
|
+
*
|
|
47
|
+
* ORPHANED. A `needs-attention` row is NEVER pruned in production, while the
|
|
48
|
+
* obligation record it must join to is pruned at
|
|
49
|
+
* {@link OPERATOR_PLANE_RETENTION_MS}. Retention is therefore inverted, and on
|
|
50
|
+
* this machine **72 of 78 rows (92%) had no record left**. Such a row cannot
|
|
51
|
+
* say what the ask was, whether it was retried, or whether it eventually
|
|
52
|
+
* succeeded. It is marked `orphaned` and rendered as UNRESOLVABLE — never as a
|
|
53
|
+
* live incident, because a reader that shows 72 fixable-looking incidents is
|
|
54
|
+
* worse than the empty screen it replaced.
|
|
55
|
+
*
|
|
56
|
+
* PURE. No clock, no disk, no env, no network: every caller passes the rows it
|
|
57
|
+
* read and the `now` it is reasoning about. (CLAUDE.md: decisions are pure
|
|
58
|
+
* functions; I/O at the edge.)
|
|
59
|
+
*
|
|
60
|
+
* @module lib/assurance/operator-plane
|
|
61
|
+
*/
|
|
62
|
+
|
|
63
|
+
"use strict";
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* How long the plane's own records survive.
|
|
67
|
+
*
|
|
68
|
+
* THE ONE DEFINITION. `scripts/daemon/assurance.mjs` imports this as the default
|
|
69
|
+
* for `ASSURANCE_RETENTION_MS` rather than restating `3 * 24 * 60 * 60 * 1000`,
|
|
70
|
+
* because this reader's `orphaned`/`stale` verdicts are *about* that window and
|
|
71
|
+
* two literals agreeing by comment is the prose-promise-instead-of-a-check
|
|
72
|
+
* pattern this repo bans.
|
|
73
|
+
*/
|
|
74
|
+
export const OPERATOR_PLANE_RETENTION_MS = 3 * 24 * 60 * 60 * 1000;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The counter names the plane bumps. One table so the emitter
|
|
78
|
+
* (`scripts/daemon/assurance.mjs`) and the readers (this module, the CLI, the
|
|
79
|
+
* presence beat) cannot drift — a typo on either side silently zeroes a number
|
|
80
|
+
* a person reads, which is `reply-debt.mjs`'s stated reason for the same table.
|
|
81
|
+
*/
|
|
82
|
+
export const OPERATOR_PLANE_COUNTERS = Object.freeze({
|
|
83
|
+
/** A machine-status notice was routed here instead of to a person. */
|
|
84
|
+
notice: "assurance.notice_to_operator",
|
|
85
|
+
/** …and the debt it was about later reached a terminal state. */
|
|
86
|
+
resolved: "assurance.operator_notice_resolved",
|
|
87
|
+
/** A resolved needs-attention row was pruned past retention. */
|
|
88
|
+
pruned: "assurance.needs_attention_pruned",
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* The journal row kinds.
|
|
93
|
+
*
|
|
94
|
+
* `kind` is NEW and absent from every row written before this change, so a row
|
|
95
|
+
* without one is a NOTICE — the only kind that existed. Spelled as data because
|
|
96
|
+
* the reader and the writer both branch on it.
|
|
97
|
+
*/
|
|
98
|
+
/**
|
|
99
|
+
* WHICH TERMINAL OUTCOMES ACTUALLY SETTLE AN ASK — and why this is not one set.
|
|
100
|
+
*
|
|
101
|
+
* `closeObligation` stamps exactly three values: "answered" (5 call sites),
|
|
102
|
+
* "failed" (4) and "undeliverable" (3). The first pass of this reader treated
|
|
103
|
+
* ANY non-empty outcome as resolved, and that single conflation made the reader
|
|
104
|
+
* report ZERO for the population it was built for: every escalated dead ask is
|
|
105
|
+
* stamped "failed" or "undeliverable" immediately after `escalate()`, so each one
|
|
106
|
+
* read as resolved, was skipped by `summarisePlane`, and left the beat field
|
|
107
|
+
* null and the alert unreachable. Measured end to end: a killed session (exit
|
|
108
|
+
* 137, nobody told) produced `0 open, 0 unrecoverable, 1 resolved`.
|
|
109
|
+
*
|
|
110
|
+
* So "the machine stopped working on it" is NOT "the person got an answer". Only
|
|
111
|
+
* `answered` settles. `failed` and `undeliverable` are terminal for the MACHINE
|
|
112
|
+
* and still OWED to the person — they are the whole reason this plane exists.
|
|
113
|
+
*
|
|
114
|
+
* TOTAL, and the default is OWED rather than settled, for the same reason
|
|
115
|
+
* `noticePlane` defaults to refusal: a fourth outcome added later should show up
|
|
116
|
+
* as a debt somebody must look at, not vanish into "resolved". Forgetting to
|
|
117
|
+
* register it makes the reader noisier, never quieter.
|
|
118
|
+
*/
|
|
119
|
+
export const SETTLED_OUTCOMES = Object.freeze(new Set(["answered"]));
|
|
120
|
+
|
|
121
|
+
/** @param {unknown} outcome @returns {boolean} true when the ASKER got something */
|
|
122
|
+
export function outcomeSettles(outcome) {
|
|
123
|
+
const v = String(outcome == null ? "" : outcome).trim().toLowerCase();
|
|
124
|
+
return v !== "" && SETTLED_OUTCOMES.has(v);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* @param {unknown} outcome
|
|
129
|
+
* @returns {boolean} true when the machine is done and the person is still owed
|
|
130
|
+
*/
|
|
131
|
+
export function outcomeIsOwed(outcome) {
|
|
132
|
+
const v = String(outcome == null ? "" : outcome).trim().toLowerCase();
|
|
133
|
+
return v !== "" && !SETTLED_OUTCOMES.has(v);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export const JOURNAL_KIND = Object.freeze({
|
|
137
|
+
NOTICE: "notice",
|
|
138
|
+
RESOLUTION: "resolution",
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Where the session log for a dispatched session lives, relative to the agent
|
|
143
|
+
* root.
|
|
144
|
+
*
|
|
145
|
+
* WHY THIS IS HERE AND NOT INLINE AT THE TWO CALL SITES. The survey's blunt
|
|
146
|
+
* finding about the plane's cause field was "no exit code, no stderr tail, no
|
|
147
|
+
* session log path anywhere in the plane" — 94% of rows read the single label
|
|
148
|
+
* `no_outcome`, which does not distinguish a timeout from a crash from a kill
|
|
149
|
+
* from a clean exit that never spoke. The log is the only artefact that does,
|
|
150
|
+
* and the path is derivable from the session id alone (`dispatcher.mjs`, which
|
|
151
|
+
* writes `logs/daemon/sessions/<sessionId>.log`). Derived in ONE place, used by
|
|
152
|
+
* the write site (so the journal is self-sufficient after the record is pruned)
|
|
153
|
+
* and by the reader (so an old row without the field still gets a pointer), and
|
|
154
|
+
* pinned against the dispatcher's own naming by a source-reading test.
|
|
155
|
+
*
|
|
156
|
+
* @param {unknown} sessionId
|
|
157
|
+
* @returns {string|null}
|
|
158
|
+
*/
|
|
159
|
+
export function sessionLogRelPath(sessionId) {
|
|
160
|
+
const id = String(sessionId == null ? "" : sessionId).trim();
|
|
161
|
+
if (!id || /[/\\]/.test(id)) return null;
|
|
162
|
+
return `logs/daemon/sessions/${id}.log`;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Filesystem-safe form of a key — the same transform `assurance.sanitiseKey`
|
|
167
|
+
* applies to name a row's file.
|
|
168
|
+
*
|
|
169
|
+
* DUPLICATED DELIBERATELY, AND PINNED. The original lives in
|
|
170
|
+
* `scripts/daemon/assurance.mjs`, a module that resolves the agent root and
|
|
171
|
+
* makes directories at import time; a CLI command must not drag that in to
|
|
172
|
+
* normalise a string. `operator-plane.test.mjs` reads that file's source and
|
|
173
|
+
* fails if the two transforms diverge, which is the house answer to a value
|
|
174
|
+
* that must exist twice.
|
|
175
|
+
*
|
|
176
|
+
* @param {unknown} key
|
|
177
|
+
* @returns {string}
|
|
178
|
+
*/
|
|
179
|
+
export function planeFileKey(key) {
|
|
180
|
+
return String(key || "unknown").replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 180);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Does `needle` name this obligation?
|
|
185
|
+
*
|
|
186
|
+
* An operator types what they have in front of them, and what they have is
|
|
187
|
+
* whichever of three forms the surface they came from used: the raw key from a
|
|
188
|
+
* log line, the sanitised filename from an `ls`, or the tail of either (the keys
|
|
189
|
+
* are `cohort:mention:<cuid>` shaped and nobody retypes a cuid in full).
|
|
190
|
+
*
|
|
191
|
+
* @param {unknown} needle
|
|
192
|
+
* @param {unknown} key
|
|
193
|
+
* @returns {boolean}
|
|
194
|
+
*/
|
|
195
|
+
export function planeKeyMatches(needle, key) {
|
|
196
|
+
const n = String(needle == null ? "" : needle).trim().toLowerCase();
|
|
197
|
+
const k = String(key == null ? "" : key).trim().toLowerCase();
|
|
198
|
+
if (!n || !k) return false;
|
|
199
|
+
if (n === k) return true;
|
|
200
|
+
if (planeFileKey(n) === planeFileKey(k)) return true;
|
|
201
|
+
return k.endsWith(n) && n.length >= 6;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
// helpers
|
|
206
|
+
// ---------------------------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
function arr(v) { return Array.isArray(v) ? v : []; }
|
|
209
|
+
function obj(v) { return v && typeof v === "object" && !Array.isArray(v) ? v : {}; }
|
|
210
|
+
function str(v) { const s = v == null ? "" : String(v).trim(); return s || null; }
|
|
211
|
+
|
|
212
|
+
/** Epoch ms from an ISO string, an epoch number, or nothing. */
|
|
213
|
+
export function planeTime(v) {
|
|
214
|
+
if (Number.isFinite(v)) return Number(v);
|
|
215
|
+
const t = Date.parse(String(v == null ? "" : v));
|
|
216
|
+
return Number.isFinite(t) ? t : null;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function firstOf(...vals) {
|
|
220
|
+
for (const v of vals) { const s = str(v); if (s) return s; }
|
|
221
|
+
return null;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function maxOf(...vals) {
|
|
225
|
+
let best = null;
|
|
226
|
+
for (const v of vals) {
|
|
227
|
+
const n = Number(v);
|
|
228
|
+
if (Number.isFinite(n) && (best === null || n > best)) best = n;
|
|
229
|
+
}
|
|
230
|
+
return best;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function bump(map, label) {
|
|
234
|
+
const k = label || "unknown";
|
|
235
|
+
map.set(k, (map.get(k) || 0) + 1);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function tally(map) {
|
|
239
|
+
return [...map.entries()]
|
|
240
|
+
.sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1))
|
|
241
|
+
.map(([label, count]) => ({ label, count }));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
// The join
|
|
246
|
+
// ---------------------------------------------------------------------------
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Join the plane's sinks into one incident per obligation.
|
|
250
|
+
*
|
|
251
|
+
* The inputs are exactly what is on disk — no reshaping happens at the edge, so
|
|
252
|
+
* a surprising incident can be traced back to the bytes that produced it:
|
|
253
|
+
*
|
|
254
|
+
* `rows` `state/obligations/needs-attention/<file>.json`, one per key,
|
|
255
|
+
* OVERWRITTEN in place, so it holds the LAST state and no sequence.
|
|
256
|
+
* `journal` `state/obligations/notice-journal.jsonl`, append-only, the only
|
|
257
|
+
* ordered record — and the only one trimmed, at `windowMs`.
|
|
258
|
+
* `records` `state/obligations/<file>.json`, the join target that carries the
|
|
259
|
+
* eventual outcome, pruned at the same `windowMs`.
|
|
260
|
+
*
|
|
261
|
+
* A key present in ANY of the three produces an incident. That matters in both
|
|
262
|
+
* directions: a journal line whose row was never written (the second
|
|
263
|
+
* needs-attention writer, in `agent-daemon.mjs`, can fail) must not vanish, and
|
|
264
|
+
* a row whose journal lines have aged out must still be listed — as `orphaned`,
|
|
265
|
+
* which is the whole point of saying so.
|
|
266
|
+
*
|
|
267
|
+
* @param {object} o
|
|
268
|
+
* @param {object[]} [o.rows]
|
|
269
|
+
* @param {object[]} [o.journal]
|
|
270
|
+
* @param {object[]} [o.records]
|
|
271
|
+
* @param {number} [o.now]
|
|
272
|
+
* @param {number} [o.windowMs]
|
|
273
|
+
* @returns {{incidents:object[], keys:number, causes:Array<{label:string,count:number}>}}
|
|
274
|
+
*/
|
|
275
|
+
export function groupIncidents(o = {}) {
|
|
276
|
+
const now = Number.isFinite(o.now) ? Number(o.now) : 0;
|
|
277
|
+
const windowMs = Number.isFinite(o.windowMs) && o.windowMs > 0
|
|
278
|
+
? Number(o.windowMs)
|
|
279
|
+
: OPERATOR_PLANE_RETENTION_MS;
|
|
280
|
+
|
|
281
|
+
/** @type {Map<string, any>} */
|
|
282
|
+
const byKey = new Map();
|
|
283
|
+
/**
|
|
284
|
+
* @param {unknown} key
|
|
285
|
+
* @param {boolean} [create] false for the RECORD pass — see the comment on it
|
|
286
|
+
*/
|
|
287
|
+
const get = (key, create = true) => {
|
|
288
|
+
const k = String(key == null ? "" : key).trim();
|
|
289
|
+
if (!k) return null;
|
|
290
|
+
let inc = byKey.get(k);
|
|
291
|
+
if (!inc && !create) return null;
|
|
292
|
+
if (!inc) {
|
|
293
|
+
inc = {
|
|
294
|
+
key: k,
|
|
295
|
+
fileKey: planeFileKey(k),
|
|
296
|
+
service: null, channel: null, sender: null, summary: null, itemPreview: null,
|
|
297
|
+
cause: null, causes: [], exitCode: null, sessionId: null, sessionLog: null, traceId: null,
|
|
298
|
+
openedAt: null, failedAt: null, firstAt: null, lastAt: null, spanMs: 0,
|
|
299
|
+
notices: [], noticeCount: 0, attempts: 0,
|
|
300
|
+
resolved: false, owed: false, outcome: null, closedAt: null, closeNote: null,
|
|
301
|
+
hasRow: false, hasRecord: false, journalLines: 0,
|
|
302
|
+
orphaned: false, live: false, stale: false, requesterWasTold: false, configFault: null,
|
|
303
|
+
heldResultChars: null, heldResultPreview: null,
|
|
304
|
+
state: "open",
|
|
305
|
+
};
|
|
306
|
+
byKey.set(k, inc);
|
|
307
|
+
}
|
|
308
|
+
return inc;
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
const causeTally = new Map();
|
|
312
|
+
/** @type {Map<string, Map<string, number>>} */
|
|
313
|
+
const causeByKey = new Map();
|
|
314
|
+
const causesFor = (key) => {
|
|
315
|
+
if (!causeByKey.has(key)) causeByKey.set(key, new Map());
|
|
316
|
+
return causeByKey.get(key);
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// 1. THE ROW — last state, richest fields, no sequence.
|
|
320
|
+
for (const raw of arr(o.rows)) {
|
|
321
|
+
const row = obj(raw);
|
|
322
|
+
const inc = get(row.key);
|
|
323
|
+
if (!inc) continue;
|
|
324
|
+
inc.hasRow = true;
|
|
325
|
+
inc.service = firstOf(row.service, inc.service);
|
|
326
|
+
inc.channel = firstOf(row.channel, inc.channel);
|
|
327
|
+
inc.sender = firstOf(row.sender, inc.sender);
|
|
328
|
+
inc.summary = firstOf(row.summary, inc.summary);
|
|
329
|
+
inc.itemPreview = firstOf(row.itemPreview, inc.itemPreview);
|
|
330
|
+
inc.traceId = firstOf(row.traceId, inc.traceId);
|
|
331
|
+
inc.sessionId = firstOf(row.sessionId, inc.sessionId);
|
|
332
|
+
inc.sessionLog = firstOf(row.sessionLog, inc.sessionLog);
|
|
333
|
+
inc.configFault = firstOf(row.configFault, inc.configFault);
|
|
334
|
+
// A result the late-reply fallback held instead of posting
|
|
335
|
+
// (`assurance.holdResultForOwner`): what the owner reads to decide a reply.
|
|
336
|
+
inc.heldResultPreview = firstOf(row.heldResultPreview, inc.heldResultPreview);
|
|
337
|
+
if (Number.isFinite(row.heldResultChars)) inc.heldResultChars = Number(row.heldResultChars);
|
|
338
|
+
if (Number.isFinite(row.exitCode)) inc.exitCode = Number(row.exitCode);
|
|
339
|
+
inc.openedAt = maxOf(planeTime(row.openedAt), inc.openedAt);
|
|
340
|
+
inc.failedAt = maxOf(planeTime(row.failedAt), inc.failedAt);
|
|
341
|
+
inc.attempts = maxOf(row.attemptsTotal, row.attempts, inc.attempts) || 0;
|
|
342
|
+
inc.requesterWasTold = inc.requesterWasTold || row.requesterWasTold === true;
|
|
343
|
+
if (str(row.cause)) bump(causesFor(inc.key), str(row.cause));
|
|
344
|
+
// The terminal fact, when the write site stamped one. A row WITHOUT it is
|
|
345
|
+
// not "still open" — it is UNKNOWN, and the two are separated below.
|
|
346
|
+
if (str(row.outcome)) {
|
|
347
|
+
// SPLIT ON THE VALUE, not on presence. "failed"/"undeliverable" mean the
|
|
348
|
+
// machine gave up and the person is still owed; only "answered" settles.
|
|
349
|
+
// Treating all three alike is what made this reader report zero.
|
|
350
|
+
inc.outcome = str(row.outcome);
|
|
351
|
+
if (outcomeSettles(inc.outcome)) inc.resolved = true;
|
|
352
|
+
else inc.owed = true;
|
|
353
|
+
inc.closedAt = maxOf(planeTime(row.resolvedAt), planeTime(row.closedAt), inc.closedAt);
|
|
354
|
+
inc.closeNote = firstOf(row.closeNote, inc.closeNote);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// 2. THE JOURNAL — the only sequence, and the only thing that can say "31
|
|
359
|
+
// times in 6 hours". Sorted by stamp, because append order survives a
|
|
360
|
+
// concurrent writer but is not guaranteed monotonic across a prune rewrite.
|
|
361
|
+
const lines = arr(o.journal)
|
|
362
|
+
.map((l) => obj(l))
|
|
363
|
+
.filter((l) => str(l.key))
|
|
364
|
+
.sort((a, b) => (planeTime(a.at) || 0) - (planeTime(b.at) || 0));
|
|
365
|
+
for (const line of lines) {
|
|
366
|
+
const inc = get(line.key);
|
|
367
|
+
if (!inc) continue;
|
|
368
|
+
inc.journalLines += 1;
|
|
369
|
+
const at = planeTime(line.at);
|
|
370
|
+
inc.firstAt = inc.firstAt === null ? at : (at === null ? inc.firstAt : Math.min(inc.firstAt, at));
|
|
371
|
+
inc.lastAt = maxOf(at, inc.lastAt);
|
|
372
|
+
inc.service = firstOf(inc.service, line.service);
|
|
373
|
+
inc.channel = firstOf(inc.channel, line.channel);
|
|
374
|
+
inc.sender = firstOf(inc.sender, line.sender);
|
|
375
|
+
inc.traceId = firstOf(inc.traceId, line.traceId);
|
|
376
|
+
inc.sessionId = firstOf(inc.sessionId, line.sessionId);
|
|
377
|
+
inc.sessionLog = firstOf(inc.sessionLog, line.sessionLog);
|
|
378
|
+
// The journal carries the ask from this change onward, so a pruned record no
|
|
379
|
+
// longer takes the subject of the incident with it.
|
|
380
|
+
inc.summary = firstOf(inc.summary, line.summary);
|
|
381
|
+
inc.itemPreview = firstOf(inc.itemPreview, line.itemPreview);
|
|
382
|
+
if (Number.isFinite(line.exitCode)) inc.exitCode = Number(line.exitCode);
|
|
383
|
+
inc.attempts = maxOf(line.attemptsTotal, line.attempt, inc.attempts) || 0;
|
|
384
|
+
|
|
385
|
+
const kind = str(line.kind) || JOURNAL_KIND.NOTICE; // absent ⇒ written before `kind` existed
|
|
386
|
+
if (kind === JOURNAL_KIND.RESOLUTION) {
|
|
387
|
+
inc.resolved = true;
|
|
388
|
+
inc.outcome = firstOf(line.outcome, inc.outcome) || "closed";
|
|
389
|
+
inc.closedAt = maxOf(at, inc.closedAt);
|
|
390
|
+
inc.closeNote = firstOf(line.closeNote, inc.closeNote);
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
inc.noticeCount += 1;
|
|
394
|
+
inc.notices.push({
|
|
395
|
+
notice: str(line.notice) || "unknown",
|
|
396
|
+
at,
|
|
397
|
+
attempt: Number.isFinite(line.attempt) ? Number(line.attempt) : null,
|
|
398
|
+
repeat: line.repeat === true,
|
|
399
|
+
cause: str(line.cause),
|
|
400
|
+
statusLine: str(line.statusLine),
|
|
401
|
+
});
|
|
402
|
+
if (str(line.cause)) bump(causesFor(inc.key), str(line.cause));
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
// 3. THE RECORD — THE JOIN TARGET ONLY, AND IT NEVER CREATES AN INCIDENT.
|
|
406
|
+
//
|
|
407
|
+
// `state/obligations/*.json` holds EVERY debt this seat has open or recently
|
|
408
|
+
// closed, the overwhelming majority of which were answered and about which
|
|
409
|
+
// the operator plane said nothing. Letting a record mint an incident put 15
|
|
410
|
+
// happy-path asks on the first screen of the live seat, listed as "resolved"
|
|
411
|
+
// plane incidents — padding a report about failures with successes, which is
|
|
412
|
+
// how a reader starts getting skimmed.
|
|
413
|
+
//
|
|
414
|
+
// So the plane IS the set of keys the plane spoke about: a needs-attention row
|
|
415
|
+
// or a journal line. The record contributes the one thing only it has — the
|
|
416
|
+
// eventual outcome in `state`/`closeNote`, pruned at windowMs, which is the
|
|
417
|
+
// join whose absence makes 92% of the live population unrecoverable.
|
|
418
|
+
for (const raw of arr(o.records)) {
|
|
419
|
+
const rec = obj(raw);
|
|
420
|
+
const inc = get(rec.key, false);
|
|
421
|
+
if (!inc) continue;
|
|
422
|
+
inc.hasRecord = true;
|
|
423
|
+
inc.service = firstOf(inc.service, rec.service);
|
|
424
|
+
inc.channel = firstOf(inc.channel, rec.channel);
|
|
425
|
+
inc.sender = firstOf(inc.sender, rec.sender);
|
|
426
|
+
inc.summary = firstOf(inc.summary, rec.summary);
|
|
427
|
+
inc.sessionId = firstOf(inc.sessionId, rec.sessionId);
|
|
428
|
+
inc.traceId = firstOf(inc.traceId, rec.traceId);
|
|
429
|
+
inc.openedAt = maxOf(planeTime(rec.openedAt), inc.openedAt);
|
|
430
|
+
inc.attempts = maxOf(rec.attemptsTotal, rec.attempts, inc.attempts) || 0;
|
|
431
|
+
if (Number.isFinite(rec.exitCode)) inc.exitCode = Number(rec.exitCode);
|
|
432
|
+
if (str(rec.lastError)) bump(causesFor(inc.key), str(rec.lastError));
|
|
433
|
+
if (str(rec.state) && rec.state !== "open") {
|
|
434
|
+
inc.resolved = true;
|
|
435
|
+
inc.outcome = firstOf(inc.outcome, rec.state);
|
|
436
|
+
inc.closedAt = maxOf(planeTime(rec.closedAt), inc.closedAt);
|
|
437
|
+
inc.closeNote = firstOf(inc.closeNote, rec.closeNote);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
const incidents = [];
|
|
442
|
+
for (const inc of byKey.values()) {
|
|
443
|
+
const causes = tally(causeByKey.get(inc.key) || new Map());
|
|
444
|
+
inc.causes = causes;
|
|
445
|
+
inc.cause = causes.length ? causes[0].label : null;
|
|
446
|
+
for (const c of causes) bump(causeTally, c.label);
|
|
447
|
+
if (!inc.sessionLog) inc.sessionLog = sessionLogRelPath(inc.sessionId);
|
|
448
|
+
if (inc.firstAt === null) inc.firstAt = maxOf(inc.openedAt, inc.failedAt);
|
|
449
|
+
inc.lastAt = maxOf(inc.lastAt, inc.failedAt, inc.closedAt, inc.firstAt);
|
|
450
|
+
inc.spanMs = (inc.firstAt !== null && inc.lastAt !== null && inc.lastAt > inc.firstAt)
|
|
451
|
+
? inc.lastAt - inc.firstAt
|
|
452
|
+
: 0;
|
|
453
|
+
// ── THE THREE STATES, AND WHY "UNKNOWN" IS NOT "OPEN" ────────────────────
|
|
454
|
+
//
|
|
455
|
+
// ORPHANED is the 92% case on this machine: the row outlived the record it
|
|
456
|
+
// must join to, so nothing can say what the ask was or how it ended. The
|
|
457
|
+
// reader must not render that as an open incident — an operator cannot act
|
|
458
|
+
// on it, and 72 fake incidents is how a reader gets ignored in week six.
|
|
459
|
+
inc.orphaned = !inc.hasRecord && !inc.resolved && !inc.owed;
|
|
460
|
+
inc.live = (inc.lastAt !== null && now > 0) ? (now - inc.lastAt) < windowMs : !inc.orphaned;
|
|
461
|
+
inc.stale = !inc.live && !inc.resolved;
|
|
462
|
+
// OWED ranks with the unresolved, not with the resolved, and it is NOT
|
|
463
|
+
// subject to the liveness window: a dead ask does not stop being owed
|
|
464
|
+
// because it is old. That is the whole population this plane exists for.
|
|
465
|
+
inc.state = inc.resolved ? "resolved" : (inc.owed ? "owed" : (inc.orphaned ? "unresolvable" : "open"));
|
|
466
|
+
incidents.push(inc);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// Worst first: unresolved before resolved, then by how much the plane has said
|
|
470
|
+
// about it (the repeat count IS the severity), then most recent.
|
|
471
|
+
incidents.sort((a, b) => {
|
|
472
|
+
const rank = (i) => (i.state === "owed" ? 0 : i.state === "open" ? 0 : i.state === "unresolvable" ? 1 : 2);
|
|
473
|
+
return (rank(a) - rank(b))
|
|
474
|
+
|| (b.noticeCount - a.noticeCount)
|
|
475
|
+
|| ((b.lastAt || 0) - (a.lastAt || 0))
|
|
476
|
+
|| (a.key < b.key ? -1 : 1);
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
return { incidents, keys: incidents.length, causes: tally(causeTally) };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Cluster sibling incidents by (channel, cause).
|
|
484
|
+
*
|
|
485
|
+
* THE SECOND SHAPE OF ACTIONABLE, and the one a per-obligation list structurally
|
|
486
|
+
* cannot show. Measured on this machine: 78 rows across 23 channels, with 54 of
|
|
487
|
+
* 144 rerouted notices landing in ONE channel and 47 in a second. Twenty asks in
|
|
488
|
+
* one room dying of `no_outcome` is not twenty incidents to triage — it is one
|
|
489
|
+
* fact about that room, and the twenty-line list is what hides it. This is the
|
|
490
|
+
* same correction `lib/cli/skills.mjs` had to make when it counted per id and
|
|
491
|
+
* missed 117 duplicate files.
|
|
492
|
+
*
|
|
493
|
+
* @param {object[]} incidents from {@link groupIncidents}
|
|
494
|
+
* @returns {object[]} clusters, worst first
|
|
495
|
+
*/
|
|
496
|
+
export function clusterIncidents(incidents) {
|
|
497
|
+
/** @type {Map<string, any>} */
|
|
498
|
+
const out = new Map();
|
|
499
|
+
for (const raw of arr(incidents)) {
|
|
500
|
+
const inc = obj(raw);
|
|
501
|
+
const channel = str(inc.channel) || "unknown";
|
|
502
|
+
const cause = str(inc.cause) || "unknown";
|
|
503
|
+
const id = `${channel}\u0000${cause}`;
|
|
504
|
+
let c = out.get(id);
|
|
505
|
+
if (!c) {
|
|
506
|
+
c = {
|
|
507
|
+
channel, cause, service: str(inc.service),
|
|
508
|
+
keys: 0, notices: 0, open: 0, unresolvable: 0, resolved: 0,
|
|
509
|
+
firstAt: null, lastAt: null, spanMs: 0, examples: [],
|
|
510
|
+
};
|
|
511
|
+
out.set(id, c);
|
|
512
|
+
}
|
|
513
|
+
c.keys += 1;
|
|
514
|
+
c.notices += Number(inc.noticeCount) || 0;
|
|
515
|
+
if (inc.state === "open") c.open += 1;
|
|
516
|
+
else if (inc.state === "unresolvable") c.unresolvable += 1;
|
|
517
|
+
else c.resolved += 1;
|
|
518
|
+
const first = planeTime(inc.firstAt);
|
|
519
|
+
const last = planeTime(inc.lastAt);
|
|
520
|
+
if (first !== null) c.firstAt = c.firstAt === null ? first : Math.min(c.firstAt, first);
|
|
521
|
+
if (last !== null) c.lastAt = maxOf(last, c.lastAt);
|
|
522
|
+
if (c.examples.length < 3) c.examples.push(inc.key);
|
|
523
|
+
}
|
|
524
|
+
const clusters = [...out.values()];
|
|
525
|
+
for (const c of clusters) {
|
|
526
|
+
c.spanMs = (c.firstAt !== null && c.lastAt !== null && c.lastAt > c.firstAt) ? c.lastAt - c.firstAt : 0;
|
|
527
|
+
}
|
|
528
|
+
clusters.sort((a, b) => (b.open - a.open) || (b.keys - a.keys) || (b.notices - a.notices)
|
|
529
|
+
|| (a.channel < b.channel ? -1 : a.channel > b.channel ? 1 : a.cause < b.cause ? -1 : 1));
|
|
530
|
+
return clusters;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* The small object the presence beat carries, or `null` when there is nothing.
|
|
535
|
+
*
|
|
536
|
+
* NULL RATHER THAN ALL-ZERO, following `replyDebtFromCounters` exactly and for
|
|
537
|
+
* its stated reason: a field present on every beat with zeros in it trains a
|
|
538
|
+
* reader to skip the field, which is the failure this whole change exists to fix.
|
|
539
|
+
*
|
|
540
|
+
* ── WHAT IS AND IS NOT COUNTED, AND WHY IT MATTERS FOR SEVERITY ─────────────
|
|
541
|
+
*
|
|
542
|
+
* `unresolved` counts only LIVE incidents — ones the plane still holds a record
|
|
543
|
+
* for. The five-week backlog of orphaned rows is reported separately as `stale`
|
|
544
|
+
* and carries NO severity, because it is permanent by construction (rows are
|
|
545
|
+
* never pruned, records are) and a critical alert that can never clear is a
|
|
546
|
+
* critical alert everybody mutes. The backlog is a thing to sweep once, with the
|
|
547
|
+
* CLI; it is not a thing to be paged about forever.
|
|
548
|
+
*
|
|
549
|
+
* @param {object} o
|
|
550
|
+
* @param {object[]} o.incidents
|
|
551
|
+
* @param {string} [o.date] UTC YYYY-MM-DD this summary is for
|
|
552
|
+
* @returns {{unresolved:number, unrecoverable:number, repeats:number,
|
|
553
|
+
* channels:number, stale:number, worstKey:string|null,
|
|
554
|
+
* worstCount:number, worstSpanMs:number, oldestUnresolvedAt:number|null,
|
|
555
|
+
* date?:string}|null}
|
|
556
|
+
*/
|
|
557
|
+
export function summarisePlane(o = {}) {
|
|
558
|
+
const incidents = arr(o.incidents).map(obj);
|
|
559
|
+
let unresolved = 0;
|
|
560
|
+
let unrecoverable = 0;
|
|
561
|
+
let stale = 0;
|
|
562
|
+
let repeats = 0;
|
|
563
|
+
let worstKey = null;
|
|
564
|
+
let worstCount = -1;
|
|
565
|
+
let worstSpanMs = 0;
|
|
566
|
+
let oldestUnresolvedAt = null;
|
|
567
|
+
const channels = new Set();
|
|
568
|
+
|
|
569
|
+
for (const inc of incidents) {
|
|
570
|
+
if (inc.state === "resolved") continue;
|
|
571
|
+
// An OWED ask is counted whether or not it is inside the liveness window.
|
|
572
|
+
// Ageing out of the window is not the same as being dealt with, and the
|
|
573
|
+
// first version of this loop dropped every owed ask into `stale` and then
|
|
574
|
+
// reported a null summary — which is how the surface went quiet for the 78
|
|
575
|
+
// people it was built to surface.
|
|
576
|
+
if (!inc.live && inc.state !== "owed") { stale += 1; continue; }
|
|
577
|
+
unresolved += 1;
|
|
578
|
+
// A live incident whose obligation record is ALREADY gone is the one state
|
|
579
|
+
// nothing can recover: the ask text, the retry history and the eventual
|
|
580
|
+
// outcome all lived there. That is a different kind of bad from "open", and
|
|
581
|
+
// the severity split in planeAlert is the only place this reader raises its
|
|
582
|
+
// voice.
|
|
583
|
+
if (inc.orphaned) unrecoverable += 1;
|
|
584
|
+
if (str(inc.channel)) channels.add(String(inc.channel));
|
|
585
|
+
const n = Number(inc.noticeCount) || 0;
|
|
586
|
+
if (n > repeats) repeats = n;
|
|
587
|
+
if (n > worstCount || (n === worstCount && (Number(inc.spanMs) || 0) > worstSpanMs)) {
|
|
588
|
+
worstKey = str(inc.key);
|
|
589
|
+
worstCount = n;
|
|
590
|
+
worstSpanMs = Number(inc.spanMs) || 0;
|
|
591
|
+
}
|
|
592
|
+
const first = planeTime(inc.firstAt);
|
|
593
|
+
if (first !== null) oldestUnresolvedAt = oldestUnresolvedAt === null ? first : Math.min(oldestUnresolvedAt, first);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
if (unresolved === 0 && stale === 0) return null;
|
|
597
|
+
const out = {
|
|
598
|
+
unresolved, unrecoverable, repeats, stale,
|
|
599
|
+
channels: channels.size,
|
|
600
|
+
worstKey, worstCount: Math.max(0, worstCount), worstSpanMs, oldestUnresolvedAt,
|
|
601
|
+
};
|
|
602
|
+
if (str(o.date)) out.date = String(o.date);
|
|
603
|
+
return out;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* The alert verdict for a plane summary — or `null` when nothing is owed.
|
|
608
|
+
*
|
|
609
|
+
* ── THERE IS NO COUNT THRESHOLD HERE, AND THAT IS DELIBERATE ────────────────
|
|
610
|
+
*
|
|
611
|
+
* The obvious move is `warnCount: 1 / critCount: 5`, copying
|
|
612
|
+
* `lib/telemetry/alerts.mjs` `DEFAULT_THRESHOLDS.replyDebt`. It is not taken,
|
|
613
|
+
* because the number cannot be defended on observed data: the routing change is
|
|
614
|
+
* unreleased, `notice-journal.jsonl` exists on NO seat in the fleet, and
|
|
615
|
+
* `assurance.notice_to_operator` reads **0** across all 38 days of counter files
|
|
616
|
+
* on this machine. The closest proxy — 78 needs-attention rows over five weeks —
|
|
617
|
+
* says only "roughly two a day", which makes any count above one fire on
|
|
618
|
+
* essentially every day and tells an operator nothing about whether a day was
|
|
619
|
+
* abnormal. Inventing that number here would be answering a policy question with
|
|
620
|
+
* a default nobody picked, which is the mistake `lib/cli/skills.mjs` explicitly
|
|
621
|
+
* refused to repeat for skill promotion.
|
|
622
|
+
*
|
|
623
|
+
* So severity is STRUCTURAL, not numeric, and both rungs are facts rather than
|
|
624
|
+
* calibrations:
|
|
625
|
+
*
|
|
626
|
+
* warning at least one live dead ask that nothing has resolved. Every one of
|
|
627
|
+
* the 78 observed rows was a person who asked and got nothing; not
|
|
628
|
+
* one was benign, so there is no noise floor to clear.
|
|
629
|
+
* critical at least one of those has already lost its obligation record, so
|
|
630
|
+
* the ask itself is unrecoverable. 72 of 78 observed rows were in
|
|
631
|
+
* that state — it is the plane's dominant failure, and it is the one
|
|
632
|
+
* an operator can still prevent for the rows that are still live.
|
|
633
|
+
*
|
|
634
|
+
* The COUNT rides in the detail text and the drill-down command, which is what a
|
|
635
|
+
* count is for. When there is enough journal data in the fleet to calibrate a
|
|
636
|
+
* rate, the knob belongs here, and the reasoning above belongs in its comment.
|
|
637
|
+
*
|
|
638
|
+
* @param {object|null} summary from {@link summarisePlane}
|
|
639
|
+
* @returns {{id:string, severity:"warning"|"critical", kind:string, detail:string}|null}
|
|
640
|
+
*/
|
|
641
|
+
export function planeAlert(summary) {
|
|
642
|
+
const s = obj(summary);
|
|
643
|
+
const unresolved = Number(s.unresolved) || 0;
|
|
644
|
+
if (unresolved <= 0) return null;
|
|
645
|
+
const unrecoverable = Number(s.unrecoverable) || 0;
|
|
646
|
+
const parts = [];
|
|
647
|
+
if (Number(s.repeats) > 1) {
|
|
648
|
+
const hrs = Number(s.worstSpanMs) > 0
|
|
649
|
+
? ` in ${Math.max(1, Math.round(Number(s.worstSpanMs) / 3_600_000))}h`
|
|
650
|
+
: "";
|
|
651
|
+
parts.push(`worst has failed ${Number(s.repeats)} times${hrs}`);
|
|
652
|
+
}
|
|
653
|
+
if (Number(s.channels) > 1) parts.push(`${Number(s.channels)} conversations`);
|
|
654
|
+
if (Number(s.stale) > 0) parts.push(`${Number(s.stale)} older rows already unrecoverable`);
|
|
655
|
+
const tail = parts.length ? ` (${parts.join("; ")})` : "";
|
|
656
|
+
if (unrecoverable > 0) {
|
|
657
|
+
return {
|
|
658
|
+
id: "operator_plane_unrecoverable",
|
|
659
|
+
severity: "critical",
|
|
660
|
+
kind: "assurance",
|
|
661
|
+
detail: `${unrecoverable} of ${unresolved} dead ${unresolved === 1 ? "ask" : "asks"} on this seat ${unrecoverable === 1 ? "has" : "have"} already lost ${unrecoverable === 1 ? "its" : "their"} obligation record — what was asked cannot be recovered${tail}. Run \`maestro attention list\`.`,
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
return {
|
|
665
|
+
id: "operator_plane_unresolved",
|
|
666
|
+
severity: "warning",
|
|
667
|
+
kind: "assurance",
|
|
668
|
+
// SAY WHAT IS TRUE OF THE READER'S SITUATION, not of the record. An owed ask
|
|
669
|
+
// DOES carry an outcome — "failed" or "undeliverable" — so "no outcome
|
|
670
|
+
// recorded" is false for exactly the population this alert is about, and it
|
|
671
|
+
// sends the operator looking for a missing field instead of an unanswered
|
|
672
|
+
// person. The fact that matters is that somebody is still waiting.
|
|
673
|
+
detail: `${unresolved} dead ${unresolved === 1 ? "ask" : "asks"} in the operator plane — ${unresolved === 1 ? "someone asked and was never answered" : "people asked and were never answered"}${tail}. Run \`maestro attention list\`.`,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
export default {
|
|
678
|
+
OPERATOR_PLANE_RETENTION_MS,
|
|
679
|
+
OPERATOR_PLANE_COUNTERS,
|
|
680
|
+
JOURNAL_KIND,
|
|
681
|
+
sessionLogRelPath,
|
|
682
|
+
planeFileKey,
|
|
683
|
+
planeKeyMatches,
|
|
684
|
+
planeTime,
|
|
685
|
+
groupIncidents,
|
|
686
|
+
clusterIncidents,
|
|
687
|
+
summarisePlane,
|
|
688
|
+
planeAlert,
|
|
689
|
+
};
|