@smthrs/notifications 0.0.0-stage → 1.0.0-rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +127 -0
- package/LICENSE +21 -0
- package/README.md +93 -2
- package/dist/cjs/AlertPolicy.d.ts +284 -0
- package/dist/cjs/AlertPolicy.d.ts.map +1 -0
- package/dist/cjs/AlertPolicy.js +124 -0
- package/dist/cjs/AlertPolicy.js.map +7 -0
- package/dist/cjs/AlertRuntime.d.ts +90 -0
- package/dist/cjs/AlertRuntime.d.ts.map +1 -0
- package/dist/cjs/AlertRuntime.js +166 -0
- package/dist/cjs/AlertRuntime.js.map +7 -0
- package/dist/cjs/AlertSink.d.ts +101 -0
- package/dist/cjs/AlertSink.d.ts.map +1 -0
- package/dist/cjs/AlertSink.js +44 -0
- package/dist/cjs/AlertSink.js.map +7 -0
- package/dist/cjs/Alerts.d.ts +57 -0
- package/dist/cjs/Alerts.d.ts.map +1 -0
- package/dist/cjs/Alerts.js +46 -0
- package/dist/cjs/Alerts.js.map +7 -0
- package/dist/cjs/Notification.d.ts +214 -0
- package/dist/cjs/Notification.d.ts.map +1 -0
- package/dist/cjs/Notification.js +62 -0
- package/dist/cjs/Notification.js.map +7 -0
- package/dist/cjs/NotificationEvent.d.ts +214 -0
- package/dist/cjs/NotificationEvent.d.ts.map +1 -0
- package/dist/cjs/NotificationEvent.js +84 -0
- package/dist/cjs/NotificationEvent.js.map +7 -0
- package/dist/cjs/NotificationQueue.d.ts +194 -0
- package/dist/cjs/NotificationQueue.d.ts.map +1 -0
- package/dist/cjs/NotificationQueue.js +477 -0
- package/dist/cjs/NotificationQueue.js.map +7 -0
- package/dist/cjs/NotificationState.d.ts +169 -0
- package/dist/cjs/NotificationState.d.ts.map +1 -0
- package/dist/cjs/NotificationState.js +110 -0
- package/dist/cjs/NotificationState.js.map +7 -0
- package/dist/cjs/Projection.d.ts +30 -0
- package/dist/cjs/Projection.d.ts.map +1 -0
- package/dist/cjs/Projection.js +53 -0
- package/dist/cjs/Projection.js.map +7 -0
- package/dist/cjs/SteerPayload.d.ts +177 -0
- package/dist/cjs/SteerPayload.d.ts.map +1 -0
- package/dist/cjs/SteerPayload.js +61 -0
- package/dist/cjs/SteerPayload.js.map +7 -0
- package/dist/cjs/index.d.ts +41 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/foldCache.d.ts +36 -0
- package/dist/cjs/internal/foldCache.d.ts.map +1 -0
- package/dist/cjs/internal/foldCache.js +54 -0
- package/dist/cjs/internal/foldCache.js.map +7 -0
- package/dist/cjs/internal/replay.d.ts +22 -0
- package/dist/cjs/internal/replay.d.ts.map +1 -0
- package/dist/cjs/internal/replay.js +46 -0
- package/dist/cjs/internal/replay.js.map +7 -0
- package/dist/cjs/layerWebhook.d.ts +60 -0
- package/dist/cjs/layerWebhook.d.ts.map +1 -0
- package/dist/cjs/layerWebhook.js +145 -0
- package/dist/cjs/layerWebhook.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/esm/AlertPolicy.d.ts +284 -0
- package/dist/esm/AlertPolicy.d.ts.map +1 -0
- package/dist/esm/AlertPolicy.js +284 -0
- package/dist/esm/AlertPolicy.js.map +1 -0
- package/dist/esm/AlertRuntime.d.ts +90 -0
- package/dist/esm/AlertRuntime.d.ts.map +1 -0
- package/dist/esm/AlertRuntime.js +191 -0
- package/dist/esm/AlertRuntime.js.map +1 -0
- package/dist/esm/AlertSink.d.ts +101 -0
- package/dist/esm/AlertSink.d.ts.map +1 -0
- package/dist/esm/AlertSink.js +63 -0
- package/dist/esm/AlertSink.js.map +1 -0
- package/dist/esm/Alerts.d.ts +57 -0
- package/dist/esm/Alerts.d.ts.map +1 -0
- package/dist/esm/Alerts.js +57 -0
- package/dist/esm/Alerts.js.map +1 -0
- package/dist/esm/Notification.d.ts +214 -0
- package/dist/esm/Notification.d.ts.map +1 -0
- package/dist/esm/Notification.js +121 -0
- package/dist/esm/Notification.js.map +1 -0
- package/dist/esm/NotificationEvent.d.ts +214 -0
- package/dist/esm/NotificationEvent.d.ts.map +1 -0
- package/dist/esm/NotificationEvent.js +158 -0
- package/dist/esm/NotificationEvent.js.map +1 -0
- package/dist/esm/NotificationQueue.d.ts +194 -0
- package/dist/esm/NotificationQueue.d.ts.map +1 -0
- package/dist/esm/NotificationQueue.js +625 -0
- package/dist/esm/NotificationQueue.js.map +1 -0
- package/dist/esm/NotificationState.d.ts +169 -0
- package/dist/esm/NotificationState.d.ts.map +1 -0
- package/dist/esm/NotificationState.js +191 -0
- package/dist/esm/NotificationState.js.map +1 -0
- package/dist/esm/Projection.d.ts +30 -0
- package/dist/esm/Projection.d.ts.map +1 -0
- package/dist/esm/Projection.js +47 -0
- package/dist/esm/Projection.js.map +1 -0
- package/dist/esm/SteerPayload.d.ts +177 -0
- package/dist/esm/SteerPayload.d.ts.map +1 -0
- package/dist/esm/SteerPayload.js +139 -0
- package/dist/esm/SteerPayload.js.map +1 -0
- package/dist/esm/index.d.ts +41 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +41 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/foldCache.d.ts +36 -0
- package/dist/esm/internal/foldCache.d.ts.map +1 -0
- package/dist/esm/internal/foldCache.js +58 -0
- package/dist/esm/internal/foldCache.js.map +1 -0
- package/dist/esm/internal/replay.d.ts +22 -0
- package/dist/esm/internal/replay.d.ts.map +1 -0
- package/dist/esm/internal/replay.js +31 -0
- package/dist/esm/internal/replay.js.map +1 -0
- package/dist/esm/layerWebhook.d.ts +60 -0
- package/dist/esm/layerWebhook.d.ts.map +1 -0
- package/dist/esm/layerWebhook.js +186 -0
- package/dist/esm/layerWebhook.js.map +1 -0
- package/docs/README.md +146 -0
- package/docs/api.md +443 -0
- package/docs/concepts/admission-and-promotion.md +144 -0
- package/docs/concepts/alerting.md +138 -0
- package/docs/concepts/journal-records.md +132 -0
- package/docs/guides/alert-on-a-stuck-run.md +140 -0
- package/docs/guides/drain-at-a-turn-boundary.md +95 -0
- package/docs/guides/handle-a-full-queue.md +102 -0
- package/docs/guides/report-pending-notifications.md +92 -0
- package/docs/guides/send-alerts-to-a-webhook.md +155 -0
- package/docs/guides/steer-a-run.md +127 -0
- package/docs/guides/testing.md +161 -0
- package/docs/installation.md +123 -0
- package/docs/quickstart.md +159 -0
- package/docs/troubleshooting.md +254 -0
- package/package.json +182 -3
- package/src/AlertPolicy.ts +377 -0
- package/src/AlertRuntime.ts +280 -0
- package/src/AlertSink.ts +105 -0
- package/src/Alerts.ts +71 -0
- package/src/Notification.ts +171 -0
- package/src/NotificationEvent.ts +213 -0
- package/src/NotificationQueue.ts +856 -0
- package/src/NotificationState.ts +295 -0
- package/src/Projection.ts +48 -0
- package/src/SteerPayload.ts +198 -0
- package/src/index.ts +47 -0
- package/src/internal/foldCache.ts +72 -0
- package/src/internal/replay.ts +39 -0
- package/src/layerWebhook.ts +239 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Alert policy: what a condition is, when a journal leaves one open, and
|
|
3
|
+
* which open conditions have outlived their delay.
|
|
4
|
+
*
|
|
5
|
+
* Everything here is pure and a function of journal time, so the same entries
|
|
6
|
+
* decide the same alerts in any process at any instant past the delay. The
|
|
7
|
+
* transport that pages about them is `./layerWebhook.ts` and the journal-backed
|
|
8
|
+
* tick that drives them is `./AlertRuntime.ts`; `./Alerts.ts` is the module a
|
|
9
|
+
* consumer imports.
|
|
10
|
+
*
|
|
11
|
+
* @since 1.0.0
|
|
12
|
+
*/
|
|
13
|
+
import { Schema } from "effect";
|
|
14
|
+
/**
|
|
15
|
+
* The journal event type one delivered alert is recorded under.
|
|
16
|
+
*
|
|
17
|
+
* @category constants
|
|
18
|
+
* @since 0.1.0
|
|
19
|
+
*/
|
|
20
|
+
export const deliveredEventType = "flows.alerts.delivered";
|
|
21
|
+
/**
|
|
22
|
+
* The journal event type one failed delivery attempt is recorded under.
|
|
23
|
+
*
|
|
24
|
+
* One entry is written per alert per failure code, never one per tick: a
|
|
25
|
+
* webhook that stays down for an hour leaves one record, not a hundred and
|
|
26
|
+
* twenty that every later tick has to read past.
|
|
27
|
+
*
|
|
28
|
+
* @category constants
|
|
29
|
+
* @since 0.1.0
|
|
30
|
+
*/
|
|
31
|
+
export const failedEventType = "flows.alerts.failed";
|
|
32
|
+
/**
|
|
33
|
+
* How loud an alert is.
|
|
34
|
+
*
|
|
35
|
+
* @category models
|
|
36
|
+
* @since 0.1.0
|
|
37
|
+
*/
|
|
38
|
+
export const Severity = Schema.Literals(["info", "warning", "critical"]);
|
|
39
|
+
/**
|
|
40
|
+
* How a condition is recognized in the journal.
|
|
41
|
+
*
|
|
42
|
+
* `field` names a payload key, `value` the value that means the condition
|
|
43
|
+
* holds, and `eventTypes` narrows which entries are consulted at all. An
|
|
44
|
+
* `eventTypes` item ending in `.*` names a family: `control.*` consults
|
|
45
|
+
* `control.run.parked` and `control.monitor.beat` alike. An entry that carries
|
|
46
|
+
* the field with a different value CLOSES the condition, which is what makes a
|
|
47
|
+
* resume clear an approval alert without a second vocabulary for "cleared".
|
|
48
|
+
*
|
|
49
|
+
* @category models
|
|
50
|
+
* @since 0.1.0
|
|
51
|
+
*/
|
|
52
|
+
export const Detector = Schema.Struct({
|
|
53
|
+
field: Schema.NonEmptyString,
|
|
54
|
+
value: Schema.NonEmptyString,
|
|
55
|
+
eventTypes: Schema.optional(Schema.Array(Schema.NonEmptyString))
|
|
56
|
+
});
|
|
57
|
+
/**
|
|
58
|
+
* What to do about one condition, and after how long.
|
|
59
|
+
*
|
|
60
|
+
* `afterMs` is a whole, non-negative number of milliseconds. The bound is the
|
|
61
|
+
* schema's job because the delay is also arithmetic on journal time: a `NaN`
|
|
62
|
+
* delay fires on the first tick and stamps a `firedAt` that JSON writes as
|
|
63
|
+
* `null`, and a negative one fires with a `firedAt` earlier than the condition
|
|
64
|
+
* it describes.
|
|
65
|
+
*
|
|
66
|
+
* @category models
|
|
67
|
+
* @since 0.1.0
|
|
68
|
+
*/
|
|
69
|
+
export const Rule = Schema.Struct({
|
|
70
|
+
afterMs: Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0))),
|
|
71
|
+
severity: Schema.optional(Severity),
|
|
72
|
+
owner: Schema.optional(Schema.String),
|
|
73
|
+
runbook: Schema.optional(Schema.String)
|
|
74
|
+
});
|
|
75
|
+
/**
|
|
76
|
+
* The conditions a policy alerts on and the delays it alerts after.
|
|
77
|
+
*
|
|
78
|
+
* `defaults` fills in what a rule leaves out, so a policy states the delay per
|
|
79
|
+
* condition and the ownership once.
|
|
80
|
+
*
|
|
81
|
+
* @category models
|
|
82
|
+
* @since 0.1.0
|
|
83
|
+
*/
|
|
84
|
+
export const Policy = Schema.Struct({
|
|
85
|
+
defaults: Schema.optional(Schema.Struct({
|
|
86
|
+
severity: Schema.optional(Severity),
|
|
87
|
+
owner: Schema.optional(Schema.String),
|
|
88
|
+
runbook: Schema.optional(Schema.String)
|
|
89
|
+
})),
|
|
90
|
+
rules: Schema.Record(Schema.String, Rule),
|
|
91
|
+
/** Detectors for conditions this deployment names itself. */
|
|
92
|
+
detectors: Schema.optional(Schema.Record(Schema.String, Detector))
|
|
93
|
+
});
|
|
94
|
+
/**
|
|
95
|
+
* The four conditions a control plane journals out of the box.
|
|
96
|
+
*
|
|
97
|
+
* `status` rides on every `control.run.*` entry, and `health` on every
|
|
98
|
+
* `control.monitor.beat`. A deployment whose supervisor journals a park reason
|
|
99
|
+
* gets `quota-parked` from the same field the run summary reports it under.
|
|
100
|
+
*
|
|
101
|
+
* Each consults the `control.*` family only. A run's journal is shared with
|
|
102
|
+
* flow steps, agent traces, and anything else that writes to the run, and a
|
|
103
|
+
* `status` or `health` field on one of those entries would otherwise open or
|
|
104
|
+
* close a condition the control plane never reported.
|
|
105
|
+
*
|
|
106
|
+
* @category constants
|
|
107
|
+
* @since 0.1.0
|
|
108
|
+
*/
|
|
109
|
+
export const defaultDetectors = {
|
|
110
|
+
"waiting-approval": { field: "status", value: "waiting-approval", eventTypes: ["control.*"] },
|
|
111
|
+
failed: { field: "status", value: "failed", eventTypes: ["control.*"] },
|
|
112
|
+
stalled: { field: "health", value: "stalled", eventTypes: ["control.*"] },
|
|
113
|
+
"quota-parked": { field: "waitingReason", value: "quota", eventTypes: ["control.*"] }
|
|
114
|
+
};
|
|
115
|
+
/**
|
|
116
|
+
* Whether a detector's `eventTypes` names an entry's event type, exactly or
|
|
117
|
+
* through a `.*` family.
|
|
118
|
+
*/
|
|
119
|
+
const names = (eventTypes, eventType) => eventTypes.some((named) => named.endsWith(".*") ? eventType.startsWith(named.slice(0, -1)) : named === eventType);
|
|
120
|
+
/**
|
|
121
|
+
* The journal source the alerter writes one outcome of one alert under.
|
|
122
|
+
*
|
|
123
|
+
* A delivery record counts as evidence that an alert went out only under this
|
|
124
|
+
* source, so another producer's `flows.alerts.delivered` record with a
|
|
125
|
+
* matching `alertId` does not suppress a page. This scopes records; it does
|
|
126
|
+
* not authenticate the writer, because the journal does not bind a source to
|
|
127
|
+
* a writer.
|
|
128
|
+
*
|
|
129
|
+
* @param id the alert's {@link alertId}
|
|
130
|
+
* @param outcome `delivered`, `refused`, or `failed/<code>`
|
|
131
|
+
* @private
|
|
132
|
+
* @since 1.0.0
|
|
133
|
+
*/
|
|
134
|
+
export const recordSourceId = (id, outcome) => `/notifications/alerts/${id}/${outcome}`;
|
|
135
|
+
/**
|
|
136
|
+
* The key an alert coalesces on: one open condition on one run.
|
|
137
|
+
*
|
|
138
|
+
* Each component is percent-encoded before it is joined, so a run id or a
|
|
139
|
+
* condition name containing the separator cannot forge another pair's key and
|
|
140
|
+
* suppress a page that belongs to somebody else. Both components are values a
|
|
141
|
+
* deployment chooses, which is why the encoding is not optional.
|
|
142
|
+
*
|
|
143
|
+
* @param runId the run the condition is open on
|
|
144
|
+
* @param condition the condition name the policy uses
|
|
145
|
+
* @category getters
|
|
146
|
+
* @since 1.0.0
|
|
147
|
+
*/
|
|
148
|
+
export const coalescingKey = (runId, condition) => `${encodeURIComponent(runId)}:${encodeURIComponent(condition)}`;
|
|
149
|
+
/**
|
|
150
|
+
* The identity a delivery is recorded under.
|
|
151
|
+
*
|
|
152
|
+
* The opening time is part of it on purpose. A condition that clears and
|
|
153
|
+
* re-opens is a NEW alert, because the second approval wait is not the first
|
|
154
|
+
* one, and a key without the time would suppress it forever.
|
|
155
|
+
*
|
|
156
|
+
* @param alert the alert
|
|
157
|
+
* @category getters
|
|
158
|
+
* @since 0.1.0
|
|
159
|
+
*/
|
|
160
|
+
export const alertId = (alert) => `alert:${alert.coalescingKey}:${alert.since}`;
|
|
161
|
+
/**
|
|
162
|
+
* The detectors one policy decides with: the four this package ships, with
|
|
163
|
+
* the deployment's own overriding them by name.
|
|
164
|
+
*
|
|
165
|
+
* @private
|
|
166
|
+
* @since 1.0.0
|
|
167
|
+
*/
|
|
168
|
+
export const detectorsOf = (policy) => ({
|
|
169
|
+
...defaultDetectors,
|
|
170
|
+
...policy.detectors
|
|
171
|
+
});
|
|
172
|
+
/**
|
|
173
|
+
* One entry's payload when it is a record a detector can read a field off,
|
|
174
|
+
* and `undefined` for the arrays and primitives that carry no fields at all.
|
|
175
|
+
*
|
|
176
|
+
* @private
|
|
177
|
+
* @since 1.0.0
|
|
178
|
+
*/
|
|
179
|
+
export const payloadRecord = (payload) => typeof payload === "object" && payload !== null && !Array.isArray(payload)
|
|
180
|
+
? payload
|
|
181
|
+
: undefined;
|
|
182
|
+
/**
|
|
183
|
+
* Folds one entry into the open-condition map. Shared by {@link conditions}
|
|
184
|
+
* and the runtime's incremental read, so both decide a condition the same
|
|
185
|
+
* way.
|
|
186
|
+
*
|
|
187
|
+
* The alerter's own records are never evidence about a condition. They are
|
|
188
|
+
* written into the same journal they are read from, and they carry the alert's
|
|
189
|
+
* own vocabulary: a refusal reports the answering HTTP `status`, which a
|
|
190
|
+
* detector watching the run's `status` would read as the condition clearing.
|
|
191
|
+
* A page would then close the condition it paged about and the next tick would
|
|
192
|
+
* re-open it, so a webhook that answered 503 once would alert forever.
|
|
193
|
+
*
|
|
194
|
+
* @private
|
|
195
|
+
* @since 1.0.0
|
|
196
|
+
*/
|
|
197
|
+
export const observe = (policy, detectors, entry, payload, open) => {
|
|
198
|
+
if (entry.eventType === deliveredEventType || entry.eventType === failedEventType)
|
|
199
|
+
return;
|
|
200
|
+
for (const condition of Object.keys(policy.rules)) {
|
|
201
|
+
const detector = detectors[condition];
|
|
202
|
+
if (detector === undefined)
|
|
203
|
+
continue;
|
|
204
|
+
if (detector.eventTypes !== undefined && !names(detector.eventTypes, entry.eventType))
|
|
205
|
+
continue;
|
|
206
|
+
// Own properties only. `in` walks `Object.prototype`, so a detector named
|
|
207
|
+
// `toString` or `constructor` would read every record-shaped entry in the
|
|
208
|
+
// run as evidence and close the condition on all of them.
|
|
209
|
+
if (!Object.hasOwn(payload, detector.field))
|
|
210
|
+
continue;
|
|
211
|
+
if (payload[detector.field] === detector.value) {
|
|
212
|
+
if (!open.has(condition))
|
|
213
|
+
open.set(condition, entry.emittedAtMs);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
open.delete(condition);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
/**
|
|
221
|
+
* Every condition a run's journal leaves open, with the time each opened.
|
|
222
|
+
*
|
|
223
|
+
* An entry that carries a detector's field opens the condition when the value
|
|
224
|
+
* matches and closes it when it does not. Entries the detector does not name
|
|
225
|
+
* are ignored entirely, so a monitor beat cannot close an approval wait and an
|
|
226
|
+
* approval cannot close a stall.
|
|
227
|
+
*
|
|
228
|
+
* @param policy the policy whose detectors decide what counts
|
|
229
|
+
* @param runId the run the entries belong to
|
|
230
|
+
* @param entries the run's journal, oldest first
|
|
231
|
+
* @category projections
|
|
232
|
+
* @since 0.1.0
|
|
233
|
+
*/
|
|
234
|
+
export const conditions = (policy, runId, entries) => {
|
|
235
|
+
const detectors = detectorsOf(policy);
|
|
236
|
+
const open = new Map();
|
|
237
|
+
for (const entry of entries) {
|
|
238
|
+
const payload = payloadRecord(entry.payload);
|
|
239
|
+
if (payload === undefined)
|
|
240
|
+
continue;
|
|
241
|
+
observe(policy, detectors, entry, payload, open);
|
|
242
|
+
}
|
|
243
|
+
return Array.from(open, ([condition, since]) => ({ runId, condition, since }));
|
|
244
|
+
};
|
|
245
|
+
/**
|
|
246
|
+
* The alerts a policy raises for the conditions open at `now`.
|
|
247
|
+
*
|
|
248
|
+
* Pure, and a function of journal time: the same journal raises the same
|
|
249
|
+
* alerts in any process, at any instant past the delay. `now` decides WHETHER
|
|
250
|
+
* an alert is raised; it never appears in one. A condition that has been open
|
|
251
|
+
* for less than its rule's delay raises nothing, which is the whole point of
|
|
252
|
+
* the delay, because most stalls clear themselves.
|
|
253
|
+
*
|
|
254
|
+
* @param policy the policy
|
|
255
|
+
* @param open the conditions the journal left open
|
|
256
|
+
* @param now the instant to judge them at
|
|
257
|
+
* @category projections
|
|
258
|
+
* @since 0.1.0
|
|
259
|
+
*/
|
|
260
|
+
export const decide = (policy, open, now) => open.flatMap((condition) => {
|
|
261
|
+
const rule = policy.rules[condition.condition];
|
|
262
|
+
if (rule === undefined)
|
|
263
|
+
return [];
|
|
264
|
+
if (now - condition.since < rule.afterMs)
|
|
265
|
+
return [];
|
|
266
|
+
const severity = rule.severity ?? policy.defaults?.severity ?? "warning";
|
|
267
|
+
const owner = rule.owner ?? policy.defaults?.owner;
|
|
268
|
+
const runbook = rule.runbook ?? policy.defaults?.runbook;
|
|
269
|
+
return [{
|
|
270
|
+
runId: condition.runId,
|
|
271
|
+
condition: condition.condition,
|
|
272
|
+
since: condition.since,
|
|
273
|
+
// Derived from the journal, not read off the clock. The alert id is
|
|
274
|
+
// stable across ticks, and the queue refuses a reused id whose content
|
|
275
|
+
// changed, so an alert stamped with the reading time would become
|
|
276
|
+
// permanently undeliverable the moment a tick refused and time moved on.
|
|
277
|
+
firedAt: condition.since + rule.afterMs,
|
|
278
|
+
severity,
|
|
279
|
+
coalescingKey: coalescingKey(condition.runId, condition.condition),
|
|
280
|
+
...(owner === undefined ? {} : { owner }),
|
|
281
|
+
...(runbook === undefined ? {} : { runbook })
|
|
282
|
+
}];
|
|
283
|
+
});
|
|
284
|
+
//# sourceMappingURL=AlertPolicy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertPolicy.js","sourceRoot":"","sources":["../../src/AlertPolicy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAE/B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,wBAAwB,CAAA;AAE1D;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,qBAAqB,CAAA;AAEpD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAA;AAUxE;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACpC,KAAK,EAAE,MAAM,CAAC,cAAc;IAC5B,KAAK,EAAE,MAAM,CAAC,cAAc;IAC5B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;CACjE,CAAC,CAAA;AAUF;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;CACxC,CAAC,CAAA;AAUF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;QACrC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;KACxC,CAAC,CAAC;IACH,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC;IACzC,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;CACnE,CAAC,CAAA;AAUF;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAuC;IAClE,kBAAkB,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE;IAC7F,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE;IACvE,OAAO,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE;IACzE,cAAc,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE;CACtF,CAAA;AAED;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,UAAiC,EAAE,SAAiB,EAAW,EAAE,CAC9E,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAA;AAoCnH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAU,EAAE,OAAe,EAAU,EAAE,CAAC,yBAAyB,EAAE,IAAI,OAAO,EAAE,CAAA;AAE/G;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CACxE,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAA;AAEjE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,KAA6C,EAAU,EAAE,CAC/E,SAAS,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,KAAK,EAAE,CAAA;AAE/C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,MAAc,EAAsC,EAAE,CAAC,CAAC;IAClF,GAAG,gBAAgB;IACnB,GAAG,MAAM,CAAC,SAAS;CACpB,CAAC,CAAA;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAgB,EAAiD,EAAE,CAC/F,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;IACxE,CAAC,CAAC,OAA4C;IAC9C,CAAC,CAAC,SAAS,CAAA;AAEf;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,MAAc,EACd,SAA6C,EAC7C,KAAyB,EACzB,OAA0C,EAC1C,IAAyB,EACnB,EAAE;IACR,IAAI,KAAK,CAAC,SAAS,KAAK,kBAAkB,IAAI,KAAK,CAAC,SAAS,KAAK,eAAe;QAAE,OAAM;IACzF,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;QACrC,IAAI,QAAQ,KAAK,SAAS;YAAE,SAAQ;QACpC,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC;YAAE,SAAQ;QAC/F,0EAA0E;QAC1E,0EAA0E;QAC1E,0DAA0D;QAC1D,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrD,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;QAClE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,MAAc,EACd,KAAa,EACb,OAA0C,EACrB,EAAE;IACvB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAC5C,IAAI,OAAO,KAAK,SAAS;YAAE,SAAQ;QACnC,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;IAClD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;AAChF,CAAC,CAAA;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,CACpB,MAAc,EACd,IAAyB,EACzB,GAAW,EACW,EAAE,CACxB,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;IACzB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAC9C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA;IACjC,IAAI,GAAG,GAAG,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,QAAQ,IAAI,SAAS,CAAA;IACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAA;IAClD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAA;IACxD,OAAO,CAAC;YACN,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,oEAAoE;YACpE,uEAAuE;YACvE,kEAAkE;YAClE,yEAAyE;YACzE,OAAO,EAAE,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO;YACvC,QAAQ;YACR,aAAa,EAAE,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC;YAClE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;YACzC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;SAC9C,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Alert policy: what a condition is, when a journal leaves one open, and\n * which open conditions have outlived their delay.\n *\n * Everything here is pure and a function of journal time, so the same entries\n * decide the same alerts in any process at any instant past the delay. The\n * transport that pages about them is `./layerWebhook.ts` and the journal-backed\n * tick that drives them is `./AlertRuntime.ts`; `./Alerts.ts` is the module a\n * consumer imports.\n *\n * @since 1.0.0\n */\n\nimport type { JournalEvent } from \"@smthrs/journal\"\nimport { Schema } from \"effect\"\n\n/**\n * The journal event type one delivered alert is recorded under.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const deliveredEventType = \"flows.alerts.delivered\"\n\n/**\n * The journal event type one failed delivery attempt is recorded under.\n *\n * One entry is written per alert per failure code, never one per tick: a\n * webhook that stays down for an hour leaves one record, not a hundred and\n * twenty that every later tick has to read past.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const failedEventType = \"flows.alerts.failed\"\n\n/**\n * How loud an alert is.\n *\n * @category models\n * @since 0.1.0\n */\nexport const Severity = Schema.Literals([\"info\", \"warning\", \"critical\"])\n\n/**\n * How loud an alert is.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Severity = typeof Severity.Type\n\n/**\n * How a condition is recognized in the journal.\n *\n * `field` names a payload key, `value` the value that means the condition\n * holds, and `eventTypes` narrows which entries are consulted at all. An\n * `eventTypes` item ending in `.*` names a family: `control.*` consults\n * `control.run.parked` and `control.monitor.beat` alike. An entry that carries\n * the field with a different value CLOSES the condition, which is what makes a\n * resume clear an approval alert without a second vocabulary for \"cleared\".\n *\n * @category models\n * @since 0.1.0\n */\nexport const Detector = Schema.Struct({\n field: Schema.NonEmptyString,\n value: Schema.NonEmptyString,\n eventTypes: Schema.optional(Schema.Array(Schema.NonEmptyString))\n})\n\n/**\n * How a condition is recognized in the journal.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Detector = typeof Detector.Type\n\n/**\n * What to do about one condition, and after how long.\n *\n * `afterMs` is a whole, non-negative number of milliseconds. The bound is the\n * schema's job because the delay is also arithmetic on journal time: a `NaN`\n * delay fires on the first tick and stamps a `firedAt` that JSON writes as\n * `null`, and a negative one fires with a `firedAt` earlier than the condition\n * it describes.\n *\n * @category models\n * @since 0.1.0\n */\nexport const Rule = Schema.Struct({\n afterMs: Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0))),\n severity: Schema.optional(Severity),\n owner: Schema.optional(Schema.String),\n runbook: Schema.optional(Schema.String)\n})\n\n/**\n * What to do about one condition, and after how long.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Rule = typeof Rule.Type\n\n/**\n * The conditions a policy alerts on and the delays it alerts after.\n *\n * `defaults` fills in what a rule leaves out, so a policy states the delay per\n * condition and the ownership once.\n *\n * @category models\n * @since 0.1.0\n */\nexport const Policy = Schema.Struct({\n defaults: Schema.optional(Schema.Struct({\n severity: Schema.optional(Severity),\n owner: Schema.optional(Schema.String),\n runbook: Schema.optional(Schema.String)\n })),\n rules: Schema.Record(Schema.String, Rule),\n /** Detectors for conditions this deployment names itself. */\n detectors: Schema.optional(Schema.Record(Schema.String, Detector))\n})\n\n/**\n * The conditions a policy alerts on and the delays it alerts after.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Policy = typeof Policy.Type\n\n/**\n * The four conditions a control plane journals out of the box.\n *\n * `status` rides on every `control.run.*` entry, and `health` on every\n * `control.monitor.beat`. A deployment whose supervisor journals a park reason\n * gets `quota-parked` from the same field the run summary reports it under.\n *\n * Each consults the `control.*` family only. A run's journal is shared with\n * flow steps, agent traces, and anything else that writes to the run, and a\n * `status` or `health` field on one of those entries would otherwise open or\n * close a condition the control plane never reported.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const defaultDetectors: Readonly<Record<string, Detector>> = {\n \"waiting-approval\": { field: \"status\", value: \"waiting-approval\", eventTypes: [\"control.*\"] },\n failed: { field: \"status\", value: \"failed\", eventTypes: [\"control.*\"] },\n stalled: { field: \"health\", value: \"stalled\", eventTypes: [\"control.*\"] },\n \"quota-parked\": { field: \"waitingReason\", value: \"quota\", eventTypes: [\"control.*\"] }\n}\n\n/**\n * Whether a detector's `eventTypes` names an entry's event type, exactly or\n * through a `.*` family.\n */\nconst names = (eventTypes: ReadonlyArray<string>, eventType: string): boolean =>\n eventTypes.some((named) => named.endsWith(\".*\") ? eventType.startsWith(named.slice(0, -1)) : named === eventType)\n\n/**\n * A condition that is open on a run, and when it opened.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Open {\n readonly runId: string\n readonly condition: string\n /** The journal time of the entry that opened the condition. */\n readonly since: number\n}\n\n/**\n * One alert a policy decided to raise.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Alert {\n readonly runId: string\n readonly condition: string\n readonly since: number\n /**\n * The journal instant the condition outlived its delay: `since` plus the\n * rule's `afterMs`, never the wall clock of whichever tick noticed.\n */\n readonly firedAt: number\n readonly severity: Severity\n readonly coalescingKey: string\n readonly owner?: string | undefined\n readonly runbook?: string | undefined\n}\n\n/**\n * The journal source the alerter writes one outcome of one alert under.\n *\n * A delivery record counts as evidence that an alert went out only under this\n * source, so another producer's `flows.alerts.delivered` record with a\n * matching `alertId` does not suppress a page. This scopes records; it does\n * not authenticate the writer, because the journal does not bind a source to\n * a writer.\n *\n * @param id the alert's {@link alertId}\n * @param outcome `delivered`, `refused`, or `failed/<code>`\n * @private\n * @since 1.0.0\n */\nexport const recordSourceId = (id: string, outcome: string): string => `/notifications/alerts/${id}/${outcome}`\n\n/**\n * The key an alert coalesces on: one open condition on one run.\n *\n * Each component is percent-encoded before it is joined, so a run id or a\n * condition name containing the separator cannot forge another pair's key and\n * suppress a page that belongs to somebody else. Both components are values a\n * deployment chooses, which is why the encoding is not optional.\n *\n * @param runId the run the condition is open on\n * @param condition the condition name the policy uses\n * @category getters\n * @since 1.0.0\n */\nexport const coalescingKey = (runId: string, condition: string): string =>\n `${encodeURIComponent(runId)}:${encodeURIComponent(condition)}`\n\n/**\n * The identity a delivery is recorded under.\n *\n * The opening time is part of it on purpose. A condition that clears and\n * re-opens is a NEW alert, because the second approval wait is not the first\n * one, and a key without the time would suppress it forever.\n *\n * @param alert the alert\n * @category getters\n * @since 0.1.0\n */\nexport const alertId = (alert: Pick<Alert, \"coalescingKey\" | \"since\">): string =>\n `alert:${alert.coalescingKey}:${alert.since}`\n\n/**\n * The detectors one policy decides with: the four this package ships, with\n * the deployment's own overriding them by name.\n *\n * @private\n * @since 1.0.0\n */\nexport const detectorsOf = (policy: Policy): Readonly<Record<string, Detector>> => ({\n ...defaultDetectors,\n ...policy.detectors\n})\n\n/**\n * One entry's payload when it is a record a detector can read a field off,\n * and `undefined` for the arrays and primitives that carry no fields at all.\n *\n * @private\n * @since 1.0.0\n */\nexport const payloadRecord = (payload: unknown): Readonly<Record<string, unknown>> | undefined =>\n typeof payload === \"object\" && payload !== null && !Array.isArray(payload)\n ? payload as Readonly<Record<string, unknown>>\n : undefined\n\n/**\n * Folds one entry into the open-condition map. Shared by {@link conditions}\n * and the runtime's incremental read, so both decide a condition the same\n * way.\n *\n * The alerter's own records are never evidence about a condition. They are\n * written into the same journal they are read from, and they carry the alert's\n * own vocabulary: a refusal reports the answering HTTP `status`, which a\n * detector watching the run's `status` would read as the condition clearing.\n * A page would then close the condition it paged about and the next tick would\n * re-open it, so a webhook that answered 503 once would alert forever.\n *\n * @private\n * @since 1.0.0\n */\nexport const observe = (\n policy: Policy,\n detectors: Readonly<Record<string, Detector>>,\n entry: JournalEvent.Entry,\n payload: Readonly<Record<string, unknown>>,\n open: Map<string, number>\n): void => {\n if (entry.eventType === deliveredEventType || entry.eventType === failedEventType) return\n for (const condition of Object.keys(policy.rules)) {\n const detector = detectors[condition]\n if (detector === undefined) continue\n if (detector.eventTypes !== undefined && !names(detector.eventTypes, entry.eventType)) continue\n // Own properties only. `in` walks `Object.prototype`, so a detector named\n // `toString` or `constructor` would read every record-shaped entry in the\n // run as evidence and close the condition on all of them.\n if (!Object.hasOwn(payload, detector.field)) continue\n if (payload[detector.field] === detector.value) {\n if (!open.has(condition)) open.set(condition, entry.emittedAtMs)\n } else {\n open.delete(condition)\n }\n }\n}\n\n/**\n * Every condition a run's journal leaves open, with the time each opened.\n *\n * An entry that carries a detector's field opens the condition when the value\n * matches and closes it when it does not. Entries the detector does not name\n * are ignored entirely, so a monitor beat cannot close an approval wait and an\n * approval cannot close a stall.\n *\n * @param policy the policy whose detectors decide what counts\n * @param runId the run the entries belong to\n * @param entries the run's journal, oldest first\n * @category projections\n * @since 0.1.0\n */\nexport const conditions = (\n policy: Policy,\n runId: string,\n entries: ReadonlyArray<JournalEvent.Entry>\n): ReadonlyArray<Open> => {\n const detectors = detectorsOf(policy)\n const open = new Map<string, number>()\n for (const entry of entries) {\n const payload = payloadRecord(entry.payload)\n if (payload === undefined) continue\n observe(policy, detectors, entry, payload, open)\n }\n return Array.from(open, ([condition, since]) => ({ runId, condition, since }))\n}\n\n/**\n * The alerts a policy raises for the conditions open at `now`.\n *\n * Pure, and a function of journal time: the same journal raises the same\n * alerts in any process, at any instant past the delay. `now` decides WHETHER\n * an alert is raised; it never appears in one. A condition that has been open\n * for less than its rule's delay raises nothing, which is the whole point of\n * the delay, because most stalls clear themselves.\n *\n * @param policy the policy\n * @param open the conditions the journal left open\n * @param now the instant to judge them at\n * @category projections\n * @since 0.1.0\n */\nexport const decide = (\n policy: Policy,\n open: ReadonlyArray<Open>,\n now: number\n): ReadonlyArray<Alert> =>\n open.flatMap((condition) => {\n const rule = policy.rules[condition.condition]\n if (rule === undefined) return []\n if (now - condition.since < rule.afterMs) return []\n const severity = rule.severity ?? policy.defaults?.severity ?? \"warning\"\n const owner = rule.owner ?? policy.defaults?.owner\n const runbook = rule.runbook ?? policy.defaults?.runbook\n return [{\n runId: condition.runId,\n condition: condition.condition,\n since: condition.since,\n // Derived from the journal, not read off the clock. The alert id is\n // stable across ticks, and the queue refuses a reused id whose content\n // changed, so an alert stamped with the reading time would become\n // permanently undeliverable the moment a tick refused and time moved on.\n firedAt: condition.since + rule.afterMs,\n severity,\n coalescingKey: coalescingKey(condition.runId, condition.condition),\n ...(owner === undefined ? {} : { owner }),\n ...(runbook === undefined ? {} : { runbook })\n }]\n })\n"]}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The journal-backed alert runtime: one tick per run.
|
|
3
|
+
*
|
|
4
|
+
* A tick folds the run's journal into the conditions it leaves open, decides
|
|
5
|
+
* which have outlived their delay, admits each one as a coalesced system
|
|
6
|
+
* event, pages the sink, and journals what the sink did with it. The fold is
|
|
7
|
+
* retained per run, so a tick costs the entries committed since the previous
|
|
8
|
+
* one rather than the run's whole journal.
|
|
9
|
+
*
|
|
10
|
+
* The rules it enforces are `./AlertPolicy.ts` and the channel it pages over
|
|
11
|
+
* is `./AlertSink.ts`; neither knows this module exists.
|
|
12
|
+
*
|
|
13
|
+
* @since 1.0.0
|
|
14
|
+
*/
|
|
15
|
+
import { Journal } from "@smthrs/journal";
|
|
16
|
+
import { Context, Effect, Layer } from "effect";
|
|
17
|
+
import { type Alert, Policy } from "./AlertPolicy.ts";
|
|
18
|
+
import { Sink } from "./AlertSink.ts";
|
|
19
|
+
import { type NotificationError, NotificationQueue } from "./NotificationQueue.ts";
|
|
20
|
+
/**
|
|
21
|
+
* What one tick decided.
|
|
22
|
+
*
|
|
23
|
+
* @category models
|
|
24
|
+
* @since 0.1.0
|
|
25
|
+
*/
|
|
26
|
+
export interface Tick {
|
|
27
|
+
/** Alerts raised and delivered on this tick. */
|
|
28
|
+
readonly delivered: ReadonlyArray<Alert>;
|
|
29
|
+
/** Alerts the sink refused on this tick. They are retried on the next one. */
|
|
30
|
+
readonly failed: ReadonlyArray<Alert>;
|
|
31
|
+
/**
|
|
32
|
+
* Alerts the notification queue refused because the run is at capacity. The
|
|
33
|
+
* sink was not called for them, and they are retried on the next tick.
|
|
34
|
+
*/
|
|
35
|
+
readonly refused: ReadonlyArray<Alert>;
|
|
36
|
+
/** Alerts that had already been delivered, and were not delivered again. */
|
|
37
|
+
readonly suppressed: ReadonlyArray<Alert>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The alert runtime: one tick per run.
|
|
41
|
+
*
|
|
42
|
+
* @category services
|
|
43
|
+
* @since 0.1.0
|
|
44
|
+
*/
|
|
45
|
+
export interface RuntimeService {
|
|
46
|
+
/**
|
|
47
|
+
* Reads one run's journal and pages about whatever has waited too long.
|
|
48
|
+
*
|
|
49
|
+
* The two failures stay apart because they mean different things to an
|
|
50
|
+
* operator. A `JournalError` is the alerter's own record channel failing: it
|
|
51
|
+
* read no entries, or it could not write the delivery record. A
|
|
52
|
+
* `NotificationError` is the notification queue rejecting the alert outright,
|
|
53
|
+
* which is a producer or storage fault rather than a full queue. Neither is
|
|
54
|
+
* a sink failure: a refused page is journaled as `flows.alerts.failed` and
|
|
55
|
+
* retried on the next tick, and it comes back in {@link Tick.failed}. A queue
|
|
56
|
+
* at capacity is not a failure either; it comes back in {@link Tick.refused}.
|
|
57
|
+
*/
|
|
58
|
+
readonly tick: (runId: string) => Effect.Effect<Tick, Journal.JournalError | NotificationError>;
|
|
59
|
+
}
|
|
60
|
+
declare const AlertRuntime_base: Context.ServiceClass<AlertRuntime, "/notifications/AlertRuntime", RuntimeService>;
|
|
61
|
+
/**
|
|
62
|
+
* The {@link RuntimeService} tag.
|
|
63
|
+
*
|
|
64
|
+
* @category services
|
|
65
|
+
* @since 0.1.0
|
|
66
|
+
*/
|
|
67
|
+
export declare class AlertRuntime extends AlertRuntime_base {
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The alert runtime over one policy.
|
|
71
|
+
*
|
|
72
|
+
* A tick reads the run's journal, derives the open conditions, decides which
|
|
73
|
+
* have outlived their delay, and for each one that has not already been
|
|
74
|
+
* delivered: admits a coalesced system event, sends it to the sink, and
|
|
75
|
+
* journals the delivery. The admission comes first because it is idempotent
|
|
76
|
+
* and the send is not: a crash between them costs a duplicate admission, which
|
|
77
|
+
* the queue drops, while a crash between an accepted send and the delivery
|
|
78
|
+
* record costs a duplicate PAGE, which is why {@link SinkService.deliver} is
|
|
79
|
+
* required to dedupe on {@link alertId}.
|
|
80
|
+
*
|
|
81
|
+
* The policy is decoded when the layer is built, so a rule with an impossible
|
|
82
|
+
* delay fails the composition by name instead of mis-paging at 3am.
|
|
83
|
+
*
|
|
84
|
+
* @param policy the rules to enforce
|
|
85
|
+
* @category layers
|
|
86
|
+
* @since 0.1.0
|
|
87
|
+
*/
|
|
88
|
+
export declare const layer: (policy: Policy) => Layer.Layer<AlertRuntime, never, Journal.Journal | NotificationQueue | Sink>;
|
|
89
|
+
export {};
|
|
90
|
+
//# sourceMappingURL=AlertRuntime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertRuntime.d.ts","sourceRoot":"","sources":["../../src/AlertRuntime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,OAAO,EAAgB,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,EAAW,KAAK,EAAkB,MAAM,QAAQ,CAAA;AAC/E,OAAO,EACL,KAAK,KAAK,EAQV,MAAM,EAEP,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AAGrC,OAAO,EAAE,KAAK,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAElF;;;;;GAKG;AACH,MAAM,WAAW,IAAI;IACnB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACxC,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACrC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACtC,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;CAC1C;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,CAAA;CAChG;;AAED;;;;;GAKG;AACH,qBAAa,YAAa,SAAQ,iBAEjC;CAAG;AA8CJ;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,KAAK,GAChB,QAAQ,MAAM,KACb,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAgI3E,CAAA"}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The journal-backed alert runtime: one tick per run.
|
|
3
|
+
*
|
|
4
|
+
* A tick folds the run's journal into the conditions it leaves open, decides
|
|
5
|
+
* which have outlived their delay, admits each one as a coalesced system
|
|
6
|
+
* event, pages the sink, and journals what the sink did with it. The fold is
|
|
7
|
+
* retained per run, so a tick costs the entries committed since the previous
|
|
8
|
+
* one rather than the run's whole journal.
|
|
9
|
+
*
|
|
10
|
+
* The rules it enforces are `./AlertPolicy.ts` and the channel it pages over
|
|
11
|
+
* is `./AlertSink.ts`; neither knows this module exists.
|
|
12
|
+
*
|
|
13
|
+
* @since 1.0.0
|
|
14
|
+
*/
|
|
15
|
+
import { Journal, JournalEvent } from "@smthrs/journal";
|
|
16
|
+
import { Clock, Context, Effect, HashSet, Layer, Result, Schema } from "effect";
|
|
17
|
+
import { alertId, decide, deliveredEventType, detectorsOf, failedEventType, observe, payloadRecord, Policy, recordSourceId } from "./AlertPolicy.js";
|
|
18
|
+
import { Sink } from "./AlertSink.js";
|
|
19
|
+
import * as FoldCache from "./internal/foldCache.js";
|
|
20
|
+
import { NotificationQueue } from "./NotificationQueue.js";
|
|
21
|
+
/**
|
|
22
|
+
* The {@link RuntimeService} tag.
|
|
23
|
+
*
|
|
24
|
+
* @category services
|
|
25
|
+
* @since 0.1.0
|
|
26
|
+
*/
|
|
27
|
+
export class AlertRuntime extends Context.Service()("/notifications/AlertRuntime") {
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* How many runs one alert runtime keeps folded at a time. The retention policy
|
|
31
|
+
* is scan-resistant, so a sweep of more runs than this reads journal tails for
|
|
32
|
+
* all but a fixed handful of them.
|
|
33
|
+
*/
|
|
34
|
+
const maximumWatchedRuns = 64;
|
|
35
|
+
const alertNotification = (alert) => ({
|
|
36
|
+
_tag: "system-event",
|
|
37
|
+
id: alertId(alert),
|
|
38
|
+
targetLineageId: alert.runId,
|
|
39
|
+
delivery: "queue",
|
|
40
|
+
coalescingKey: alert.coalescingKey,
|
|
41
|
+
provenance: {
|
|
42
|
+
sourceRunId: alert.runId,
|
|
43
|
+
sourceLineageId: alert.runId,
|
|
44
|
+
sourceTurn: 0,
|
|
45
|
+
sourceActor: "alerts"
|
|
46
|
+
},
|
|
47
|
+
payload: {
|
|
48
|
+
condition: alert.condition,
|
|
49
|
+
severity: alert.severity,
|
|
50
|
+
since: alert.since,
|
|
51
|
+
firedAt: alert.firedAt,
|
|
52
|
+
...(alert.owner === undefined ? {} : { owner: alert.owner }),
|
|
53
|
+
...(alert.runbook === undefined ? {} : { runbook: alert.runbook })
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
/**
|
|
57
|
+
* The alert runtime over one policy.
|
|
58
|
+
*
|
|
59
|
+
* A tick reads the run's journal, derives the open conditions, decides which
|
|
60
|
+
* have outlived their delay, and for each one that has not already been
|
|
61
|
+
* delivered: admits a coalesced system event, sends it to the sink, and
|
|
62
|
+
* journals the delivery. The admission comes first because it is idempotent
|
|
63
|
+
* and the send is not: a crash between them costs a duplicate admission, which
|
|
64
|
+
* the queue drops, while a crash between an accepted send and the delivery
|
|
65
|
+
* record costs a duplicate PAGE, which is why {@link SinkService.deliver} is
|
|
66
|
+
* required to dedupe on {@link alertId}.
|
|
67
|
+
*
|
|
68
|
+
* The policy is decoded when the layer is built, so a rule with an impossible
|
|
69
|
+
* delay fails the composition by name instead of mis-paging at 3am.
|
|
70
|
+
*
|
|
71
|
+
* @param policy the rules to enforce
|
|
72
|
+
* @category layers
|
|
73
|
+
* @since 0.1.0
|
|
74
|
+
*/
|
|
75
|
+
export const layer = (policy) => Layer.effect(AlertRuntime, Effect.gen(function* () {
|
|
76
|
+
const checked = yield* Effect.orDie(Schema.decodeUnknownEffect(Policy)(policy));
|
|
77
|
+
const detectors = detectorsOf(checked);
|
|
78
|
+
const journal = yield* Journal.Journal;
|
|
79
|
+
const queue = yield* NotificationQueue;
|
|
80
|
+
const sink = yield* Sink;
|
|
81
|
+
// Folded history per run, so a tick costs the entries committed since
|
|
82
|
+
// the previous one rather than the run's whole journal.
|
|
83
|
+
const watched = FoldCache.make(maximumWatchedRuns);
|
|
84
|
+
const observed = (runId) => Effect.gen(function* () {
|
|
85
|
+
const base = watched.get(runId) ??
|
|
86
|
+
{ open: new Map(), delivered: HashSet.empty(), cursor: undefined };
|
|
87
|
+
const fresh = [];
|
|
88
|
+
let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor);
|
|
89
|
+
while (true) {
|
|
90
|
+
const page = yield* journal.entries({ runId, ...(after === undefined ? {} : { after }), limit: 512 });
|
|
91
|
+
fresh.push(...page.entries);
|
|
92
|
+
if (!page.hasMore || page.entries.length === 0)
|
|
93
|
+
break;
|
|
94
|
+
after = page.entries.at(-1).seq;
|
|
95
|
+
}
|
|
96
|
+
if (fresh.length === 0)
|
|
97
|
+
return base;
|
|
98
|
+
// `open` is bounded by the policy's rules, so it is copied; the
|
|
99
|
+
// delivery history is not, so it grows by one persistent node.
|
|
100
|
+
const open = new Map(base.open);
|
|
101
|
+
let delivered = base.delivered;
|
|
102
|
+
let cursor = base.cursor;
|
|
103
|
+
for (const entry of fresh) {
|
|
104
|
+
cursor = entry.seq;
|
|
105
|
+
const payload = payloadRecord(entry.payload);
|
|
106
|
+
if (payload === undefined)
|
|
107
|
+
continue;
|
|
108
|
+
if (entry.eventType === deliveredEventType) {
|
|
109
|
+
const id = payload["alertId"];
|
|
110
|
+
// Only a record under the source the alerter writes counts, so
|
|
111
|
+
// another producer's delivered record cannot suppress a page.
|
|
112
|
+
// The journal does not bind a source to a writer.
|
|
113
|
+
if (typeof id === "string" && entry.sourceId === recordSourceId(id, "delivered")) {
|
|
114
|
+
delivered = HashSet.add(delivered, id);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
observe(checked, detectors, entry, payload, open);
|
|
118
|
+
}
|
|
119
|
+
const next = { open, delivered, cursor };
|
|
120
|
+
// Published at commit, exactly as `NotificationQueue.load` publishes
|
|
121
|
+
// its fold. A tick inside an enclosing transaction reads that
|
|
122
|
+
// transaction's uncommitted rows, and caching them would outlive the
|
|
123
|
+
// rollback that erased them: the run would page about a condition
|
|
124
|
+
// durable history never held.
|
|
125
|
+
yield* journal.whenCommitted(Effect.sync(() => watched.put(runId, next)));
|
|
126
|
+
return next;
|
|
127
|
+
});
|
|
128
|
+
const record = (eventType, alert, source, outcome) => journal.emitDurableUnfenced(new JournalEvent.Input({
|
|
129
|
+
runId: JournalEvent.RunId.make(alert.runId),
|
|
130
|
+
sourceId: JournalEvent.SourceId.make(recordSourceId(alertId(alert), source)),
|
|
131
|
+
sourceSeq: JournalEvent.SourceSeq.make(0),
|
|
132
|
+
// One record per alert per outcome. Without an explicit identity
|
|
133
|
+
// the journal allocates a new sequence on every attempt, and a
|
|
134
|
+
// webhook that stays down appends a row per tick forever.
|
|
135
|
+
dedupe: "identity",
|
|
136
|
+
eventType,
|
|
137
|
+
payload: {
|
|
138
|
+
runId: alert.runId,
|
|
139
|
+
condition: alert.condition,
|
|
140
|
+
since: alert.since,
|
|
141
|
+
severity: alert.severity,
|
|
142
|
+
alertId: alertId(alert),
|
|
143
|
+
...(outcome.code === undefined ? {} : { code: outcome.code }),
|
|
144
|
+
...(outcome.status === undefined ? {} : { status: outcome.status })
|
|
145
|
+
}
|
|
146
|
+
})).pipe(Effect.asVoid);
|
|
147
|
+
return AlertRuntime.of({
|
|
148
|
+
tick: Effect.fn("AlertRuntime.tick")(function* (runId) {
|
|
149
|
+
const journalRunId = JournalEvent.RunId.make(runId);
|
|
150
|
+
const history = yield* observed(journalRunId);
|
|
151
|
+
const now = yield* Clock.currentTimeMillis;
|
|
152
|
+
const open = Array.from(history.open, ([condition, since]) => ({ runId, condition, since }));
|
|
153
|
+
const raised = decide(checked, open, now);
|
|
154
|
+
const delivered = [];
|
|
155
|
+
const failed = [];
|
|
156
|
+
const refused = [];
|
|
157
|
+
const suppressed = [];
|
|
158
|
+
for (const alert of raised) {
|
|
159
|
+
if (HashSet.has(history.delivered, alertId(alert))) {
|
|
160
|
+
suppressed.push(alert);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
// Raised unchanged: a queue that rejects the admission outright is
|
|
164
|
+
// not the journal failing, and calling it `sink_failed` would send
|
|
165
|
+
// an operator looking at the webhook.
|
|
166
|
+
const receipt = yield* queue.admit(runId, alertNotification(alert));
|
|
167
|
+
if (receipt.decision === "rejected-full") {
|
|
168
|
+
// The alert has nowhere durable to sit, so paging about it would
|
|
169
|
+
// tell an operator about something the run will never read.
|
|
170
|
+
yield* record(failedEventType, alert, "refused", { code: "notification_full" });
|
|
171
|
+
refused.push(alert);
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const outcome = yield* Effect.result(sink.deliver(alert));
|
|
175
|
+
if (Result.isFailure(outcome)) {
|
|
176
|
+
yield* record(failedEventType, alert, `failed/${outcome.failure.code}`, {
|
|
177
|
+
code: outcome.failure.code,
|
|
178
|
+
status: outcome.failure.status
|
|
179
|
+
});
|
|
180
|
+
failed.push(alert);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
yield* record(deliveredEventType, alert, "delivered", {});
|
|
184
|
+
delivered.push(alert);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return { delivered, failed, refused, suppressed };
|
|
188
|
+
})
|
|
189
|
+
});
|
|
190
|
+
}));
|
|
191
|
+
//# sourceMappingURL=AlertRuntime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertRuntime.js","sourceRoot":"","sources":["../../src/AlertRuntime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/E,OAAO,EAEL,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,OAAO,EACP,aAAa,EACb,MAAM,EACN,cAAc,EACf,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AACrC,OAAO,KAAK,SAAS,MAAM,yBAAyB,CAAA;AAEpD,OAAO,EAA0B,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AA4ClF;;;;;GAKG;AACH,MAAM,OAAO,YAAa,SAAQ,OAAO,CAAC,OAAO,EAAgC,CAC/E,6BAA6B,CAC9B;CAAG;AAiBJ;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,MAAM,iBAAiB,GAAG,CAAC,KAAY,EAAkC,EAAE,CAAC,CAAC;IAC3E,IAAI,EAAE,cAAc;IACpB,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC;IAClB,eAAe,EAAE,KAAK,CAAC,KAAK;IAC5B,QAAQ,EAAE,OAAO;IACjB,aAAa,EAAE,KAAK,CAAC,aAAa;IAClC,UAAU,EAAE;QACV,WAAW,EAAE,KAAK,CAAC,KAAK;QACxB,eAAe,EAAE,KAAK,CAAC,KAAK;QAC5B,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,QAAQ;KACtB;IACD,OAAO,EAAE;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;KACnE;CACF,CAAC,CAAA;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CACnB,MAAc,EACgE,EAAE,CAChF,KAAK,CAAC,MAAM,CACV,YAAY,EACZ,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAA;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAA;IACxB,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAU,kBAAkB,CAAC,CAAA;IAE3D,MAAM,QAAQ,GAAG,CAAC,KAAyB,EAAgD,EAAE,CAC3F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,IAAI,GAAY,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YACtC,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;QACpE,MAAM,KAAK,GAA8B,EAAE,CAAA;QAC3C,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACtF,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YACrG,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;YAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAK;YACrD,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,CAAA;QAClC,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAEnC,gEAAgE;QAChE,+DAA+D;QAC/D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,CAAA;QAC9B,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QACxB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,KAAK,CAAC,GAAG,CAAA;YAClB,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAC5C,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAQ;YACnC,IAAI,KAAK,CAAC,SAAS,KAAK,kBAAkB,EAAE,CAAC;gBAC3C,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAA;gBAC7B,+DAA+D;gBAC/D,8DAA8D;gBAC9D,kDAAkD;gBAClD,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,KAAK,cAAc,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC;oBACjF,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;gBACxC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC;QACD,MAAM,IAAI,GAAY,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAA;QACjD,qEAAqE;QACrE,8DAA8D;QAC9D,qEAAqE;QACrE,kEAAkE;QAClE,8BAA8B;QAC9B,KAAK,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QACzE,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;IAEJ,MAAM,MAAM,GAAG,CACb,SAAiB,EACjB,KAAY,EACZ,MAAc,EACd,OAAqF,EAC1C,EAAE,CAC7C,OAAO,CAAC,mBAAmB,CACzB,IAAI,YAAY,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3C,QAAQ,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;QAC5E,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,iEAAiE;QACjE,+DAA+D;QAC/D,0DAA0D;QAC1D,MAAM,EAAE,UAAU;QAClB,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;YACvB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SACpE;KACF,CAAC,CACH,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAEvB,OAAO,YAAY,CAAC,EAAE,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,EAAC,KAAa;YAC1D,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;YAC7C,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;YAC5F,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAA;YACzC,MAAM,SAAS,GAAiB,EAAE,CAAA;YAClC,MAAM,MAAM,GAAiB,EAAE,CAAA;YAC/B,MAAM,OAAO,GAAiB,EAAE,CAAA;YAChC,MAAM,UAAU,GAAiB,EAAE,CAAA;YACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACnD,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACtB,SAAQ;gBACV,CAAC;gBACD,mEAAmE;gBACnE,mEAAmE;gBACnE,sCAAsC;gBACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAA;gBACnE,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;oBACzC,iEAAiE;oBACjE,4DAA4D;oBAC5D,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;oBAC/E,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACnB,SAAQ;gBACV,CAAC;gBACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;gBACzD,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,UAAU,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE;wBACtE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;wBAC1B,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;qBAC/B,CAAC,CAAA;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACpB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,CAAC,MAAM,CAAC,kBAAkB,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;oBACzD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;QACnD,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAA","sourcesContent":["/**\n * The journal-backed alert runtime: one tick per run.\n *\n * A tick folds the run's journal into the conditions it leaves open, decides\n * which have outlived their delay, admits each one as a coalesced system\n * event, pages the sink, and journals what the sink did with it. The fold is\n * retained per run, so a tick costs the entries committed since the previous\n * one rather than the run's whole journal.\n *\n * The rules it enforces are `./AlertPolicy.ts` and the channel it pages over\n * is `./AlertSink.ts`; neither knows this module exists.\n *\n * @since 1.0.0\n */\n\nimport { Journal, JournalEvent } from \"@smthrs/journal\"\nimport { Clock, Context, Effect, HashSet, Layer, Result, Schema } from \"effect\"\nimport {\n type Alert,\n alertId,\n decide,\n deliveredEventType,\n detectorsOf,\n failedEventType,\n observe,\n payloadRecord,\n Policy,\n recordSourceId\n} from \"./AlertPolicy.ts\"\nimport { Sink } from \"./AlertSink.ts\"\nimport * as FoldCache from \"./internal/foldCache.ts\"\nimport type * as NotificationModel from \"./Notification.ts\"\nimport { type NotificationError, NotificationQueue } from \"./NotificationQueue.ts\"\n\n/**\n * What one tick decided.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tick {\n /** Alerts raised and delivered on this tick. */\n readonly delivered: ReadonlyArray<Alert>\n /** Alerts the sink refused on this tick. They are retried on the next one. */\n readonly failed: ReadonlyArray<Alert>\n /**\n * Alerts the notification queue refused because the run is at capacity. The\n * sink was not called for them, and they are retried on the next tick.\n */\n readonly refused: ReadonlyArray<Alert>\n /** Alerts that had already been delivered, and were not delivered again. */\n readonly suppressed: ReadonlyArray<Alert>\n}\n\n/**\n * The alert runtime: one tick per run.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface RuntimeService {\n /**\n * Reads one run's journal and pages about whatever has waited too long.\n *\n * The two failures stay apart because they mean different things to an\n * operator. A `JournalError` is the alerter's own record channel failing: it\n * read no entries, or it could not write the delivery record. A\n * `NotificationError` is the notification queue rejecting the alert outright,\n * which is a producer or storage fault rather than a full queue. Neither is\n * a sink failure: a refused page is journaled as `flows.alerts.failed` and\n * retried on the next tick, and it comes back in {@link Tick.failed}. A queue\n * at capacity is not a failure either; it comes back in {@link Tick.refused}.\n */\n readonly tick: (runId: string) => Effect.Effect<Tick, Journal.JournalError | NotificationError>\n}\n\n/**\n * The {@link RuntimeService} tag.\n *\n * @category services\n * @since 0.1.0\n */\nexport class AlertRuntime extends Context.Service<AlertRuntime, RuntimeService>()(\n \"/notifications/AlertRuntime\"\n) {}\n\n/**\n * One run's folded alert history: the conditions still open, the alerts\n * already paged, and the sequence the fold stopped at.\n */\ninterface Watched {\n readonly open: ReadonlyMap<string, number>\n /**\n * The alerts already paged, as a persistent set: a tick adds a node and\n * leaves the fold the previous tick published untouched, where copying the\n * set would cost the run's whole delivery history on every tick.\n */\n readonly delivered: HashSet.HashSet<string>\n readonly cursor: number | undefined\n}\n\n/**\n * How many runs one alert runtime keeps folded at a time. The retention policy\n * is scan-resistant, so a sweep of more runs than this reads journal tails for\n * all but a fixed handful of them.\n */\nconst maximumWatchedRuns = 64\n\nconst alertNotification = (alert: Alert): NotificationModel.Notification => ({\n _tag: \"system-event\",\n id: alertId(alert),\n targetLineageId: alert.runId,\n delivery: \"queue\",\n coalescingKey: alert.coalescingKey,\n provenance: {\n sourceRunId: alert.runId,\n sourceLineageId: alert.runId,\n sourceTurn: 0,\n sourceActor: \"alerts\"\n },\n payload: {\n condition: alert.condition,\n severity: alert.severity,\n since: alert.since,\n firedAt: alert.firedAt,\n ...(alert.owner === undefined ? {} : { owner: alert.owner }),\n ...(alert.runbook === undefined ? {} : { runbook: alert.runbook })\n }\n})\n\n/**\n * The alert runtime over one policy.\n *\n * A tick reads the run's journal, derives the open conditions, decides which\n * have outlived their delay, and for each one that has not already been\n * delivered: admits a coalesced system event, sends it to the sink, and\n * journals the delivery. The admission comes first because it is idempotent\n * and the send is not: a crash between them costs a duplicate admission, which\n * the queue drops, while a crash between an accepted send and the delivery\n * record costs a duplicate PAGE, which is why {@link SinkService.deliver} is\n * required to dedupe on {@link alertId}.\n *\n * The policy is decoded when the layer is built, so a rule with an impossible\n * delay fails the composition by name instead of mis-paging at 3am.\n *\n * @param policy the rules to enforce\n * @category layers\n * @since 0.1.0\n */\nexport const layer = (\n policy: Policy\n): Layer.Layer<AlertRuntime, never, Journal.Journal | NotificationQueue | Sink> =>\n Layer.effect(\n AlertRuntime,\n Effect.gen(function*() {\n const checked = yield* Effect.orDie(Schema.decodeUnknownEffect(Policy)(policy))\n const detectors = detectorsOf(checked)\n const journal = yield* Journal.Journal\n const queue = yield* NotificationQueue\n const sink = yield* Sink\n // Folded history per run, so a tick costs the entries committed since\n // the previous one rather than the run's whole journal.\n const watched = FoldCache.make<Watched>(maximumWatchedRuns)\n\n const observed = (runId: JournalEvent.RunId): Effect.Effect<Watched, Journal.JournalError> =>\n Effect.gen(function*() {\n const base: Watched = watched.get(runId) ??\n { open: new Map(), delivered: HashSet.empty(), cursor: undefined }\n const fresh: Array<JournalEvent.Entry> = []\n let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor)\n while (true) {\n const page = yield* journal.entries({ runId, ...(after === undefined ? {} : { after }), limit: 512 })\n fresh.push(...page.entries)\n if (!page.hasMore || page.entries.length === 0) break\n after = page.entries.at(-1)!.seq\n }\n if (fresh.length === 0) return base\n\n // `open` is bounded by the policy's rules, so it is copied; the\n // delivery history is not, so it grows by one persistent node.\n const open = new Map(base.open)\n let delivered = base.delivered\n let cursor = base.cursor\n for (const entry of fresh) {\n cursor = entry.seq\n const payload = payloadRecord(entry.payload)\n if (payload === undefined) continue\n if (entry.eventType === deliveredEventType) {\n const id = payload[\"alertId\"]\n // Only a record under the source the alerter writes counts, so\n // another producer's delivered record cannot suppress a page.\n // The journal does not bind a source to a writer.\n if (typeof id === \"string\" && entry.sourceId === recordSourceId(id, \"delivered\")) {\n delivered = HashSet.add(delivered, id)\n }\n }\n observe(checked, detectors, entry, payload, open)\n }\n const next: Watched = { open, delivered, cursor }\n // Published at commit, exactly as `NotificationQueue.load` publishes\n // its fold. A tick inside an enclosing transaction reads that\n // transaction's uncommitted rows, and caching them would outlive the\n // rollback that erased them: the run would page about a condition\n // durable history never held.\n yield* journal.whenCommitted(Effect.sync(() => watched.put(runId, next)))\n return next\n })\n\n const record = (\n eventType: string,\n alert: Alert,\n source: string,\n outcome: { readonly code?: string | undefined; readonly status?: number | undefined }\n ): Effect.Effect<void, Journal.JournalError> =>\n journal.emitDurableUnfenced(\n new JournalEvent.Input({\n runId: JournalEvent.RunId.make(alert.runId),\n sourceId: JournalEvent.SourceId.make(recordSourceId(alertId(alert), source)),\n sourceSeq: JournalEvent.SourceSeq.make(0),\n // One record per alert per outcome. Without an explicit identity\n // the journal allocates a new sequence on every attempt, and a\n // webhook that stays down appends a row per tick forever.\n dedupe: \"identity\",\n eventType,\n payload: {\n runId: alert.runId,\n condition: alert.condition,\n since: alert.since,\n severity: alert.severity,\n alertId: alertId(alert),\n ...(outcome.code === undefined ? {} : { code: outcome.code }),\n ...(outcome.status === undefined ? {} : { status: outcome.status })\n }\n })\n ).pipe(Effect.asVoid)\n\n return AlertRuntime.of({\n tick: Effect.fn(\"AlertRuntime.tick\")(function*(runId: string) {\n const journalRunId = JournalEvent.RunId.make(runId)\n const history = yield* observed(journalRunId)\n const now = yield* Clock.currentTimeMillis\n const open = Array.from(history.open, ([condition, since]) => ({ runId, condition, since }))\n const raised = decide(checked, open, now)\n const delivered: Array<Alert> = []\n const failed: Array<Alert> = []\n const refused: Array<Alert> = []\n const suppressed: Array<Alert> = []\n for (const alert of raised) {\n if (HashSet.has(history.delivered, alertId(alert))) {\n suppressed.push(alert)\n continue\n }\n // Raised unchanged: a queue that rejects the admission outright is\n // not the journal failing, and calling it `sink_failed` would send\n // an operator looking at the webhook.\n const receipt = yield* queue.admit(runId, alertNotification(alert))\n if (receipt.decision === \"rejected-full\") {\n // The alert has nowhere durable to sit, so paging about it would\n // tell an operator about something the run will never read.\n yield* record(failedEventType, alert, \"refused\", { code: \"notification_full\" })\n refused.push(alert)\n continue\n }\n const outcome = yield* Effect.result(sink.deliver(alert))\n if (Result.isFailure(outcome)) {\n yield* record(failedEventType, alert, `failed/${outcome.failure.code}`, {\n code: outcome.failure.code,\n status: outcome.failure.status\n })\n failed.push(alert)\n } else {\n yield* record(deliveredEventType, alert, \"delivered\", {})\n delivered.push(alert)\n }\n }\n return { delivered, failed, refused, suppressed }\n })\n })\n })\n )\n"]}
|