@smthrs/notifications 0.0.0-stage → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +127 -0
- package/LICENSE +21 -0
- package/README.md +91 -2
- package/dist/cjs/AlertPolicy.d.ts +284 -0
- package/dist/cjs/AlertPolicy.d.ts.map +1 -0
- package/dist/cjs/AlertPolicy.js +124 -0
- package/dist/cjs/AlertPolicy.js.map +7 -0
- package/dist/cjs/AlertRuntime.d.ts +90 -0
- package/dist/cjs/AlertRuntime.d.ts.map +1 -0
- package/dist/cjs/AlertRuntime.js +166 -0
- package/dist/cjs/AlertRuntime.js.map +7 -0
- package/dist/cjs/AlertSink.d.ts +101 -0
- package/dist/cjs/AlertSink.d.ts.map +1 -0
- package/dist/cjs/AlertSink.js +44 -0
- package/dist/cjs/AlertSink.js.map +7 -0
- package/dist/cjs/Alerts.d.ts +57 -0
- package/dist/cjs/Alerts.d.ts.map +1 -0
- package/dist/cjs/Alerts.js +46 -0
- package/dist/cjs/Alerts.js.map +7 -0
- package/dist/cjs/Notification.d.ts +214 -0
- package/dist/cjs/Notification.d.ts.map +1 -0
- package/dist/cjs/Notification.js +62 -0
- package/dist/cjs/Notification.js.map +7 -0
- package/dist/cjs/NotificationEvent.d.ts +214 -0
- package/dist/cjs/NotificationEvent.d.ts.map +1 -0
- package/dist/cjs/NotificationEvent.js +84 -0
- package/dist/cjs/NotificationEvent.js.map +7 -0
- package/dist/cjs/NotificationQueue.d.ts +193 -0
- package/dist/cjs/NotificationQueue.d.ts.map +1 -0
- package/dist/cjs/NotificationQueue.js +478 -0
- package/dist/cjs/NotificationQueue.js.map +7 -0
- package/dist/cjs/NotificationState.d.ts +169 -0
- package/dist/cjs/NotificationState.d.ts.map +1 -0
- package/dist/cjs/NotificationState.js +110 -0
- package/dist/cjs/NotificationState.js.map +7 -0
- package/dist/cjs/Projection.d.ts +27 -0
- package/dist/cjs/Projection.d.ts.map +1 -0
- package/dist/cjs/Projection.js +54 -0
- package/dist/cjs/Projection.js.map +7 -0
- package/dist/cjs/SteerPayload.d.ts +177 -0
- package/dist/cjs/SteerPayload.d.ts.map +1 -0
- package/dist/cjs/SteerPayload.js +61 -0
- package/dist/cjs/SteerPayload.js.map +7 -0
- package/dist/cjs/index.d.ts +41 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/foldCache.d.ts +36 -0
- package/dist/cjs/internal/foldCache.d.ts.map +1 -0
- package/dist/cjs/internal/foldCache.js +54 -0
- package/dist/cjs/internal/foldCache.js.map +7 -0
- package/dist/cjs/layerWebhook.d.ts +53 -0
- package/dist/cjs/layerWebhook.d.ts.map +1 -0
- package/dist/cjs/layerWebhook.js +107 -0
- package/dist/cjs/layerWebhook.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/esm/AlertPolicy.d.ts +284 -0
- package/dist/esm/AlertPolicy.d.ts.map +1 -0
- package/dist/esm/AlertPolicy.js +284 -0
- package/dist/esm/AlertPolicy.js.map +1 -0
- package/dist/esm/AlertRuntime.d.ts +90 -0
- package/dist/esm/AlertRuntime.d.ts.map +1 -0
- package/dist/esm/AlertRuntime.js +191 -0
- package/dist/esm/AlertRuntime.js.map +1 -0
- package/dist/esm/AlertSink.d.ts +101 -0
- package/dist/esm/AlertSink.d.ts.map +1 -0
- package/dist/esm/AlertSink.js +63 -0
- package/dist/esm/AlertSink.js.map +1 -0
- package/dist/esm/Alerts.d.ts +57 -0
- package/dist/esm/Alerts.d.ts.map +1 -0
- package/dist/esm/Alerts.js +57 -0
- package/dist/esm/Alerts.js.map +1 -0
- package/dist/esm/Notification.d.ts +214 -0
- package/dist/esm/Notification.d.ts.map +1 -0
- package/dist/esm/Notification.js +121 -0
- package/dist/esm/Notification.js.map +1 -0
- package/dist/esm/NotificationEvent.d.ts +214 -0
- package/dist/esm/NotificationEvent.d.ts.map +1 -0
- package/dist/esm/NotificationEvent.js +158 -0
- package/dist/esm/NotificationEvent.js.map +1 -0
- package/dist/esm/NotificationQueue.d.ts +193 -0
- package/dist/esm/NotificationQueue.d.ts.map +1 -0
- package/dist/esm/NotificationQueue.js +626 -0
- package/dist/esm/NotificationQueue.js.map +1 -0
- package/dist/esm/NotificationState.d.ts +169 -0
- package/dist/esm/NotificationState.d.ts.map +1 -0
- package/dist/esm/NotificationState.js +191 -0
- package/dist/esm/NotificationState.js.map +1 -0
- package/dist/esm/Projection.d.ts +27 -0
- package/dist/esm/Projection.d.ts.map +1 -0
- package/dist/esm/Projection.js +43 -0
- package/dist/esm/Projection.js.map +1 -0
- package/dist/esm/SteerPayload.d.ts +177 -0
- package/dist/esm/SteerPayload.d.ts.map +1 -0
- package/dist/esm/SteerPayload.js +139 -0
- package/dist/esm/SteerPayload.js.map +1 -0
- package/dist/esm/index.d.ts +41 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +41 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/foldCache.d.ts +36 -0
- package/dist/esm/internal/foldCache.d.ts.map +1 -0
- package/dist/esm/internal/foldCache.js +58 -0
- package/dist/esm/internal/foldCache.js.map +1 -0
- package/dist/esm/layerWebhook.d.ts +53 -0
- package/dist/esm/layerWebhook.d.ts.map +1 -0
- package/dist/esm/layerWebhook.js +118 -0
- package/dist/esm/layerWebhook.js.map +1 -0
- package/docs/README.md +144 -0
- package/docs/api.md +439 -0
- package/docs/concepts/admission-and-promotion.md +144 -0
- package/docs/concepts/alerting.md +138 -0
- package/docs/concepts/journal-records.md +132 -0
- package/docs/guides/alert-on-a-stuck-run.md +140 -0
- package/docs/guides/drain-at-a-turn-boundary.md +95 -0
- package/docs/guides/handle-a-full-queue.md +102 -0
- package/docs/guides/report-pending-notifications.md +92 -0
- package/docs/guides/send-alerts-to-a-webhook.md +130 -0
- package/docs/guides/steer-a-run.md +127 -0
- package/docs/guides/testing.md +161 -0
- package/docs/installation.md +119 -0
- package/docs/quickstart.md +159 -0
- package/docs/troubleshooting.md +254 -0
- package/package.json +182 -3
- package/src/AlertPolicy.ts +377 -0
- package/src/AlertRuntime.ts +280 -0
- package/src/AlertSink.ts +105 -0
- package/src/Alerts.ts +71 -0
- package/src/Notification.ts +171 -0
- package/src/NotificationEvent.ts +213 -0
- package/src/NotificationQueue.ts +857 -0
- package/src/NotificationState.ts +295 -0
- package/src/Projection.ts +46 -0
- package/src/SteerPayload.ts +198 -0
- package/src/index.ts +47 -0
- package/src/internal/foldCache.ts +72 -0
- package/src/layerWebhook.ts +161 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The journal-backed alert runtime: one tick per run.
|
|
3
|
+
*
|
|
4
|
+
* A tick folds the run's journal into the conditions it leaves open, decides
|
|
5
|
+
* which have outlived their delay, admits each one as a coalesced system
|
|
6
|
+
* event, pages the sink, and journals what the sink did with it. The fold is
|
|
7
|
+
* retained per run, so a tick costs the entries committed since the previous
|
|
8
|
+
* one rather than the run's whole journal.
|
|
9
|
+
*
|
|
10
|
+
* The rules it enforces are `./AlertPolicy.ts` and the channel it pages over
|
|
11
|
+
* is `./AlertSink.ts`; neither knows this module exists.
|
|
12
|
+
*
|
|
13
|
+
* @since 1.0.0
|
|
14
|
+
*/
|
|
15
|
+
import { Journal } from "@smthrs/journal";
|
|
16
|
+
import { Context, Effect, Layer } from "effect";
|
|
17
|
+
import { type Alert, Policy } from "./AlertPolicy.ts";
|
|
18
|
+
import { Sink } from "./AlertSink.ts";
|
|
19
|
+
import { type NotificationError, NotificationQueue } from "./NotificationQueue.ts";
|
|
20
|
+
/**
|
|
21
|
+
* What one tick decided.
|
|
22
|
+
*
|
|
23
|
+
* @category models
|
|
24
|
+
* @since 0.1.0
|
|
25
|
+
*/
|
|
26
|
+
export interface Tick {
|
|
27
|
+
/** Alerts raised and delivered on this tick. */
|
|
28
|
+
readonly delivered: ReadonlyArray<Alert>;
|
|
29
|
+
/** Alerts the sink refused on this tick. They are retried on the next one. */
|
|
30
|
+
readonly failed: ReadonlyArray<Alert>;
|
|
31
|
+
/**
|
|
32
|
+
* Alerts the notification queue refused because the run is at capacity. The
|
|
33
|
+
* sink was not called for them, and they are retried on the next tick.
|
|
34
|
+
*/
|
|
35
|
+
readonly refused: ReadonlyArray<Alert>;
|
|
36
|
+
/** Alerts that had already been delivered, and were not delivered again. */
|
|
37
|
+
readonly suppressed: ReadonlyArray<Alert>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The alert runtime: one tick per run.
|
|
41
|
+
*
|
|
42
|
+
* @category services
|
|
43
|
+
* @since 0.1.0
|
|
44
|
+
*/
|
|
45
|
+
export interface RuntimeService {
|
|
46
|
+
/**
|
|
47
|
+
* Reads one run's journal and pages about whatever has waited too long.
|
|
48
|
+
*
|
|
49
|
+
* The two failures stay apart because they mean different things to an
|
|
50
|
+
* operator. A `JournalError` is the alerter's own record channel failing: it
|
|
51
|
+
* read no entries, or it could not write the delivery record. A
|
|
52
|
+
* `NotificationError` is the notification queue rejecting the alert outright,
|
|
53
|
+
* which is a producer or storage fault rather than a full queue. Neither is
|
|
54
|
+
* a sink failure: a refused page is journaled as `flows.alerts.failed` and
|
|
55
|
+
* retried on the next tick, and it comes back in {@link Tick.failed}. A queue
|
|
56
|
+
* at capacity is not a failure either; it comes back in {@link Tick.refused}.
|
|
57
|
+
*/
|
|
58
|
+
readonly tick: (runId: string) => Effect.Effect<Tick, Journal.JournalError | NotificationError>;
|
|
59
|
+
}
|
|
60
|
+
declare const AlertRuntime_base: Context.ServiceClass<AlertRuntime, "/notifications/AlertRuntime", RuntimeService>;
|
|
61
|
+
/**
|
|
62
|
+
* The {@link RuntimeService} tag.
|
|
63
|
+
*
|
|
64
|
+
* @category services
|
|
65
|
+
* @since 0.1.0
|
|
66
|
+
*/
|
|
67
|
+
export declare class AlertRuntime extends AlertRuntime_base {
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The alert runtime over one policy.
|
|
71
|
+
*
|
|
72
|
+
* A tick reads the run's journal, derives the open conditions, decides which
|
|
73
|
+
* have outlived their delay, and for each one that has not already been
|
|
74
|
+
* delivered: admits a coalesced system event, sends it to the sink, and
|
|
75
|
+
* journals the delivery. The admission comes first because it is idempotent
|
|
76
|
+
* and the send is not: a crash between them costs a duplicate admission, which
|
|
77
|
+
* the queue drops, while a crash between an accepted send and the delivery
|
|
78
|
+
* record costs a duplicate PAGE, which is why {@link SinkService.deliver} is
|
|
79
|
+
* required to dedupe on {@link alertId}.
|
|
80
|
+
*
|
|
81
|
+
* The policy is decoded when the layer is built, so a rule with an impossible
|
|
82
|
+
* delay fails the composition by name instead of mis-paging at 3am.
|
|
83
|
+
*
|
|
84
|
+
* @param policy the rules to enforce
|
|
85
|
+
* @category layers
|
|
86
|
+
* @since 0.1.0
|
|
87
|
+
*/
|
|
88
|
+
export declare const layer: (policy: Policy) => Layer.Layer<AlertRuntime, never, Journal.Journal | NotificationQueue | Sink>;
|
|
89
|
+
export {};
|
|
90
|
+
//# sourceMappingURL=AlertRuntime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertRuntime.d.ts","sourceRoot":"","sources":["../../src/AlertRuntime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,OAAO,EAAgB,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,EAAW,KAAK,EAAkB,MAAM,QAAQ,CAAA;AAC/E,OAAO,EACL,KAAK,KAAK,EAQV,MAAM,EAEP,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AAGrC,OAAO,EAAE,KAAK,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAElF;;;;;GAKG;AACH,MAAM,WAAW,IAAI;IACnB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACxC,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACrC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;IACtC,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,KAAK,CAAC,CAAA;CAC1C;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,CAAA;CAChG;;AAED;;;;;GAKG;AACH,qBAAa,YAAa,SAAQ,iBAEjC;CAAG;AA8CJ;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,KAAK,GAChB,QAAQ,MAAM,KACb,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAgI3E,CAAA"}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The journal-backed alert runtime: one tick per run.
|
|
3
|
+
*
|
|
4
|
+
* A tick folds the run's journal into the conditions it leaves open, decides
|
|
5
|
+
* which have outlived their delay, admits each one as a coalesced system
|
|
6
|
+
* event, pages the sink, and journals what the sink did with it. The fold is
|
|
7
|
+
* retained per run, so a tick costs the entries committed since the previous
|
|
8
|
+
* one rather than the run's whole journal.
|
|
9
|
+
*
|
|
10
|
+
* The rules it enforces are `./AlertPolicy.ts` and the channel it pages over
|
|
11
|
+
* is `./AlertSink.ts`; neither knows this module exists.
|
|
12
|
+
*
|
|
13
|
+
* @since 1.0.0
|
|
14
|
+
*/
|
|
15
|
+
import { Journal, JournalEvent } from "@smthrs/journal";
|
|
16
|
+
import { Clock, Context, Effect, HashSet, Layer, Result, Schema } from "effect";
|
|
17
|
+
import { alertId, decide, deliveredEventType, detectorsOf, failedEventType, observe, payloadRecord, Policy, recordSourceId } from "./AlertPolicy.js";
|
|
18
|
+
import { Sink } from "./AlertSink.js";
|
|
19
|
+
import * as FoldCache from "./internal/foldCache.js";
|
|
20
|
+
import { NotificationQueue } from "./NotificationQueue.js";
|
|
21
|
+
/**
|
|
22
|
+
* The {@link RuntimeService} tag.
|
|
23
|
+
*
|
|
24
|
+
* @category services
|
|
25
|
+
* @since 0.1.0
|
|
26
|
+
*/
|
|
27
|
+
export class AlertRuntime extends Context.Service()("/notifications/AlertRuntime") {
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* How many runs one alert runtime keeps folded at a time. The retention policy
|
|
31
|
+
* is scan-resistant, so a sweep of more runs than this reads journal tails for
|
|
32
|
+
* all but a fixed handful of them.
|
|
33
|
+
*/
|
|
34
|
+
const maximumWatchedRuns = 64;
|
|
35
|
+
const alertNotification = (alert) => ({
|
|
36
|
+
_tag: "system-event",
|
|
37
|
+
id: alertId(alert),
|
|
38
|
+
targetLineageId: alert.runId,
|
|
39
|
+
delivery: "queue",
|
|
40
|
+
coalescingKey: alert.coalescingKey,
|
|
41
|
+
provenance: {
|
|
42
|
+
sourceRunId: alert.runId,
|
|
43
|
+
sourceLineageId: alert.runId,
|
|
44
|
+
sourceTurn: 0,
|
|
45
|
+
sourceActor: "alerts"
|
|
46
|
+
},
|
|
47
|
+
payload: {
|
|
48
|
+
condition: alert.condition,
|
|
49
|
+
severity: alert.severity,
|
|
50
|
+
since: alert.since,
|
|
51
|
+
firedAt: alert.firedAt,
|
|
52
|
+
...(alert.owner === undefined ? {} : { owner: alert.owner }),
|
|
53
|
+
...(alert.runbook === undefined ? {} : { runbook: alert.runbook })
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
/**
|
|
57
|
+
* The alert runtime over one policy.
|
|
58
|
+
*
|
|
59
|
+
* A tick reads the run's journal, derives the open conditions, decides which
|
|
60
|
+
* have outlived their delay, and for each one that has not already been
|
|
61
|
+
* delivered: admits a coalesced system event, sends it to the sink, and
|
|
62
|
+
* journals the delivery. The admission comes first because it is idempotent
|
|
63
|
+
* and the send is not: a crash between them costs a duplicate admission, which
|
|
64
|
+
* the queue drops, while a crash between an accepted send and the delivery
|
|
65
|
+
* record costs a duplicate PAGE, which is why {@link SinkService.deliver} is
|
|
66
|
+
* required to dedupe on {@link alertId}.
|
|
67
|
+
*
|
|
68
|
+
* The policy is decoded when the layer is built, so a rule with an impossible
|
|
69
|
+
* delay fails the composition by name instead of mis-paging at 3am.
|
|
70
|
+
*
|
|
71
|
+
* @param policy the rules to enforce
|
|
72
|
+
* @category layers
|
|
73
|
+
* @since 0.1.0
|
|
74
|
+
*/
|
|
75
|
+
export const layer = (policy) => Layer.effect(AlertRuntime, Effect.gen(function* () {
|
|
76
|
+
const checked = yield* Effect.orDie(Schema.decodeUnknownEffect(Policy)(policy));
|
|
77
|
+
const detectors = detectorsOf(checked);
|
|
78
|
+
const journal = yield* Journal.Journal;
|
|
79
|
+
const queue = yield* NotificationQueue;
|
|
80
|
+
const sink = yield* Sink;
|
|
81
|
+
// Folded history per run, so a tick costs the entries committed since
|
|
82
|
+
// the previous one rather than the run's whole journal.
|
|
83
|
+
const watched = FoldCache.make(maximumWatchedRuns);
|
|
84
|
+
const observed = (runId) => Effect.gen(function* () {
|
|
85
|
+
const base = watched.get(runId) ??
|
|
86
|
+
{ open: new Map(), delivered: HashSet.empty(), cursor: undefined };
|
|
87
|
+
const fresh = [];
|
|
88
|
+
let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor);
|
|
89
|
+
while (true) {
|
|
90
|
+
const page = yield* journal.entries({ runId, ...(after === undefined ? {} : { after }), limit: 512 });
|
|
91
|
+
fresh.push(...page.entries);
|
|
92
|
+
if (!page.hasMore || page.entries.length === 0)
|
|
93
|
+
break;
|
|
94
|
+
after = page.entries.at(-1).seq;
|
|
95
|
+
}
|
|
96
|
+
if (fresh.length === 0)
|
|
97
|
+
return base;
|
|
98
|
+
// `open` is bounded by the policy's rules, so it is copied; the
|
|
99
|
+
// delivery history is not, so it grows by one persistent node.
|
|
100
|
+
const open = new Map(base.open);
|
|
101
|
+
let delivered = base.delivered;
|
|
102
|
+
let cursor = base.cursor;
|
|
103
|
+
for (const entry of fresh) {
|
|
104
|
+
cursor = entry.seq;
|
|
105
|
+
const payload = payloadRecord(entry.payload);
|
|
106
|
+
if (payload === undefined)
|
|
107
|
+
continue;
|
|
108
|
+
if (entry.eventType === deliveredEventType) {
|
|
109
|
+
const id = payload["alertId"];
|
|
110
|
+
// Only a record under the source the alerter writes counts, so
|
|
111
|
+
// another producer's delivered record cannot suppress a page.
|
|
112
|
+
// The journal does not bind a source to a writer.
|
|
113
|
+
if (typeof id === "string" && entry.sourceId === recordSourceId(id, "delivered")) {
|
|
114
|
+
delivered = HashSet.add(delivered, id);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
observe(checked, detectors, entry, payload, open);
|
|
118
|
+
}
|
|
119
|
+
const next = { open, delivered, cursor };
|
|
120
|
+
// Published at commit, exactly as `NotificationQueue.load` publishes
|
|
121
|
+
// its fold. A tick inside an enclosing transaction reads that
|
|
122
|
+
// transaction's uncommitted rows, and caching them would outlive the
|
|
123
|
+
// rollback that erased them: the run would page about a condition
|
|
124
|
+
// durable history never held.
|
|
125
|
+
yield* journal.whenCommitted(Effect.sync(() => watched.put(runId, next)));
|
|
126
|
+
return next;
|
|
127
|
+
});
|
|
128
|
+
const record = (eventType, alert, source, outcome) => journal.emitDurableUnfenced(new JournalEvent.Input({
|
|
129
|
+
runId: JournalEvent.RunId.make(alert.runId),
|
|
130
|
+
sourceId: JournalEvent.SourceId.make(recordSourceId(alertId(alert), source)),
|
|
131
|
+
sourceSeq: JournalEvent.SourceSeq.make(0),
|
|
132
|
+
// One record per alert per outcome. Without an explicit identity
|
|
133
|
+
// the journal allocates a new sequence on every attempt, and a
|
|
134
|
+
// webhook that stays down appends a row per tick forever.
|
|
135
|
+
dedupe: "identity",
|
|
136
|
+
eventType,
|
|
137
|
+
payload: {
|
|
138
|
+
runId: alert.runId,
|
|
139
|
+
condition: alert.condition,
|
|
140
|
+
since: alert.since,
|
|
141
|
+
severity: alert.severity,
|
|
142
|
+
alertId: alertId(alert),
|
|
143
|
+
...(outcome.code === undefined ? {} : { code: outcome.code }),
|
|
144
|
+
...(outcome.status === undefined ? {} : { status: outcome.status })
|
|
145
|
+
}
|
|
146
|
+
})).pipe(Effect.asVoid);
|
|
147
|
+
return AlertRuntime.of({
|
|
148
|
+
tick: Effect.fn("AlertRuntime.tick")(function* (runId) {
|
|
149
|
+
const journalRunId = JournalEvent.RunId.make(runId);
|
|
150
|
+
const history = yield* observed(journalRunId);
|
|
151
|
+
const now = yield* Clock.currentTimeMillis;
|
|
152
|
+
const open = Array.from(history.open, ([condition, since]) => ({ runId, condition, since }));
|
|
153
|
+
const raised = decide(checked, open, now);
|
|
154
|
+
const delivered = [];
|
|
155
|
+
const failed = [];
|
|
156
|
+
const refused = [];
|
|
157
|
+
const suppressed = [];
|
|
158
|
+
for (const alert of raised) {
|
|
159
|
+
if (HashSet.has(history.delivered, alertId(alert))) {
|
|
160
|
+
suppressed.push(alert);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
// Raised unchanged: a queue that rejects the admission outright is
|
|
164
|
+
// not the journal failing, and calling it `sink_failed` would send
|
|
165
|
+
// an operator looking at the webhook.
|
|
166
|
+
const receipt = yield* queue.admit(runId, alertNotification(alert));
|
|
167
|
+
if (receipt.decision === "rejected-full") {
|
|
168
|
+
// The alert has nowhere durable to sit, so paging about it would
|
|
169
|
+
// tell an operator about something the run will never read.
|
|
170
|
+
yield* record(failedEventType, alert, "refused", { code: "notification_full" });
|
|
171
|
+
refused.push(alert);
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const outcome = yield* Effect.result(sink.deliver(alert));
|
|
175
|
+
if (Result.isFailure(outcome)) {
|
|
176
|
+
yield* record(failedEventType, alert, `failed/${outcome.failure.code}`, {
|
|
177
|
+
code: outcome.failure.code,
|
|
178
|
+
status: outcome.failure.status
|
|
179
|
+
});
|
|
180
|
+
failed.push(alert);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
yield* record(deliveredEventType, alert, "delivered", {});
|
|
184
|
+
delivered.push(alert);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return { delivered, failed, refused, suppressed };
|
|
188
|
+
})
|
|
189
|
+
});
|
|
190
|
+
}));
|
|
191
|
+
//# sourceMappingURL=AlertRuntime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertRuntime.js","sourceRoot":"","sources":["../../src/AlertRuntime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/E,OAAO,EAEL,OAAO,EACP,MAAM,EACN,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,OAAO,EACP,aAAa,EACb,MAAM,EACN,cAAc,EACf,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AACrC,OAAO,KAAK,SAAS,MAAM,yBAAyB,CAAA;AAEpD,OAAO,EAA0B,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AA4ClF;;;;;GAKG;AACH,MAAM,OAAO,YAAa,SAAQ,OAAO,CAAC,OAAO,EAAgC,CAC/E,6BAA6B,CAC9B;CAAG;AAiBJ;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,MAAM,iBAAiB,GAAG,CAAC,KAAY,EAAkC,EAAE,CAAC,CAAC;IAC3E,IAAI,EAAE,cAAc;IACpB,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC;IAClB,eAAe,EAAE,KAAK,CAAC,KAAK;IAC5B,QAAQ,EAAE,OAAO;IACjB,aAAa,EAAE,KAAK,CAAC,aAAa;IAClC,UAAU,EAAE;QACV,WAAW,EAAE,KAAK,CAAC,KAAK;QACxB,eAAe,EAAE,KAAK,CAAC,KAAK;QAC5B,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,QAAQ;KACtB;IACD,OAAO,EAAE;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;KACnE;CACF,CAAC,CAAA;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CACnB,MAAc,EACgE,EAAE,CAChF,KAAK,CAAC,MAAM,CACV,YAAY,EACZ,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,iBAAiB,CAAA;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAA;IACxB,sEAAsE;IACtE,wDAAwD;IACxD,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAU,kBAAkB,CAAC,CAAA;IAE3D,MAAM,QAAQ,GAAG,CAAC,KAAyB,EAAgD,EAAE,CAC3F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,IAAI,GAAY,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YACtC,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;QACpE,MAAM,KAAK,GAA8B,EAAE,CAAA;QAC3C,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACtF,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YACrG,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;YAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAK;YACrD,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,CAAA;QAClC,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAEnC,gEAAgE;QAChE,+DAA+D;QAC/D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,SAAS,GAAG,IAAI,CAAC,SAAS,CAAA;QAC9B,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QACxB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,KAAK,CAAC,GAAG,CAAA;YAClB,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAC5C,IAAI,OAAO,KAAK,SAAS;gBAAE,SAAQ;YACnC,IAAI,KAAK,CAAC,SAAS,KAAK,kBAAkB,EAAE,CAAC;gBAC3C,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAA;gBAC7B,+DAA+D;gBAC/D,8DAA8D;gBAC9D,kDAAkD;gBAClD,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,KAAK,cAAc,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC;oBACjF,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;gBACxC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC;QACD,MAAM,IAAI,GAAY,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAA;QACjD,qEAAqE;QACrE,8DAA8D;QAC9D,qEAAqE;QACrE,kEAAkE;QAClE,8BAA8B;QAC9B,KAAK,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QACzE,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAA;IAEJ,MAAM,MAAM,GAAG,CACb,SAAiB,EACjB,KAAY,EACZ,MAAc,EACd,OAAqF,EAC1C,EAAE,CAC7C,OAAO,CAAC,mBAAmB,CACzB,IAAI,YAAY,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAC3C,QAAQ,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;QAC5E,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,iEAAiE;QACjE,+DAA+D;QAC/D,0DAA0D;QAC1D,MAAM,EAAE,UAAU;QAClB,SAAS;QACT,OAAO,EAAE;YACP,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;YACvB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SACpE;KACF,CAAC,CACH,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAEvB,OAAO,YAAY,CAAC,EAAE,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,EAAC,KAAa;YAC1D,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;YAC7C,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;YAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;YAC5F,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAA;YACzC,MAAM,SAAS,GAAiB,EAAE,CAAA;YAClC,MAAM,MAAM,GAAiB,EAAE,CAAA;YAC/B,MAAM,OAAO,GAAiB,EAAE,CAAA;YAChC,MAAM,UAAU,GAAiB,EAAE,CAAA;YACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACnD,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACtB,SAAQ;gBACV,CAAC;gBACD,mEAAmE;gBACnE,mEAAmE;gBACnE,sCAAsC;gBACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAA;gBACnE,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;oBACzC,iEAAiE;oBACjE,4DAA4D;oBAC5D,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAA;oBAC/E,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACnB,SAAQ;gBACV,CAAC;gBACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;gBACzD,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9B,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,UAAU,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE;wBACtE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;wBAC1B,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;qBAC/B,CAAC,CAAA;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACpB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,CAAC,MAAM,CAAC,kBAAkB,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;oBACzD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;QACnD,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAA","sourcesContent":["/**\n * The journal-backed alert runtime: one tick per run.\n *\n * A tick folds the run's journal into the conditions it leaves open, decides\n * which have outlived their delay, admits each one as a coalesced system\n * event, pages the sink, and journals what the sink did with it. The fold is\n * retained per run, so a tick costs the entries committed since the previous\n * one rather than the run's whole journal.\n *\n * The rules it enforces are `./AlertPolicy.ts` and the channel it pages over\n * is `./AlertSink.ts`; neither knows this module exists.\n *\n * @since 1.0.0\n */\n\nimport { Journal, JournalEvent } from \"@smthrs/journal\"\nimport { Clock, Context, Effect, HashSet, Layer, Result, Schema } from \"effect\"\nimport {\n type Alert,\n alertId,\n decide,\n deliveredEventType,\n detectorsOf,\n failedEventType,\n observe,\n payloadRecord,\n Policy,\n recordSourceId\n} from \"./AlertPolicy.ts\"\nimport { Sink } from \"./AlertSink.ts\"\nimport * as FoldCache from \"./internal/foldCache.ts\"\nimport type * as NotificationModel from \"./Notification.ts\"\nimport { type NotificationError, NotificationQueue } from \"./NotificationQueue.ts\"\n\n/**\n * What one tick decided.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tick {\n /** Alerts raised and delivered on this tick. */\n readonly delivered: ReadonlyArray<Alert>\n /** Alerts the sink refused on this tick. They are retried on the next one. */\n readonly failed: ReadonlyArray<Alert>\n /**\n * Alerts the notification queue refused because the run is at capacity. The\n * sink was not called for them, and they are retried on the next tick.\n */\n readonly refused: ReadonlyArray<Alert>\n /** Alerts that had already been delivered, and were not delivered again. */\n readonly suppressed: ReadonlyArray<Alert>\n}\n\n/**\n * The alert runtime: one tick per run.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface RuntimeService {\n /**\n * Reads one run's journal and pages about whatever has waited too long.\n *\n * The two failures stay apart because they mean different things to an\n * operator. A `JournalError` is the alerter's own record channel failing: it\n * read no entries, or it could not write the delivery record. A\n * `NotificationError` is the notification queue rejecting the alert outright,\n * which is a producer or storage fault rather than a full queue. Neither is\n * a sink failure: a refused page is journaled as `flows.alerts.failed` and\n * retried on the next tick, and it comes back in {@link Tick.failed}. A queue\n * at capacity is not a failure either; it comes back in {@link Tick.refused}.\n */\n readonly tick: (runId: string) => Effect.Effect<Tick, Journal.JournalError | NotificationError>\n}\n\n/**\n * The {@link RuntimeService} tag.\n *\n * @category services\n * @since 0.1.0\n */\nexport class AlertRuntime extends Context.Service<AlertRuntime, RuntimeService>()(\n \"/notifications/AlertRuntime\"\n) {}\n\n/**\n * One run's folded alert history: the conditions still open, the alerts\n * already paged, and the sequence the fold stopped at.\n */\ninterface Watched {\n readonly open: ReadonlyMap<string, number>\n /**\n * The alerts already paged, as a persistent set: a tick adds a node and\n * leaves the fold the previous tick published untouched, where copying the\n * set would cost the run's whole delivery history on every tick.\n */\n readonly delivered: HashSet.HashSet<string>\n readonly cursor: number | undefined\n}\n\n/**\n * How many runs one alert runtime keeps folded at a time. The retention policy\n * is scan-resistant, so a sweep of more runs than this reads journal tails for\n * all but a fixed handful of them.\n */\nconst maximumWatchedRuns = 64\n\nconst alertNotification = (alert: Alert): NotificationModel.Notification => ({\n _tag: \"system-event\",\n id: alertId(alert),\n targetLineageId: alert.runId,\n delivery: \"queue\",\n coalescingKey: alert.coalescingKey,\n provenance: {\n sourceRunId: alert.runId,\n sourceLineageId: alert.runId,\n sourceTurn: 0,\n sourceActor: \"alerts\"\n },\n payload: {\n condition: alert.condition,\n severity: alert.severity,\n since: alert.since,\n firedAt: alert.firedAt,\n ...(alert.owner === undefined ? {} : { owner: alert.owner }),\n ...(alert.runbook === undefined ? {} : { runbook: alert.runbook })\n }\n})\n\n/**\n * The alert runtime over one policy.\n *\n * A tick reads the run's journal, derives the open conditions, decides which\n * have outlived their delay, and for each one that has not already been\n * delivered: admits a coalesced system event, sends it to the sink, and\n * journals the delivery. The admission comes first because it is idempotent\n * and the send is not: a crash between them costs a duplicate admission, which\n * the queue drops, while a crash between an accepted send and the delivery\n * record costs a duplicate PAGE, which is why {@link SinkService.deliver} is\n * required to dedupe on {@link alertId}.\n *\n * The policy is decoded when the layer is built, so a rule with an impossible\n * delay fails the composition by name instead of mis-paging at 3am.\n *\n * @param policy the rules to enforce\n * @category layers\n * @since 0.1.0\n */\nexport const layer = (\n policy: Policy\n): Layer.Layer<AlertRuntime, never, Journal.Journal | NotificationQueue | Sink> =>\n Layer.effect(\n AlertRuntime,\n Effect.gen(function*() {\n const checked = yield* Effect.orDie(Schema.decodeUnknownEffect(Policy)(policy))\n const detectors = detectorsOf(checked)\n const journal = yield* Journal.Journal\n const queue = yield* NotificationQueue\n const sink = yield* Sink\n // Folded history per run, so a tick costs the entries committed since\n // the previous one rather than the run's whole journal.\n const watched = FoldCache.make<Watched>(maximumWatchedRuns)\n\n const observed = (runId: JournalEvent.RunId): Effect.Effect<Watched, Journal.JournalError> =>\n Effect.gen(function*() {\n const base: Watched = watched.get(runId) ??\n { open: new Map(), delivered: HashSet.empty(), cursor: undefined }\n const fresh: Array<JournalEvent.Entry> = []\n let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor)\n while (true) {\n const page = yield* journal.entries({ runId, ...(after === undefined ? {} : { after }), limit: 512 })\n fresh.push(...page.entries)\n if (!page.hasMore || page.entries.length === 0) break\n after = page.entries.at(-1)!.seq\n }\n if (fresh.length === 0) return base\n\n // `open` is bounded by the policy's rules, so it is copied; the\n // delivery history is not, so it grows by one persistent node.\n const open = new Map(base.open)\n let delivered = base.delivered\n let cursor = base.cursor\n for (const entry of fresh) {\n cursor = entry.seq\n const payload = payloadRecord(entry.payload)\n if (payload === undefined) continue\n if (entry.eventType === deliveredEventType) {\n const id = payload[\"alertId\"]\n // Only a record under the source the alerter writes counts, so\n // another producer's delivered record cannot suppress a page.\n // The journal does not bind a source to a writer.\n if (typeof id === \"string\" && entry.sourceId === recordSourceId(id, \"delivered\")) {\n delivered = HashSet.add(delivered, id)\n }\n }\n observe(checked, detectors, entry, payload, open)\n }\n const next: Watched = { open, delivered, cursor }\n // Published at commit, exactly as `NotificationQueue.load` publishes\n // its fold. A tick inside an enclosing transaction reads that\n // transaction's uncommitted rows, and caching them would outlive the\n // rollback that erased them: the run would page about a condition\n // durable history never held.\n yield* journal.whenCommitted(Effect.sync(() => watched.put(runId, next)))\n return next\n })\n\n const record = (\n eventType: string,\n alert: Alert,\n source: string,\n outcome: { readonly code?: string | undefined; readonly status?: number | undefined }\n ): Effect.Effect<void, Journal.JournalError> =>\n journal.emitDurableUnfenced(\n new JournalEvent.Input({\n runId: JournalEvent.RunId.make(alert.runId),\n sourceId: JournalEvent.SourceId.make(recordSourceId(alertId(alert), source)),\n sourceSeq: JournalEvent.SourceSeq.make(0),\n // One record per alert per outcome. Without an explicit identity\n // the journal allocates a new sequence on every attempt, and a\n // webhook that stays down appends a row per tick forever.\n dedupe: \"identity\",\n eventType,\n payload: {\n runId: alert.runId,\n condition: alert.condition,\n since: alert.since,\n severity: alert.severity,\n alertId: alertId(alert),\n ...(outcome.code === undefined ? {} : { code: outcome.code }),\n ...(outcome.status === undefined ? {} : { status: outcome.status })\n }\n })\n ).pipe(Effect.asVoid)\n\n return AlertRuntime.of({\n tick: Effect.fn(\"AlertRuntime.tick\")(function*(runId: string) {\n const journalRunId = JournalEvent.RunId.make(runId)\n const history = yield* observed(journalRunId)\n const now = yield* Clock.currentTimeMillis\n const open = Array.from(history.open, ([condition, since]) => ({ runId, condition, since }))\n const raised = decide(checked, open, now)\n const delivered: Array<Alert> = []\n const failed: Array<Alert> = []\n const refused: Array<Alert> = []\n const suppressed: Array<Alert> = []\n for (const alert of raised) {\n if (HashSet.has(history.delivered, alertId(alert))) {\n suppressed.push(alert)\n continue\n }\n // Raised unchanged: a queue that rejects the admission outright is\n // not the journal failing, and calling it `sink_failed` would send\n // an operator looking at the webhook.\n const receipt = yield* queue.admit(runId, alertNotification(alert))\n if (receipt.decision === \"rejected-full\") {\n // The alert has nowhere durable to sit, so paging about it would\n // tell an operator about something the run will never read.\n yield* record(failedEventType, alert, \"refused\", { code: \"notification_full\" })\n refused.push(alert)\n continue\n }\n const outcome = yield* Effect.result(sink.deliver(alert))\n if (Result.isFailure(outcome)) {\n yield* record(failedEventType, alert, `failed/${outcome.failure.code}`, {\n code: outcome.failure.code,\n status: outcome.failure.status\n })\n failed.push(alert)\n } else {\n yield* record(deliveredEventType, alert, \"delivered\", {})\n delivered.push(alert)\n }\n }\n return { delivered, failed, refused, suppressed }\n })\n })\n })\n )\n"]}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The alert sink port: where a raised alert is sent, and why a send failed.
|
|
3
|
+
*
|
|
4
|
+
* The port is a tag and an error so a deployment supplies the transport and
|
|
5
|
+
* the policy needs none. `./layerWebhook.ts` is the transport this package
|
|
6
|
+
* ships, and {@link layerNoop} is the one a test composes.
|
|
7
|
+
*
|
|
8
|
+
* @since 1.0.0
|
|
9
|
+
*/
|
|
10
|
+
import { Context, Effect, Layer, Schema } from "effect";
|
|
11
|
+
import type { Alert } from "./AlertPolicy.ts";
|
|
12
|
+
/**
|
|
13
|
+
* Why a sink did not take an alert.
|
|
14
|
+
*
|
|
15
|
+
* `sink_rejected` is an answer that refused the page, `sink_unreachable` a
|
|
16
|
+
* request that never got one, and `sink_timeout` one that got no answer inside
|
|
17
|
+
* the sink's bound. `sink_misconfigured` is an endpoint the sink may not POST
|
|
18
|
+
* to at all; it is raised when the layer is built, before any alert exists,
|
|
19
|
+
* so it never reaches a journal.
|
|
20
|
+
*
|
|
21
|
+
* @category models
|
|
22
|
+
* @since 1.0.0
|
|
23
|
+
*/
|
|
24
|
+
export declare const FailureCode: Schema.Literals<readonly ["sink_rejected", "sink_unreachable", "sink_timeout", "sink_misconfigured"]>;
|
|
25
|
+
/**
|
|
26
|
+
* Why a sink did not take an alert.
|
|
27
|
+
*
|
|
28
|
+
* @category models
|
|
29
|
+
* @since 1.0.0
|
|
30
|
+
*/
|
|
31
|
+
export type FailureCode = typeof FailureCode.Type;
|
|
32
|
+
declare const AlertError_base: Schema.Class<AlertError, Schema.TaggedStruct<"/notifications/AlertError", {
|
|
33
|
+
readonly code: Schema.withConstructorDefault<Schema.Literals<readonly ["sink_rejected", "sink_unreachable", "sink_timeout", "sink_misconfigured"]>>;
|
|
34
|
+
readonly message: Schema.String;
|
|
35
|
+
/** The HTTP status that refused the page, when the sink speaks HTTP. */
|
|
36
|
+
readonly status: Schema.optional<Schema.Int>;
|
|
37
|
+
/** A short, credential-free description of the transport failure. */
|
|
38
|
+
readonly reason: Schema.optional<Schema.String>;
|
|
39
|
+
}>, import("effect/Cause").YieldableError>;
|
|
40
|
+
/**
|
|
41
|
+
* A sink refused or could not take an alert.
|
|
42
|
+
*
|
|
43
|
+
* The error carries a code, the answering status when there was one, and a
|
|
44
|
+
* short reason. It deliberately holds no request: a webhook request carries
|
|
45
|
+
* the credential the deployment handed {@link layerWebhook}, and an error is
|
|
46
|
+
* logged, encoded, and journaled in places a credential must never reach.
|
|
47
|
+
*
|
|
48
|
+
* @category errors
|
|
49
|
+
* @since 0.1.0
|
|
50
|
+
*/
|
|
51
|
+
export declare class AlertError extends AlertError_base {
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Where a raised alert is sent.
|
|
55
|
+
*
|
|
56
|
+
* Injected rather than fixed, because who gets paged is a deployment's
|
|
57
|
+
* decision and the policy is not: the same rules feed a webhook in production
|
|
58
|
+
* and nothing at all in a test.
|
|
59
|
+
*
|
|
60
|
+
* @category services
|
|
61
|
+
* @since 0.1.0
|
|
62
|
+
*/
|
|
63
|
+
export interface SinkService {
|
|
64
|
+
/**
|
|
65
|
+
* Sends one alert, and does so idempotently on {@link alertId}.
|
|
66
|
+
*
|
|
67
|
+
* Delivery is at-least-once: the `flows.alerts.delivered` record is written
|
|
68
|
+
* after this effect succeeds, so a process that dies in between pages again
|
|
69
|
+
* on the next tick. Every field of the alert, {@link Alert.firedAt}
|
|
70
|
+
* included, is derived from the journal, so the same alert is byte-identical
|
|
71
|
+
* on every attempt and `alertId(alert)` is the deduplication key a receiving
|
|
72
|
+
* system should key on. A sink that cannot dedupe is a sink that will
|
|
73
|
+
* occasionally page twice about one condition.
|
|
74
|
+
*
|
|
75
|
+
* A failure means the page did not go out. Succeeding on a page that was
|
|
76
|
+
* dropped is the one thing this port must never do.
|
|
77
|
+
*/
|
|
78
|
+
readonly deliver: (alert: Alert) => Effect.Effect<void, AlertError>;
|
|
79
|
+
}
|
|
80
|
+
declare const Sink_base: Context.ServiceClass<Sink, "/notifications/AlertSink", SinkService>;
|
|
81
|
+
/**
|
|
82
|
+
* The {@link SinkService} tag.
|
|
83
|
+
*
|
|
84
|
+
* @category services
|
|
85
|
+
* @since 0.1.0
|
|
86
|
+
*/
|
|
87
|
+
export declare class Sink extends Sink_base {
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* A sink that accepts every alert and sends it nowhere.
|
|
91
|
+
*
|
|
92
|
+
* The admission and the delivery record still happen, so a composition without
|
|
93
|
+
* an outbound channel still has the durable evidence of what it would have
|
|
94
|
+
* paged about.
|
|
95
|
+
*
|
|
96
|
+
* @category layers
|
|
97
|
+
* @since 0.1.0
|
|
98
|
+
*/
|
|
99
|
+
export declare const layerNoop: Layer.Layer<Sink>;
|
|
100
|
+
export {};
|
|
101
|
+
//# sourceMappingURL=AlertSink.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertSink.d.ts","sourceRoot":"","sources":["../../src/AlertSink.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAE7C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,WAAW,uGAA+F,CAAA;AAEvH;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,WAAW,CAAC,IAAI,CAAA;;;;IAkB7C,wEAAwE;;IAExE,qEAAqE;;;AAlBzE;;;;;;;;;;GAUG;AACH,qBAAa,UAAW,SAAQ,eAU/B;CAAG;AAEJ;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IAC1B;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;CACpE;;AAED;;;;;GAKG;AACH,qBAAa,IAAK,SAAQ,SAAgE;CAAG;AAE7F;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAuD,CAAA"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The alert sink port: where a raised alert is sent, and why a send failed.
|
|
3
|
+
*
|
|
4
|
+
* The port is a tag and an error so a deployment supplies the transport and
|
|
5
|
+
* the policy needs none. `./layerWebhook.ts` is the transport this package
|
|
6
|
+
* ships, and {@link layerNoop} is the one a test composes.
|
|
7
|
+
*
|
|
8
|
+
* @since 1.0.0
|
|
9
|
+
*/
|
|
10
|
+
import { Context, Effect, Layer, Schema } from "effect";
|
|
11
|
+
/**
|
|
12
|
+
* Why a sink did not take an alert.
|
|
13
|
+
*
|
|
14
|
+
* `sink_rejected` is an answer that refused the page, `sink_unreachable` a
|
|
15
|
+
* request that never got one, and `sink_timeout` one that got no answer inside
|
|
16
|
+
* the sink's bound. `sink_misconfigured` is an endpoint the sink may not POST
|
|
17
|
+
* to at all; it is raised when the layer is built, before any alert exists,
|
|
18
|
+
* so it never reaches a journal.
|
|
19
|
+
*
|
|
20
|
+
* @category models
|
|
21
|
+
* @since 1.0.0
|
|
22
|
+
*/
|
|
23
|
+
export const FailureCode = Schema.Literals(["sink_rejected", "sink_unreachable", "sink_timeout", "sink_misconfigured"]);
|
|
24
|
+
/**
|
|
25
|
+
* A sink refused or could not take an alert.
|
|
26
|
+
*
|
|
27
|
+
* The error carries a code, the answering status when there was one, and a
|
|
28
|
+
* short reason. It deliberately holds no request: a webhook request carries
|
|
29
|
+
* the credential the deployment handed {@link layerWebhook}, and an error is
|
|
30
|
+
* logged, encoded, and journaled in places a credential must never reach.
|
|
31
|
+
*
|
|
32
|
+
* @category errors
|
|
33
|
+
* @since 0.1.0
|
|
34
|
+
*/
|
|
35
|
+
export class AlertError extends Schema.TaggedError()("/notifications/AlertError", {
|
|
36
|
+
code: FailureCode.pipe(Schema.withConstructorDefault(Effect.succeed("sink_rejected"))),
|
|
37
|
+
message: Schema.String,
|
|
38
|
+
/** The HTTP status that refused the page, when the sink speaks HTTP. */
|
|
39
|
+
status: Schema.optional(Schema.Int),
|
|
40
|
+
/** A short, credential-free description of the transport failure. */
|
|
41
|
+
reason: Schema.optional(Schema.String)
|
|
42
|
+
}) {
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The {@link SinkService} tag.
|
|
46
|
+
*
|
|
47
|
+
* @category services
|
|
48
|
+
* @since 0.1.0
|
|
49
|
+
*/
|
|
50
|
+
export class Sink extends Context.Service()("/notifications/AlertSink") {
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A sink that accepts every alert and sends it nowhere.
|
|
54
|
+
*
|
|
55
|
+
* The admission and the delivery record still happen, so a composition without
|
|
56
|
+
* an outbound channel still has the durable evidence of what it would have
|
|
57
|
+
* paged about.
|
|
58
|
+
*
|
|
59
|
+
* @category layers
|
|
60
|
+
* @since 0.1.0
|
|
61
|
+
*/
|
|
62
|
+
export const layerNoop = Layer.succeed(Sink)({ deliver: () => Effect.void });
|
|
63
|
+
//# sourceMappingURL=AlertSink.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AlertSink.js","sourceRoot":"","sources":["../../src/AlertSink.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAGvD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,eAAe,EAAE,kBAAkB,EAAE,cAAc,EAAE,oBAAoB,CAAC,CAAC,CAAA;AAUvH;;;;;;;;;;GAUG;AACH,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,WAAW,EAAc,CAC9D,2BAA2B,EAC3B;IACE,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,eAAwB,CAAC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,CAAC,MAAM;IACtB,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC;IACnC,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;CACvC,CACF;CAAG;AA8BJ;;;;;GAKG;AACH,MAAM,OAAO,IAAK,SAAQ,OAAO,CAAC,OAAO,EAAqB,CAAC,0BAA0B,CAAC;CAAG;AAE7F;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,SAAS,GAAsB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA","sourcesContent":["/**\n * The alert sink port: where a raised alert is sent, and why a send failed.\n *\n * The port is a tag and an error so a deployment supplies the transport and\n * the policy needs none. `./layerWebhook.ts` is the transport this package\n * ships, and {@link layerNoop} is the one a test composes.\n *\n * @since 1.0.0\n */\n\nimport { Context, Effect, Layer, Schema } from \"effect\"\nimport type { Alert } from \"./AlertPolicy.ts\"\n\n/**\n * Why a sink did not take an alert.\n *\n * `sink_rejected` is an answer that refused the page, `sink_unreachable` a\n * request that never got one, and `sink_timeout` one that got no answer inside\n * the sink's bound. `sink_misconfigured` is an endpoint the sink may not POST\n * to at all; it is raised when the layer is built, before any alert exists,\n * so it never reaches a journal.\n *\n * @category models\n * @since 1.0.0\n */\nexport const FailureCode = Schema.Literals([\"sink_rejected\", \"sink_unreachable\", \"sink_timeout\", \"sink_misconfigured\"])\n\n/**\n * Why a sink did not take an alert.\n *\n * @category models\n * @since 1.0.0\n */\nexport type FailureCode = typeof FailureCode.Type\n\n/**\n * A sink refused or could not take an alert.\n *\n * The error carries a code, the answering status when there was one, and a\n * short reason. It deliberately holds no request: a webhook request carries\n * the credential the deployment handed {@link layerWebhook}, and an error is\n * logged, encoded, and journaled in places a credential must never reach.\n *\n * @category errors\n * @since 0.1.0\n */\nexport class AlertError extends Schema.TaggedError<AlertError>()(\n \"/notifications/AlertError\",\n {\n code: FailureCode.pipe(Schema.withConstructorDefault(Effect.succeed(\"sink_rejected\" as const))),\n message: Schema.String,\n /** The HTTP status that refused the page, when the sink speaks HTTP. */\n status: Schema.optional(Schema.Int),\n /** A short, credential-free description of the transport failure. */\n reason: Schema.optional(Schema.String)\n }\n) {}\n\n/**\n * Where a raised alert is sent.\n *\n * Injected rather than fixed, because who gets paged is a deployment's\n * decision and the policy is not: the same rules feed a webhook in production\n * and nothing at all in a test.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface SinkService {\n /**\n * Sends one alert, and does so idempotently on {@link alertId}.\n *\n * Delivery is at-least-once: the `flows.alerts.delivered` record is written\n * after this effect succeeds, so a process that dies in between pages again\n * on the next tick. Every field of the alert, {@link Alert.firedAt}\n * included, is derived from the journal, so the same alert is byte-identical\n * on every attempt and `alertId(alert)` is the deduplication key a receiving\n * system should key on. A sink that cannot dedupe is a sink that will\n * occasionally page twice about one condition.\n *\n * A failure means the page did not go out. Succeeding on a page that was\n * dropped is the one thing this port must never do.\n */\n readonly deliver: (alert: Alert) => Effect.Effect<void, AlertError>\n}\n\n/**\n * The {@link SinkService} tag.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Sink extends Context.Service<Sink, SinkService>()(\"/notifications/AlertSink\") {}\n\n/**\n * A sink that accepts every alert and sends it nowhere.\n *\n * The admission and the delivery record still happen, so a composition without\n * an outbound channel still has the durable evidence of what it would have\n * paged about.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Sink> = Layer.succeed(Sink)({ deliver: () => Effect.void })\n"]}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Alert policy: run conditions that have lasted too long, turned into durable,
|
|
3
|
+
* coalesced, delivered-once notifications.
|
|
4
|
+
*
|
|
5
|
+
* A notification queue answers "tell this run something". An alert answers the
|
|
6
|
+
* question nobody is around to ask: a run has been waiting for an approval for
|
|
7
|
+
* an hour, and the person who could grant it does not know. The two are the
|
|
8
|
+
* same machinery, since an alert is admitted as a coalesced system event, and
|
|
9
|
+
* they differ only in who decides to write one.
|
|
10
|
+
*
|
|
11
|
+
* ## Journal time, not wall time
|
|
12
|
+
*
|
|
13
|
+
* A condition's clock starts at the journal entry that opened it, which makes
|
|
14
|
+
* the decision replayable: the same journal produces the same alerts, whatever
|
|
15
|
+
* process reads it and whenever. A restart re-derives every open condition
|
|
16
|
+
* from the entries rather than from a timer it lost.
|
|
17
|
+
*
|
|
18
|
+
* ## Delivered at least once
|
|
19
|
+
*
|
|
20
|
+
* The admission is idempotent on the notification id, so re-admitting the same
|
|
21
|
+
* alert writes nothing new, and `flows.alerts.delivered` is journaled AFTER
|
|
22
|
+
* the sink accepted the page and is checked before the sink is called again.
|
|
23
|
+
* That ordering is deliberate and it is the reason the guarantee is
|
|
24
|
+
* at-least-once rather than exactly-once: a process that dies between the
|
|
25
|
+
* accepted send and the delivery record has paged, left no evidence of it, and
|
|
26
|
+
* will page again on the next tick. Recording the delivery first turns the
|
|
27
|
+
* same crash into a page nobody ever receives, which for an alert is the worse
|
|
28
|
+
* failure.
|
|
29
|
+
*
|
|
30
|
+
* So the sink owns the last mile: {@link SinkService.deliver} MUST be
|
|
31
|
+
* idempotent on {@link alertId}, which is stable for the life of one condition
|
|
32
|
+
* and reaches the sink on every alert it is handed. A sink that fails journals
|
|
33
|
+
* `flows.alerts.failed` once per failure code and the alert is retried on the
|
|
34
|
+
* next tick, because a refused page is not a delivered page.
|
|
35
|
+
*
|
|
36
|
+
* ## What a refusal costs
|
|
37
|
+
*
|
|
38
|
+
* The queue can refuse the admission when a run already holds its capacity of
|
|
39
|
+
* pending notifications. A tick that is refused pages nobody: the alert comes
|
|
40
|
+
* back in {@link Tick.refused}, the refusal is journaled, and the next tick
|
|
41
|
+
* tries again once a boundary has drained. Calling the sink anyway would page
|
|
42
|
+
* about an alert the run will never see.
|
|
43
|
+
*
|
|
44
|
+
* ## Conditions are data
|
|
45
|
+
*
|
|
46
|
+
* A condition is a payload field with a value: `status` is `waiting-approval`,
|
|
47
|
+
* `health` is `stalled`. Those are the fields control-plane entries and monitor
|
|
48
|
+
* beats already carry, so a deployment that journals a different vocabulary
|
|
49
|
+
* supplies its own detectors instead of a fork of this module.
|
|
50
|
+
*
|
|
51
|
+
* @since 0.1.0
|
|
52
|
+
*/
|
|
53
|
+
export { type Alert, alertId, coalescingKey, conditions, decide, defaultDetectors, deliveredEventType, Detector, failedEventType, type Open, Policy, Rule, Severity } from "./AlertPolicy.ts";
|
|
54
|
+
export { AlertRuntime, layer, type RuntimeService, type Tick } from "./AlertRuntime.ts";
|
|
55
|
+
export { AlertError, FailureCode, layerNoop, Sink, type SinkService } from "./AlertSink.ts";
|
|
56
|
+
export { defaultWebhookTimeout, layerWebhook } from "./layerWebhook.ts";
|
|
57
|
+
//# sourceMappingURL=Alerts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Alerts.d.ts","sourceRoot":"","sources":["../../src/Alerts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,EACL,KAAK,KAAK,EACV,OAAO,EACP,aAAa,EACb,UAAU,EACV,MAAM,EACN,gBAAgB,EAChB,kBAAkB,EAClB,QAAQ,EACR,eAAe,EACf,KAAK,IAAI,EACT,MAAM,EACN,IAAI,EACJ,QAAQ,EACT,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,cAAc,EAAE,KAAK,IAAI,EAAE,MAAM,mBAAmB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC3F,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Alert policy: run conditions that have lasted too long, turned into durable,
|
|
3
|
+
* coalesced, delivered-once notifications.
|
|
4
|
+
*
|
|
5
|
+
* A notification queue answers "tell this run something". An alert answers the
|
|
6
|
+
* question nobody is around to ask: a run has been waiting for an approval for
|
|
7
|
+
* an hour, and the person who could grant it does not know. The two are the
|
|
8
|
+
* same machinery, since an alert is admitted as a coalesced system event, and
|
|
9
|
+
* they differ only in who decides to write one.
|
|
10
|
+
*
|
|
11
|
+
* ## Journal time, not wall time
|
|
12
|
+
*
|
|
13
|
+
* A condition's clock starts at the journal entry that opened it, which makes
|
|
14
|
+
* the decision replayable: the same journal produces the same alerts, whatever
|
|
15
|
+
* process reads it and whenever. A restart re-derives every open condition
|
|
16
|
+
* from the entries rather than from a timer it lost.
|
|
17
|
+
*
|
|
18
|
+
* ## Delivered at least once
|
|
19
|
+
*
|
|
20
|
+
* The admission is idempotent on the notification id, so re-admitting the same
|
|
21
|
+
* alert writes nothing new, and `flows.alerts.delivered` is journaled AFTER
|
|
22
|
+
* the sink accepted the page and is checked before the sink is called again.
|
|
23
|
+
* That ordering is deliberate and it is the reason the guarantee is
|
|
24
|
+
* at-least-once rather than exactly-once: a process that dies between the
|
|
25
|
+
* accepted send and the delivery record has paged, left no evidence of it, and
|
|
26
|
+
* will page again on the next tick. Recording the delivery first turns the
|
|
27
|
+
* same crash into a page nobody ever receives, which for an alert is the worse
|
|
28
|
+
* failure.
|
|
29
|
+
*
|
|
30
|
+
* So the sink owns the last mile: {@link SinkService.deliver} MUST be
|
|
31
|
+
* idempotent on {@link alertId}, which is stable for the life of one condition
|
|
32
|
+
* and reaches the sink on every alert it is handed. A sink that fails journals
|
|
33
|
+
* `flows.alerts.failed` once per failure code and the alert is retried on the
|
|
34
|
+
* next tick, because a refused page is not a delivered page.
|
|
35
|
+
*
|
|
36
|
+
* ## What a refusal costs
|
|
37
|
+
*
|
|
38
|
+
* The queue can refuse the admission when a run already holds its capacity of
|
|
39
|
+
* pending notifications. A tick that is refused pages nobody: the alert comes
|
|
40
|
+
* back in {@link Tick.refused}, the refusal is journaled, and the next tick
|
|
41
|
+
* tries again once a boundary has drained. Calling the sink anyway would page
|
|
42
|
+
* about an alert the run will never see.
|
|
43
|
+
*
|
|
44
|
+
* ## Conditions are data
|
|
45
|
+
*
|
|
46
|
+
* A condition is a payload field with a value: `status` is `waiting-approval`,
|
|
47
|
+
* `health` is `stalled`. Those are the fields control-plane entries and monitor
|
|
48
|
+
* beats already carry, so a deployment that journals a different vocabulary
|
|
49
|
+
* supplies its own detectors instead of a fork of this module.
|
|
50
|
+
*
|
|
51
|
+
* @since 0.1.0
|
|
52
|
+
*/
|
|
53
|
+
export { alertId, coalescingKey, conditions, decide, defaultDetectors, deliveredEventType, Detector, failedEventType, Policy, Rule, Severity } from "./AlertPolicy.js";
|
|
54
|
+
export { AlertRuntime, layer } from "./AlertRuntime.js";
|
|
55
|
+
export { AlertError, FailureCode, layerNoop, Sink } from "./AlertSink.js";
|
|
56
|
+
export { defaultWebhookTimeout, layerWebhook } from "./layerWebhook.js";
|
|
57
|
+
//# sourceMappingURL=Alerts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Alerts.js","sourceRoot":"","sources":["../../src/Alerts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,EAEL,OAAO,EACP,aAAa,EACb,UAAU,EACV,MAAM,EACN,gBAAgB,EAChB,kBAAkB,EAClB,QAAQ,EACR,eAAe,EAEf,MAAM,EACN,IAAI,EACJ,QAAQ,EACT,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,YAAY,EAAE,KAAK,EAAkC,MAAM,mBAAmB,CAAA;AACvF,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAoB,MAAM,gBAAgB,CAAA;AAC3F,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA","sourcesContent":["/**\n * Alert policy: run conditions that have lasted too long, turned into durable,\n * coalesced, delivered-once notifications.\n *\n * A notification queue answers \"tell this run something\". An alert answers the\n * question nobody is around to ask: a run has been waiting for an approval for\n * an hour, and the person who could grant it does not know. The two are the\n * same machinery, since an alert is admitted as a coalesced system event, and\n * they differ only in who decides to write one.\n *\n * ## Journal time, not wall time\n *\n * A condition's clock starts at the journal entry that opened it, which makes\n * the decision replayable: the same journal produces the same alerts, whatever\n * process reads it and whenever. A restart re-derives every open condition\n * from the entries rather than from a timer it lost.\n *\n * ## Delivered at least once\n *\n * The admission is idempotent on the notification id, so re-admitting the same\n * alert writes nothing new, and `flows.alerts.delivered` is journaled AFTER\n * the sink accepted the page and is checked before the sink is called again.\n * That ordering is deliberate and it is the reason the guarantee is\n * at-least-once rather than exactly-once: a process that dies between the\n * accepted send and the delivery record has paged, left no evidence of it, and\n * will page again on the next tick. Recording the delivery first turns the\n * same crash into a page nobody ever receives, which for an alert is the worse\n * failure.\n *\n * So the sink owns the last mile: {@link SinkService.deliver} MUST be\n * idempotent on {@link alertId}, which is stable for the life of one condition\n * and reaches the sink on every alert it is handed. A sink that fails journals\n * `flows.alerts.failed` once per failure code and the alert is retried on the\n * next tick, because a refused page is not a delivered page.\n *\n * ## What a refusal costs\n *\n * The queue can refuse the admission when a run already holds its capacity of\n * pending notifications. A tick that is refused pages nobody: the alert comes\n * back in {@link Tick.refused}, the refusal is journaled, and the next tick\n * tries again once a boundary has drained. Calling the sink anyway would page\n * about an alert the run will never see.\n *\n * ## Conditions are data\n *\n * A condition is a payload field with a value: `status` is `waiting-approval`,\n * `health` is `stalled`. Those are the fields control-plane entries and monitor\n * beats already carry, so a deployment that journals a different vocabulary\n * supplies its own detectors instead of a fork of this module.\n *\n * @since 0.1.0\n */\n\nexport {\n type Alert,\n alertId,\n coalescingKey,\n conditions,\n decide,\n defaultDetectors,\n deliveredEventType,\n Detector,\n failedEventType,\n type Open,\n Policy,\n Rule,\n Severity\n} from \"./AlertPolicy.ts\"\nexport { AlertRuntime, layer, type RuntimeService, type Tick } from \"./AlertRuntime.ts\"\nexport { AlertError, FailureCode, layerNoop, Sink, type SinkService } from \"./AlertSink.ts\"\nexport { defaultWebhookTimeout, layerWebhook } from \"./layerWebhook.ts\"\n"]}
|