@smthrs/notifications 0.0.0-stage → 1.0.0-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +127 -0
- package/LICENSE +21 -0
- package/README.md +93 -2
- package/dist/cjs/AlertPolicy.d.ts +284 -0
- package/dist/cjs/AlertPolicy.d.ts.map +1 -0
- package/dist/cjs/AlertPolicy.js +124 -0
- package/dist/cjs/AlertPolicy.js.map +7 -0
- package/dist/cjs/AlertRuntime.d.ts +90 -0
- package/dist/cjs/AlertRuntime.d.ts.map +1 -0
- package/dist/cjs/AlertRuntime.js +166 -0
- package/dist/cjs/AlertRuntime.js.map +7 -0
- package/dist/cjs/AlertSink.d.ts +101 -0
- package/dist/cjs/AlertSink.d.ts.map +1 -0
- package/dist/cjs/AlertSink.js +44 -0
- package/dist/cjs/AlertSink.js.map +7 -0
- package/dist/cjs/Alerts.d.ts +57 -0
- package/dist/cjs/Alerts.d.ts.map +1 -0
- package/dist/cjs/Alerts.js +46 -0
- package/dist/cjs/Alerts.js.map +7 -0
- package/dist/cjs/Notification.d.ts +214 -0
- package/dist/cjs/Notification.d.ts.map +1 -0
- package/dist/cjs/Notification.js +62 -0
- package/dist/cjs/Notification.js.map +7 -0
- package/dist/cjs/NotificationEvent.d.ts +214 -0
- package/dist/cjs/NotificationEvent.d.ts.map +1 -0
- package/dist/cjs/NotificationEvent.js +84 -0
- package/dist/cjs/NotificationEvent.js.map +7 -0
- package/dist/cjs/NotificationQueue.d.ts +194 -0
- package/dist/cjs/NotificationQueue.d.ts.map +1 -0
- package/dist/cjs/NotificationQueue.js +477 -0
- package/dist/cjs/NotificationQueue.js.map +7 -0
- package/dist/cjs/NotificationState.d.ts +169 -0
- package/dist/cjs/NotificationState.d.ts.map +1 -0
- package/dist/cjs/NotificationState.js +110 -0
- package/dist/cjs/NotificationState.js.map +7 -0
- package/dist/cjs/Projection.d.ts +30 -0
- package/dist/cjs/Projection.d.ts.map +1 -0
- package/dist/cjs/Projection.js +53 -0
- package/dist/cjs/Projection.js.map +7 -0
- package/dist/cjs/SteerPayload.d.ts +177 -0
- package/dist/cjs/SteerPayload.d.ts.map +1 -0
- package/dist/cjs/SteerPayload.js +61 -0
- package/dist/cjs/SteerPayload.js.map +7 -0
- package/dist/cjs/index.d.ts +41 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/foldCache.d.ts +36 -0
- package/dist/cjs/internal/foldCache.d.ts.map +1 -0
- package/dist/cjs/internal/foldCache.js +54 -0
- package/dist/cjs/internal/foldCache.js.map +7 -0
- package/dist/cjs/internal/replay.d.ts +22 -0
- package/dist/cjs/internal/replay.d.ts.map +1 -0
- package/dist/cjs/internal/replay.js +46 -0
- package/dist/cjs/internal/replay.js.map +7 -0
- package/dist/cjs/layerWebhook.d.ts +60 -0
- package/dist/cjs/layerWebhook.d.ts.map +1 -0
- package/dist/cjs/layerWebhook.js +145 -0
- package/dist/cjs/layerWebhook.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/esm/AlertPolicy.d.ts +284 -0
- package/dist/esm/AlertPolicy.d.ts.map +1 -0
- package/dist/esm/AlertPolicy.js +284 -0
- package/dist/esm/AlertPolicy.js.map +1 -0
- package/dist/esm/AlertRuntime.d.ts +90 -0
- package/dist/esm/AlertRuntime.d.ts.map +1 -0
- package/dist/esm/AlertRuntime.js +191 -0
- package/dist/esm/AlertRuntime.js.map +1 -0
- package/dist/esm/AlertSink.d.ts +101 -0
- package/dist/esm/AlertSink.d.ts.map +1 -0
- package/dist/esm/AlertSink.js +63 -0
- package/dist/esm/AlertSink.js.map +1 -0
- package/dist/esm/Alerts.d.ts +57 -0
- package/dist/esm/Alerts.d.ts.map +1 -0
- package/dist/esm/Alerts.js +57 -0
- package/dist/esm/Alerts.js.map +1 -0
- package/dist/esm/Notification.d.ts +214 -0
- package/dist/esm/Notification.d.ts.map +1 -0
- package/dist/esm/Notification.js +121 -0
- package/dist/esm/Notification.js.map +1 -0
- package/dist/esm/NotificationEvent.d.ts +214 -0
- package/dist/esm/NotificationEvent.d.ts.map +1 -0
- package/dist/esm/NotificationEvent.js +158 -0
- package/dist/esm/NotificationEvent.js.map +1 -0
- package/dist/esm/NotificationQueue.d.ts +194 -0
- package/dist/esm/NotificationQueue.d.ts.map +1 -0
- package/dist/esm/NotificationQueue.js +625 -0
- package/dist/esm/NotificationQueue.js.map +1 -0
- package/dist/esm/NotificationState.d.ts +169 -0
- package/dist/esm/NotificationState.d.ts.map +1 -0
- package/dist/esm/NotificationState.js +191 -0
- package/dist/esm/NotificationState.js.map +1 -0
- package/dist/esm/Projection.d.ts +30 -0
- package/dist/esm/Projection.d.ts.map +1 -0
- package/dist/esm/Projection.js +47 -0
- package/dist/esm/Projection.js.map +1 -0
- package/dist/esm/SteerPayload.d.ts +177 -0
- package/dist/esm/SteerPayload.d.ts.map +1 -0
- package/dist/esm/SteerPayload.js +139 -0
- package/dist/esm/SteerPayload.js.map +1 -0
- package/dist/esm/index.d.ts +41 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +41 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/foldCache.d.ts +36 -0
- package/dist/esm/internal/foldCache.d.ts.map +1 -0
- package/dist/esm/internal/foldCache.js +58 -0
- package/dist/esm/internal/foldCache.js.map +1 -0
- package/dist/esm/internal/replay.d.ts +22 -0
- package/dist/esm/internal/replay.d.ts.map +1 -0
- package/dist/esm/internal/replay.js +31 -0
- package/dist/esm/internal/replay.js.map +1 -0
- package/dist/esm/layerWebhook.d.ts +60 -0
- package/dist/esm/layerWebhook.d.ts.map +1 -0
- package/dist/esm/layerWebhook.js +186 -0
- package/dist/esm/layerWebhook.js.map +1 -0
- package/docs/README.md +146 -0
- package/docs/api.md +443 -0
- package/docs/concepts/admission-and-promotion.md +144 -0
- package/docs/concepts/alerting.md +138 -0
- package/docs/concepts/journal-records.md +132 -0
- package/docs/guides/alert-on-a-stuck-run.md +140 -0
- package/docs/guides/drain-at-a-turn-boundary.md +95 -0
- package/docs/guides/handle-a-full-queue.md +102 -0
- package/docs/guides/report-pending-notifications.md +92 -0
- package/docs/guides/send-alerts-to-a-webhook.md +155 -0
- package/docs/guides/steer-a-run.md +127 -0
- package/docs/guides/testing.md +161 -0
- package/docs/installation.md +123 -0
- package/docs/quickstart.md +159 -0
- package/docs/troubleshooting.md +254 -0
- package/package.json +182 -3
- package/src/AlertPolicy.ts +377 -0
- package/src/AlertRuntime.ts +280 -0
- package/src/AlertSink.ts +105 -0
- package/src/Alerts.ts +71 -0
- package/src/Notification.ts +171 -0
- package/src/NotificationEvent.ts +213 -0
- package/src/NotificationQueue.ts +856 -0
- package/src/NotificationState.ts +295 -0
- package/src/Projection.ts +48 -0
- package/src/SteerPayload.ts +198 -0
- package/src/index.ts +47 -0
- package/src/internal/foldCache.ts +72 -0
- package/src/internal/replay.ts +39 -0
- package/src/layerWebhook.ts +239 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "How alerting decides"
|
|
3
|
+
description: "Why an alert is a function of journal time rather than wall time, how a detector opens and closes a condition, and why delivery is at-least-once rather than exactly-once."
|
|
4
|
+
sidebar:
|
|
5
|
+
order: 3
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
The queue answers "tell this run something". An alert answers the question
|
|
9
|
+
nobody is around to ask: a run has been waiting an hour for an approval, and the
|
|
10
|
+
person who could grant it does not know.
|
|
11
|
+
|
|
12
|
+
The two are the same machinery. An alert is admitted as a coalesced system event
|
|
13
|
+
on the same durable queue, under the same idempotency and the same bound. They
|
|
14
|
+
differ only in who decides to write one.
|
|
15
|
+
|
|
16
|
+
## A condition is a payload field with a value
|
|
17
|
+
|
|
18
|
+
`Alerts.Detector` names a payload key, the value that means the condition holds,
|
|
19
|
+
and optionally the event types worth consulting at all:
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { Alerts } from "@smthrs/notifications"
|
|
23
|
+
|
|
24
|
+
const detector: Alerts.Detector = { field: "status", value: "waiting-approval" }
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
An entry that carries the field with the matching value **opens** the condition,
|
|
28
|
+
if it is not open already. An entry that carries the field with any other value
|
|
29
|
+
**closes** it. That symmetry is why a resume clears an approval alert without a
|
|
30
|
+
second vocabulary for "cleared": the run's next status entry says something
|
|
31
|
+
else, and the condition is gone.
|
|
32
|
+
|
|
33
|
+
An entry that does not carry the field at all is ignored entirely, which is what
|
|
34
|
+
stops a health report from closing an approval wait. `eventTypes` narrows
|
|
35
|
+
further, for a field two different producers write under different meanings. An
|
|
36
|
+
item ending in `.*`, such as `control.*`, names every event type with that
|
|
37
|
+
prefix.
|
|
38
|
+
|
|
39
|
+
`Alerts.defaultDetectors` ships four conditions, each of them one field and one
|
|
40
|
+
value: `waiting-approval` and `failed` read `status`, `stalled` reads `health`,
|
|
41
|
+
and `quota-parked` reads `waitingReason` for `quota`. Those are the fields
|
|
42
|
+
[`@smthrs/control`](/api/control) writes, and each default consults only the
|
|
43
|
+
`control.*` event types: a flow step or agent trace that writes a `status` field
|
|
44
|
+
into the same run's journal neither opens nor closes a default condition. A policy's own `detectors` are merged
|
|
45
|
+
over them, so a deployment that journals a different vocabulary names its own
|
|
46
|
+
rather than forking the module.
|
|
47
|
+
|
|
48
|
+
Only a condition the policy has a rule for is watched at all, and only own
|
|
49
|
+
properties count: a detector named `toString` reads no evidence off
|
|
50
|
+
`Object.prototype`.
|
|
51
|
+
|
|
52
|
+
## Journal time, not wall time
|
|
53
|
+
|
|
54
|
+
A condition's clock starts at `emittedAtMs` of the entry that opened it.
|
|
55
|
+
`Alerts.decide(policy, open, now)` compares that against the rule's `afterMs`,
|
|
56
|
+
and the alert it produces is stamped `firedAt: since + afterMs`.
|
|
57
|
+
|
|
58
|
+
`now` decides **whether** an alert is raised. It never appears in one. The same
|
|
59
|
+
journal therefore raises the same alerts in any process, at any instant past the
|
|
60
|
+
delay, and a tick that runs an hour late produces the byte-identical alert the
|
|
61
|
+
on-time tick would have.
|
|
62
|
+
|
|
63
|
+
That is not a stylistic choice. Retries re-admit the same alert id, and the
|
|
64
|
+
queue refuses a reused id whose content changed. An alert stamped with the
|
|
65
|
+
reading time would become permanently undeliverable the moment one tick was
|
|
66
|
+
refused and time moved on.
|
|
67
|
+
|
|
68
|
+
## The delay is the policy
|
|
69
|
+
|
|
70
|
+
`Alerts.Rule.afterMs` is how long a condition must last before it is worth
|
|
71
|
+
paging about. Most stalls clear themselves, and a policy that fired immediately
|
|
72
|
+
would page on every one of them. `afterMs` is a whole, non-negative number of
|
|
73
|
+
milliseconds, checked by the schema when `Alerts.layer` builds, so an impossible
|
|
74
|
+
delay fails the composition by name instead of mis-paging at 3am.
|
|
75
|
+
|
|
76
|
+
## Delivery is at-least-once
|
|
77
|
+
|
|
78
|
+
One tick, for each alert past its delay that has not already been delivered:
|
|
79
|
+
|
|
80
|
+
1. Admits the alert to the notification queue as a coalesced system event.
|
|
81
|
+
2. Calls `Alerts.SinkService.deliver`.
|
|
82
|
+
3. Journals `flows.alerts.delivered` once the sink accepted it.
|
|
83
|
+
|
|
84
|
+
The order is deliberate. Admission is idempotent, so a crash between steps 1 and
|
|
85
|
+
2 costs a duplicate admission the queue drops. Recording the delivery before the
|
|
86
|
+
send would turn a crash into a page nobody ever receives, which for an alert is
|
|
87
|
+
the worse failure.
|
|
88
|
+
|
|
89
|
+
The cost of that choice is a duplicate page: a process that dies between an
|
|
90
|
+
accepted send and the delivery record pages again on the next tick. So the sink
|
|
91
|
+
owns the last mile. `deliver` must be idempotent on `Alerts.alertId(alert)`,
|
|
92
|
+
which is `alert:<coalescingKey>:<since>` and stable for the life of one
|
|
93
|
+
condition. A condition that clears and re-opens gets a new id, because the second
|
|
94
|
+
approval wait is not the first one.
|
|
95
|
+
|
|
96
|
+
## Four outcomes, and only one is a failure
|
|
97
|
+
|
|
98
|
+
`Alerts.Tick` sorts every alert the tick considered:
|
|
99
|
+
|
|
100
|
+
| Field | What it means |
|
|
101
|
+
| ------------ | -------------------------------------------------------------------- |
|
|
102
|
+
| `delivered` | The sink accepted the page, and `flows.alerts.delivered` records it. |
|
|
103
|
+
| `suppressed` | A delivery record already exists. The sink was not called. |
|
|
104
|
+
| `failed` | The sink refused or could not be reached. Retried on the next tick. |
|
|
105
|
+
| `refused` | The notification queue was at capacity. The sink was not called. |
|
|
106
|
+
|
|
107
|
+
A refused page is not a delivered page, so it is retried. A refusal from the
|
|
108
|
+
queue pages nobody at all: the alert has nowhere durable to sit, and paging
|
|
109
|
+
about something the run will never read would send an operator looking in the
|
|
110
|
+
wrong place.
|
|
111
|
+
|
|
112
|
+
Both non-delivery outcomes journal `flows.alerts.failed`, one record per alert
|
|
113
|
+
per failure code, never one per tick. A webhook that stays down for an hour
|
|
114
|
+
leaves one row rather than a hundred and twenty every later tick has to read
|
|
115
|
+
past.
|
|
116
|
+
|
|
117
|
+
## The alerter never reads its own records
|
|
118
|
+
|
|
119
|
+
`flows.alerts.delivered` and `flows.alerts.failed` are written into the same
|
|
120
|
+
journal the detectors read. They carry the alert's own vocabulary: a refusal
|
|
121
|
+
reports the answering HTTP `status`, which a detector watching the run's
|
|
122
|
+
`status` would read as the condition clearing. So both event types are excluded
|
|
123
|
+
from condition detection outright. Without that exclusion, a page would close the
|
|
124
|
+
condition it paged about, the next tick would re-open it, and a webhook that
|
|
125
|
+
answered 503 once would alert forever.
|
|
126
|
+
|
|
127
|
+
A delivery record suppresses a repeat page only under the source the alerter
|
|
128
|
+
writes it with, `/notifications/alerts/<alertId>/delivered`. Another producer's
|
|
129
|
+
record naming the same `alertId` under its own source is ignored. The journal
|
|
130
|
+
does not bind a source to a writer, so this scopes records; it does not
|
|
131
|
+
authenticate the writer.
|
|
132
|
+
|
|
133
|
+
## Next
|
|
134
|
+
|
|
135
|
+
- [Alert on a stuck run](../guides/alert-on-a-stuck-run.md): a policy, a tick,
|
|
136
|
+
and what each field of the result means.
|
|
137
|
+
- [Send alerts to a webhook](../guides/send-alerts-to-a-webhook.md): the shipped
|
|
138
|
+
sink, its idempotency header, and its timeout.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "The journal records"
|
|
3
|
+
description: "The two durable events this package owns, why their spelling is frozen, how a queue folds a run's history back into state, and what a projection over a shared journal may assume."
|
|
4
|
+
sidebar:
|
|
5
|
+
order: 2
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
The queue keeps no state of its own. Everything it knows comes from two journal
|
|
9
|
+
records, and rebuilding from them is what lets a second process answer for a run
|
|
10
|
+
the first one admitted to.
|
|
11
|
+
|
|
12
|
+
## Two event types
|
|
13
|
+
|
|
14
|
+
| Constant | Event type | What it records |
|
|
15
|
+
| ------------------------------------- | ------------------------------ | ------------------------------------------------------------- |
|
|
16
|
+
| `NotificationEvent.AdmittedEventType` | `flows/notifications/Admitted` | One notification and the decision the queue made about it. |
|
|
17
|
+
| `NotificationEvent.PromotedEventType` | `flows/notifications/Promoted` | Which notification ids one boundary of one lineage delivered. |
|
|
18
|
+
|
|
19
|
+
Both spellings are frozen wire format. The values are already durable in every
|
|
20
|
+
database that has run this package, and renaming them would silently stop
|
|
21
|
+
matching the projections that consumers key on. Treat both strings, and the JSON
|
|
22
|
+
stored under them, as fixed.
|
|
23
|
+
|
|
24
|
+
An `Admitted` payload is `{ notification, decision }`. A `Promoted` payload is
|
|
25
|
+
`{ boundary, targetLineageId, ids }`.
|
|
26
|
+
|
|
27
|
+
## Replay never re-decides
|
|
28
|
+
|
|
29
|
+
The decision is recorded because it is a fact, not a derivation. When a queue
|
|
30
|
+
folds history, `NotificationState.applyAdmission` applies the committed decision
|
|
31
|
+
whatever this process's state would have chosen. A run that filled up under a
|
|
32
|
+
capacity of 128 and is now folded by a layer built at 512 replays as what
|
|
33
|
+
happened, not as what would happen today.
|
|
34
|
+
|
|
35
|
+
`rejected-full` is the decision that is never written. The queue refuses a full
|
|
36
|
+
queue in the receipt alone, because a rejection recorded as an admission would
|
|
37
|
+
match on every later attempt and burn the id forever. The literal stays in
|
|
38
|
+
`NotificationEvent.AdmissionDecision` so that a reader is total over a record any
|
|
39
|
+
writer could have produced.
|
|
40
|
+
|
|
41
|
+
## Folding is incremental
|
|
42
|
+
|
|
43
|
+
A layer folds each run once and then pages only the entries committed since its
|
|
44
|
+
last read. It keeps at most 64 folds at a time, and a run evicted from that set
|
|
45
|
+
is folded again from the beginning on its next call. Reading a run's history
|
|
46
|
+
therefore costs what has been journaled since the previous call, not the run's
|
|
47
|
+
whole journal.
|
|
48
|
+
|
|
49
|
+
Eviction sacrifices the newest fold no later read has come back for, never the
|
|
50
|
+
fold an ordered sweep is about to want. Dropping the least recently folded run
|
|
51
|
+
instead drops the run the next read wants, so a supervisor polling 65 or more
|
|
52
|
+
runs in order would replay every journal from sequence zero on every pass. A
|
|
53
|
+
sweep of any width instead refolds a fixed handful of runs and pages the tail
|
|
54
|
+
of the rest. When every retained fold has been read a second time the oldest is
|
|
55
|
+
evicted and the reuse marks are cleared, so no fold holds a slot on one ancient
|
|
56
|
+
read forever.
|
|
57
|
+
|
|
58
|
+
A fold retains the run's pending state, the sequence it stopped at, and the
|
|
59
|
+
identities it has seen: a notification id with the fingerprint and sequence of
|
|
60
|
+
its admission, and a drain identity with the sequence of its record. It retains
|
|
61
|
+
no admitted payload and no drain record, so a run's memory does not grow with
|
|
62
|
+
the notifications it has already drained, and a queue of any capacity costs the
|
|
63
|
+
same per event however long the run has been running. The two paths that must
|
|
64
|
+
report an already committed notification, a replayed drain and the content
|
|
65
|
+
comparison for a legacy admission with no fingerprint, read the record back at
|
|
66
|
+
its sequence.
|
|
67
|
+
|
|
68
|
+
Eviction is always safe, and so is an interleaved fold: a fold reads the cached
|
|
69
|
+
value, pages what has been committed since, and writes a fresh, self-consistent
|
|
70
|
+
pair back, so the worst case is one extra page on the next call. No fold can
|
|
71
|
+
report a state the journal does not hold: a fold taken inside a transaction is
|
|
72
|
+
published through `Journal.whenCommitted`, so a rollback leaves behind neither
|
|
73
|
+
the condition it observed nor the cursor it stopped at.
|
|
74
|
+
|
|
75
|
+
## Foreign entries are not errors
|
|
76
|
+
|
|
77
|
+
Journals are shared. `NotificationEvent.fromEntry` answers `Option.none()` for
|
|
78
|
+
an entry this package does not own, and for an owned entry whose payload does
|
|
79
|
+
not decode, so a projection over a busy journal stays total instead of failing
|
|
80
|
+
on somebody else's record.
|
|
81
|
+
|
|
82
|
+
An event type alone does not make an entry owned, because any producer on a
|
|
83
|
+
shared journal can choose one. An `Admitted` entry counts only under
|
|
84
|
+
`NotificationEvent.admissionSourceId` of the notification it carries, and a
|
|
85
|
+
`Promoted` entry only under a `/notifications/drain/` source. Another producer's
|
|
86
|
+
record with the same event type therefore stays out of the projection. The
|
|
87
|
+
check scopes records by source; it does not authenticate the writer. The journal
|
|
88
|
+
fences writes on run ownership, not on source, so a writer that sets this
|
|
89
|
+
package's source id is read as this package. The drain check is a prefix because
|
|
90
|
+
drain records written before the lineage joined the identity carry the boundary
|
|
91
|
+
alone.
|
|
92
|
+
|
|
93
|
+
`NotificationEvent.isAdmitted` and `NotificationEvent.isPromoted` tell the two
|
|
94
|
+
owned events apart. They are refinements over the decoded shape rather than a
|
|
95
|
+
stored discriminant, because a discriminant would be a durable field this
|
|
96
|
+
vocabulary does not have.
|
|
97
|
+
|
|
98
|
+
## Projections
|
|
99
|
+
|
|
100
|
+
`Projection.derive` is a `Journal.Projection` that replays the same fold and
|
|
101
|
+
emits the state after every entry:
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
import { Journal, JournalEvent } from "@smthrs/journal"
|
|
105
|
+
import { Projection } from "@smthrs/notifications"
|
|
106
|
+
import * as Effect from "effect/Effect"
|
|
107
|
+
import * as Stream from "effect/Stream"
|
|
108
|
+
|
|
109
|
+
const watchPending = (runId: string) =>
|
|
110
|
+
Effect.flatMap(
|
|
111
|
+
Journal.Journal,
|
|
112
|
+
(journal) =>
|
|
113
|
+
journal.project(Projection.derive, { runId: JournalEvent.RunId.make(runId) }).pipe(
|
|
114
|
+
Stream.runForEach((state) => Effect.log(`${state.items.length} pending`))
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
`journal.project` replays the run's history and then follows its committed
|
|
120
|
+
tail, so the stream does not end on its own. That is what a projection is for:
|
|
121
|
+
a live view that stays correct as entries arrive. Bound it with `Stream.take`
|
|
122
|
+
when you want a snapshot, or fork it into a scope when you want a follower.
|
|
123
|
+
|
|
124
|
+
`Projection.derive` starts at `NotificationState.empty(NotificationState.defaultCapacity)`.
|
|
125
|
+
That bound is the initial state's, not the run's: replay applies committed
|
|
126
|
+
decisions rather than re-deciding, so a deployment that raised the bound with
|
|
127
|
+
`NotificationQueue.layerWith` sees every admission it wrote replayed, past 128.
|
|
128
|
+
No journal record carries the layer's bound, so the projected `capacity` stays
|
|
129
|
+
at the default and `items` is the field to read.
|
|
130
|
+
|
|
131
|
+
For the live answer rather than a replay, use `NotificationQueue.pending`; see
|
|
132
|
+
[Report what a run is waiting on](../guides/report-pending-notifications.md).
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Alert on a stuck run"
|
|
3
|
+
description: "Write an alert policy, add a detector for a condition your deployment journals itself, tick the runtime, and read the four outcomes it reports."
|
|
4
|
+
sidebar:
|
|
5
|
+
order: 5
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
An alert policy watches a run's journal for a condition that has stayed open too
|
|
9
|
+
long, admits it to the notification queue, and pages a sink.
|
|
10
|
+
|
|
11
|
+
## Write the policy
|
|
12
|
+
|
|
13
|
+
A policy is delays plus ownership. `rules` names the conditions to watch and how
|
|
14
|
+
long each one may last:
|
|
15
|
+
|
|
16
|
+
```ts
|
|
17
|
+
import { Alerts } from "@smthrs/notifications"
|
|
18
|
+
|
|
19
|
+
export const policy: Alerts.Policy = {
|
|
20
|
+
defaults: { severity: "warning", owner: "oncall" },
|
|
21
|
+
rules: {
|
|
22
|
+
"waiting-approval": { afterMs: 900_000, runbook: "https://runbook.example/approvals" },
|
|
23
|
+
stalled: { afterMs: 300_000, severity: "critical" }
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
`defaults` fills in what a rule leaves out, so the delay is stated per condition
|
|
29
|
+
and the ownership once. `afterMs` must be a whole, non-negative number of
|
|
30
|
+
milliseconds; `Alerts.layer` decodes the policy when it builds, so an impossible
|
|
31
|
+
delay fails the composition rather than mis-paging later.
|
|
32
|
+
|
|
33
|
+
`waiting-approval` and `stalled` are two of the four conditions
|
|
34
|
+
`Alerts.defaultDetectors` recognizes out of the box, alongside `failed` and
|
|
35
|
+
`quota-parked`. A rule for a condition with no detector raises nothing.
|
|
36
|
+
|
|
37
|
+
## Add a detector for your own condition
|
|
38
|
+
|
|
39
|
+
A condition is a payload field with a value. To alert on something your own
|
|
40
|
+
supervisor journals, name the field, the value, and the entries worth reading:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { Alerts } from "@smthrs/notifications"
|
|
44
|
+
|
|
45
|
+
export const wedged: Alerts.Policy = {
|
|
46
|
+
defaults: { severity: "warning", owner: "oncall" },
|
|
47
|
+
rules: { "wedged-node": { afterMs: 900_000, runbook: "https://runbook.example/wedged-runs" } },
|
|
48
|
+
detectors: {
|
|
49
|
+
"wedged-node": {
|
|
50
|
+
field: "health",
|
|
51
|
+
value: "wedged-node",
|
|
52
|
+
eventTypes: ["supervisor.heartbeat"]
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`eventTypes` matters more than it looks. Without it, every entry in the run's
|
|
59
|
+
journal carrying a `health` field is evidence, including entries some other
|
|
60
|
+
writer keys the same way for its own reasons, and any value other than
|
|
61
|
+
`wedged-node` closes the condition. One unrelated writer between two heartbeats
|
|
62
|
+
therefore restarts the clock, and a run that is wedged the whole time never
|
|
63
|
+
outlives the delay. Narrow to the entries that actually report the condition.
|
|
64
|
+
|
|
65
|
+
Your `detectors` are merged over `Alerts.defaultDetectors`, so naming one of the
|
|
66
|
+
four built-in conditions replaces its detector and leaves the rest alone.
|
|
67
|
+
|
|
68
|
+
## Compose the runtime
|
|
69
|
+
|
|
70
|
+
`Alerts.layer(policy)` needs the journal, the notification queue, and a sink:
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
import { Alerts, NotificationQueue } from "@smthrs/notifications"
|
|
74
|
+
import * as Layer from "effect/Layer"
|
|
75
|
+
|
|
76
|
+
export const alerting = Alerts.layer(policy).pipe(
|
|
77
|
+
Layer.provideMerge(Layer.mergeAll(NotificationQueue.layer, Alerts.layerNoop))
|
|
78
|
+
)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`Alerts.layerNoop` accepts every alert and sends it nowhere. The admission and
|
|
82
|
+
the delivery record still happen, so a deployment with no outbound channel keeps
|
|
83
|
+
the durable evidence of what it would have paged about, and keeps the detection
|
|
84
|
+
path exercised rather than switched off. For a real pager, see
|
|
85
|
+
[Send alerts to a webhook](./send-alerts-to-a-webhook.md).
|
|
86
|
+
|
|
87
|
+
## Tick it
|
|
88
|
+
|
|
89
|
+
One tick reads one run:
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
import { Alerts } from "@smthrs/notifications"
|
|
93
|
+
import * as Effect from "effect/Effect"
|
|
94
|
+
|
|
95
|
+
export const check = (runId: string) =>
|
|
96
|
+
Effect.gen(function*() {
|
|
97
|
+
const alerts = yield* Alerts.AlertRuntime
|
|
98
|
+
const tick = yield* alerts.tick(runId)
|
|
99
|
+
return {
|
|
100
|
+
paged: tick.delivered.map((alert) => alert.condition),
|
|
101
|
+
retrying: tick.failed.map((alert) => alert.condition),
|
|
102
|
+
noRoom: tick.refused.map((alert) => alert.condition),
|
|
103
|
+
alreadySent: tick.suppressed.map((alert) => alert.condition)
|
|
104
|
+
}
|
|
105
|
+
})
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Call it on whatever schedule suits the deployment. A tick costs the journal
|
|
109
|
+
entries committed since the previous one, because the runtime keeps each run's
|
|
110
|
+
folded history for the 64 most recently watched runs.
|
|
111
|
+
|
|
112
|
+
Only two things make `tick` fail. A `Journal.JournalError` is the alerter's own
|
|
113
|
+
record channel failing: it read no entries, or it could not write the delivery
|
|
114
|
+
record. A `NotificationError` is the notification queue rejecting the alert
|
|
115
|
+
outright, which is a producer or storage fault. A sink that refuses the page is
|
|
116
|
+
neither: it comes back in `failed` and is retried.
|
|
117
|
+
|
|
118
|
+
## Decide without a runtime
|
|
119
|
+
|
|
120
|
+
`Alerts.conditions` and `Alerts.decide` are pure, so a report can answer "what
|
|
121
|
+
would page, and when" without admitting anything:
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
import type { JournalEvent } from "@smthrs/journal"
|
|
125
|
+
import { Alerts } from "@smthrs/notifications"
|
|
126
|
+
|
|
127
|
+
export const wouldPage = (entries: ReadonlyArray<JournalEvent.Entry>, runId: string, now: number) =>
|
|
128
|
+
Alerts.decide(policy, Alerts.conditions(policy, runId, entries), now)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`now` decides whether an alert is raised and never appears in one: every field of
|
|
132
|
+
the result, `firedAt` included, comes from the journal. See
|
|
133
|
+
[How alerting decides](../concepts/alerting.md).
|
|
134
|
+
|
|
135
|
+
## A worked example
|
|
136
|
+
|
|
137
|
+
[`examples/src/38-monitor-and-alert.ts`](https://github.com/smithersai/smithers/blob/main/examples/src/38-monitor-and-alert.ts)
|
|
138
|
+
runs the whole supervision loop over one SQLite file: two durable runs, a monitor
|
|
139
|
+
that beats over them, and the same policy at a production delay and at a zero
|
|
140
|
+
delay, so the quiet case and the paging case are visible side by side.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Drain at a turn boundary"
|
|
3
|
+
description: "Ask the queue what one boundary may deliver: naming the boundary, setting the turn cutoff, reporting idleness, and reading a duplicate receipt."
|
|
4
|
+
sidebar:
|
|
5
|
+
order: 2
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
A run drains when it reaches a point where reading a new instruction is safe.
|
|
9
|
+
The queue decides what that boundary may deliver, records the decision, and
|
|
10
|
+
answers with it.
|
|
11
|
+
|
|
12
|
+
## Call drain at the safe point
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { NotificationQueue } from "@smthrs/notifications"
|
|
16
|
+
import * as Effect from "effect/Effect"
|
|
17
|
+
|
|
18
|
+
export const atTurnClose = (runId: string, lineageId: string, turn: number, turnOpenedAt: number) =>
|
|
19
|
+
Effect.gen(function*() {
|
|
20
|
+
const queue = yield* NotificationQueue.NotificationQueue
|
|
21
|
+
const receipt = yield* queue.drain({
|
|
22
|
+
runId,
|
|
23
|
+
targetLineageId: lineageId,
|
|
24
|
+
boundary: `turn-${turn}`,
|
|
25
|
+
wouldIdle: false,
|
|
26
|
+
cutoffSeq: turnOpenedAt
|
|
27
|
+
})
|
|
28
|
+
return receipt.notifications
|
|
29
|
+
})
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Each field decides something:
|
|
33
|
+
|
|
34
|
+
- **`boundary`** names this safe point. With `runId` and `targetLineageId` it
|
|
35
|
+
forms the drain identity, so the name must be stable for one boundary and
|
|
36
|
+
distinct between boundaries. A turn number is the usual choice.
|
|
37
|
+
- **`cutoffSeq`** is the journal sequence that opened the turn now closing. A
|
|
38
|
+
steer admitted after it is held for the next boundary, which keeps a message
|
|
39
|
+
that arrived mid-turn out of the turn already in flight. Omit it only when you
|
|
40
|
+
have no turn to compare against; omitting it delivers everything pending for
|
|
41
|
+
the lineage.
|
|
42
|
+
- **`wouldIdle`** says whether the run would have nothing to do if this boundary
|
|
43
|
+
delivered nothing. When it is `true` and no steer was promoted, the drain also
|
|
44
|
+
promotes exactly one queued notification.
|
|
45
|
+
|
|
46
|
+
Legacy journal admissions with an `outside_change` payload remain pending at
|
|
47
|
+
coding park boundaries (`<frame>:<64-character lowercase hex digest>:park:<index>`).
|
|
48
|
+
An ordinary boundary promotes them at its cutoff even when `wouldIdle` is false.
|
|
49
|
+
New admissions of these reserved payloads are refused.
|
|
50
|
+
|
|
51
|
+
## Read the receipt
|
|
52
|
+
|
|
53
|
+
`DrainReceipt.notifications` are the notifications the committed promotion
|
|
54
|
+
record names, in admission order. Render them, and act on the steering items
|
|
55
|
+
they carry with `SteerPayload.decode`:
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import type * as NotificationQueue from "@smthrs/notifications/NotificationQueue"
|
|
59
|
+
import * as SteerPayload from "@smthrs/notifications/SteerPayload"
|
|
60
|
+
|
|
61
|
+
const messages = (receipt: NotificationQueue.DrainReceipt) =>
|
|
62
|
+
receipt.notifications.flatMap((notification) => {
|
|
63
|
+
const item = SteerPayload.decode(notification)
|
|
64
|
+
return item?.kind === "Message" ? [item.body] : []
|
|
65
|
+
})
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`DrainReceipt.duplicate` is `true` when this boundary had already drained. The
|
|
69
|
+
notifications are still the right ones: they are read back from the committed
|
|
70
|
+
record rather than decided again, so two processes draining one boundary report
|
|
71
|
+
the same delivery instead of two divergent guesses. A resumed run that walks its
|
|
72
|
+
boundaries looking for the first one it has not consulted branches on exactly
|
|
73
|
+
this flag.
|
|
74
|
+
|
|
75
|
+
## Drain the lineage you are
|
|
76
|
+
|
|
77
|
+
`targetLineageId` is not decoration. Two lineages closing a turn under the same
|
|
78
|
+
boundary name are two separate drains, so a child lineage that passes its
|
|
79
|
+
parent's id drains the parent's notifications and leaves its own pending. Pass
|
|
80
|
+
the lineage the running code belongs to.
|
|
81
|
+
|
|
82
|
+
## The harness does this for you
|
|
83
|
+
|
|
84
|
+
[`@smthrs/harness`](/api/harness) already implements this loop. Its
|
|
85
|
+
`Notifications.layer({ runId, lineageId })` captures the queue as the harness
|
|
86
|
+
steering source, calls `drain` at each boundary, and folds the result into the
|
|
87
|
+
inserts, seat changes, and thinking changes the turn controller acts on. Write
|
|
88
|
+
your own drain only when you are building a host that is not that harness.
|
|
89
|
+
|
|
90
|
+
## Next
|
|
91
|
+
|
|
92
|
+
- [Handle a full queue](./handle-a-full-queue.md): the writer-side contract that
|
|
93
|
+
keeps the boundary from having nothing to read.
|
|
94
|
+
- [Report what a run is waiting on](./report-pending-notifications.md): the same
|
|
95
|
+
fold, without promoting anything.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Handle a full queue"
|
|
3
|
+
description: "Read AdmissionReceipt.decision, tell rejected-full apart from a failure, and retry the same notification once a boundary has drained."
|
|
4
|
+
sidebar:
|
|
5
|
+
order: 3
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
`admit` does not fail when the queue is full. It succeeds, and says so in the
|
|
9
|
+
receipt. A caller that treats the receipt as an acknowledgement loses the
|
|
10
|
+
notification in silence, which is the one failure this queue must never hide.
|
|
11
|
+
|
|
12
|
+
## Branch on the decision
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { NotificationQueue } from "@smthrs/notifications"
|
|
16
|
+
import type { Notification } from "@smthrs/notifications/Notification"
|
|
17
|
+
import * as Effect from "effect/Effect"
|
|
18
|
+
|
|
19
|
+
export const tell = (runId: string, notification: Notification) =>
|
|
20
|
+
Effect.gen(function*() {
|
|
21
|
+
const queue = yield* NotificationQueue.NotificationQueue
|
|
22
|
+
const receipt = yield* queue.admit(runId, notification)
|
|
23
|
+
switch (receipt.decision) {
|
|
24
|
+
case "admitted":
|
|
25
|
+
case "coalesced":
|
|
26
|
+
// The queue retained it. `receipt.seq` is the sequence it committed at.
|
|
27
|
+
return { retained: true as const, seq: receipt.seq }
|
|
28
|
+
case "rejected-full":
|
|
29
|
+
// Nothing was retained and nothing was journaled. The id is still free.
|
|
30
|
+
return { retained: false as const, seq: undefined }
|
|
31
|
+
}
|
|
32
|
+
})
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
| Decision | Retained | Journaled | `seq` |
|
|
36
|
+
| --------------- | -------- | --------- | ----------- |
|
|
37
|
+
| `admitted` | Yes | Yes | present |
|
|
38
|
+
| `coalesced` | Yes | Yes | present |
|
|
39
|
+
| `rejected-full` | No | No | `undefined` |
|
|
40
|
+
|
|
41
|
+
`receipt.seq` is absent exactly when nothing was written, so an absent sequence
|
|
42
|
+
and a `rejected-full` decision are the same fact reported twice.
|
|
43
|
+
|
|
44
|
+
## Retry the same notification, unchanged
|
|
45
|
+
|
|
46
|
+
Because nothing was journaled, the id stays admissible. Once a boundary drains
|
|
47
|
+
and the run drops below its capacity, the identical call succeeds:
|
|
48
|
+
|
|
49
|
+
```ts
|
|
50
|
+
import * as Effect from "effect/Effect"
|
|
51
|
+
import * as Schedule from "effect/Schedule"
|
|
52
|
+
|
|
53
|
+
export const tellEventually = (runId: string, notification: Notification) =>
|
|
54
|
+
Effect.gen(function*() {
|
|
55
|
+
const result = yield* tell(runId, notification)
|
|
56
|
+
if (result.retained) return result
|
|
57
|
+
return yield* Effect.fail("queue-full" as const)
|
|
58
|
+
}).pipe(Effect.retry({ schedule: Schedule.spaced("1 second"), times: 5 }))
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Retry with the same `id` and the same content. Changing either turns the retry
|
|
62
|
+
into a different notification, and changing only the content is refused as
|
|
63
|
+
`notification_id_reused`.
|
|
64
|
+
|
|
65
|
+
## Choose the bound deliberately
|
|
66
|
+
|
|
67
|
+
`NotificationState.defaultCapacity` is 128 pending notifications per run.
|
|
68
|
+
`NotificationQueue.layerWith({ capacity })` sets a different bound for one
|
|
69
|
+
composition:
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
import { NotificationQueue } from "@smthrs/notifications"
|
|
73
|
+
|
|
74
|
+
export const roomyQueue = NotificationQueue.layerWith({ capacity: 512 })
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Raising it makes a run absorb more before it refuses, and makes an undrained run
|
|
78
|
+
hold more. Two things follow from a non-default bound:
|
|
79
|
+
|
|
80
|
+
- `Projection.derive` replays every admission the run committed, whatever bound
|
|
81
|
+
it was built at, but the state it projects still carries the default
|
|
82
|
+
`capacity`. Read `items`, not `capacity`. See
|
|
83
|
+
[the journal records](../concepts/journal-records.md).
|
|
84
|
+
- A capacity that is not a finite number becomes zero, which refuses everything.
|
|
85
|
+
That is deliberate: a misconfigured bound fails loudly rather than retaining an
|
|
86
|
+
unbounded backlog.
|
|
87
|
+
|
|
88
|
+
## What a refusal is not
|
|
89
|
+
|
|
90
|
+
A `rejected-full` decision is not an error, and no `NotificationError` is
|
|
91
|
+
raised. The typed failures mean something else entirely:
|
|
92
|
+
`notification_unavailable` is the seam reporting that it serves nothing,
|
|
93
|
+
`notification_id_reused` is a producer bug, and `notification_invalid` is a value
|
|
94
|
+
that is not a notification. Storage problems arrive as a `Journal.JournalError`.
|
|
95
|
+
See [Troubleshooting](../troubleshooting.md).
|
|
96
|
+
|
|
97
|
+
## When the alerter is refused
|
|
98
|
+
|
|
99
|
+
`Alerts` implements exactly this contract. A tick whose admission is refused
|
|
100
|
+
returns the alert in `Alerts.Tick.refused`, journals the refusal, and does not
|
|
101
|
+
call the sink, because paging about an alert the run will never read would send
|
|
102
|
+
an operator looking in the wrong place. The next tick tries again.
|